From f4da522953de057237356527a8b629d830e0acae Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 19 Jul 2023 11:21:55 +0200 Subject: [PATCH 01/13] wip: data extensions editor without ql submodule --- .../data-extensions-editor-module.ts | 31 +++++++++++++++++++ .../data-extensions-editor-view.ts | 1 + .../external-api-usage-query.ts | 24 ++------------ 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index ea63d5fd9..8748f2a48 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -99,6 +99,36 @@ export class DataExtensionsEditorModule { return; } + // TODO: Copy the files to a temporary directory and install pack dependencies + + const queryDir = (await dir({ unsafeCleanup: true })).path; + + // TODO: Write both the application mode and framework mode query + const queryFile = join(queryDir, "FetchExternalApis.ql"); + await writeFile(queryFile, query[queryName], "utf8"); + + if (query.dependencies) { + for (const [filename, contents] of Object.entries( + query.dependencies, + )) { + const dependencyFile = join(queryDir, filename); + await writeFile(dependencyFile, contents, "utf8"); + } + } + + const syntheticQueryPack = { + name: "codeql/external-api-usage", + version: "0.0.0", + dependencies: { + [`codeql/${databaseItem.language}-all`]: "*", + }, + }; + + const qlpackFile = join(queryDir, "codeql-pack.yml"); + await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8"); + + // TODO: install pack dependencies in temporary file (`codeql pack install`) + const view = new DataExtensionsEditorView( this.ctx, this.app, @@ -106,6 +136,7 @@ export class DataExtensionsEditorModule { this.cliServer, this.queryRunner, this.queryStorageDir, + queryDir, db, modelFile, ); diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 888893c94..79ea5bc16 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -71,6 +71,7 @@ export class DataExtensionsEditorView extends AbstractWebview< private readonly cliServer: CodeQLCliServer, private readonly queryRunner: QueryRunner, private readonly queryStorageDir: string, + private readonly queryDir: string, private readonly databaseItem: DatabaseItem, private readonly extensionPack: ExtensionPack, private mode: Mode = Mode.Application, diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index 254968618..e12b0eea1 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -35,6 +35,7 @@ export async function runQuery( databaseItem, queryStorageDir, progress, + queryDir, token, }: RunQueryOptions, ): Promise { @@ -63,27 +64,7 @@ export async function runQuery( return; } - const queryDir = (await dir({ unsafeCleanup: true })).path; - const queryFile = join(queryDir, "FetchExternalApis.ql"); - await writeFile(queryFile, query[queryName], "utf8"); - - if (query.dependencies) { - for (const [filename, contents] of Object.entries(query.dependencies)) { - const dependencyFile = join(queryDir, filename); - await writeFile(dependencyFile, contents, "utf8"); - } - } - - const syntheticQueryPack = { - name: "codeql/external-api-usage", - version: "0.0.0", - dependencies: { - [`codeql/${databaseItem.language}-all`]: "*", - }, - }; - - const qlpackFile = join(queryDir, "codeql-pack.yml"); - await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8"); + // TODO: install dependencies const additionalPacks = getOnDiskWorkspaceFolders(); const extensionPacks = Object.keys( @@ -93,6 +74,7 @@ export async function runQuery( const queryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, { + // TODO: select correct query file based on the `queryName` and `queryDir` queryPath: queryFile, quickEvalPosition: undefined, quickEvalCountOnly: false, From 9fd6cb8c1f4fbc00c204af62be3003b5f78b2ec0 Mon Sep 17 00:00:00 2001 From: Nora Date: Thu, 20 Jul 2023 15:23:03 +0000 Subject: [PATCH 02/13] Cleanup and install pack dependencies --- .../auto-model-usages-query.ts | 6 ++- .../data-extensions-editor-module.ts | 54 +++++++++++++++---- .../data-extensions-editor-view.ts | 24 ++++----- .../external-api-usage-query.ts | 38 ++++--------- 4 files changed, 70 insertions(+), 52 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts index e78c2395e..560015a8f 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts @@ -6,12 +6,14 @@ import { QueryRunner } from "../query-server"; import { DatabaseItem } from "../databases/local-databases"; import { interpretResultsSarif } from "../query-results"; import { ProgressCallback } from "../common/vscode/progress"; +import { Mode } from "./shared/mode"; type Options = { cliServer: CodeQLCliServer; queryRunner: QueryRunner; databaseItem: DatabaseItem; queryStorageDir: string; + queryDir: string; progress: ProgressCallback; }; @@ -23,6 +25,7 @@ export async function getAutoModelUsages({ queryRunner, databaseItem, queryStorageDir, + queryDir, progress, }: Options): Promise { const maxStep = 1500; @@ -32,11 +35,12 @@ export async function getAutoModelUsages({ // This will re-run the query that was already run when opening the data extensions editor. This // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we // need to interpret the results. - const queryResult = await runQuery("applicationModeQuery", { + const queryResult = await runQuery(Mode.Application, { cliServer, queryRunner, queryStorageDir, databaseItem, + queryDir, progress: (update) => progress({ maxStep, diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 8748f2a48..77f5141aa 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -4,12 +4,23 @@ import { DataExtensionsEditorCommands } from "../common/commands"; import { CliVersionConstraint, CodeQLCliServer } from "../codeql-cli/cli"; import { QueryRunner } from "../query-server"; import { DatabaseManager } from "../databases/local-databases"; -import { ensureDir } from "fs-extra"; +import { ensureDir, writeFile } from "fs-extra"; import { join } from "path"; import { App } from "../common/app"; import { withProgress } from "../common/vscode/progress"; import { pickExtensionPack } from "./extension-pack-picker"; -import { showAndLogErrorMessage } from "../common/logging"; +import { + showAndLogErrorMessage, + showAndLogExceptionWithTelemetry, +} from "../common/logging"; +import { dir } from "tmp-promise"; +import { dump as dumpYaml } from "js-yaml"; +import { fetchExternalApiQueries } from "./queries"; +import { telemetryListener } from "../common/vscode/telemetry"; +import { redactableError } from "../common/errors"; +import { extLogger } from "../common/logging/vscode"; +import { isQueryLanguage } from "../common/query-language"; +import { Mode } from "./shared/mode"; const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"]; @@ -99,13 +110,37 @@ export class DataExtensionsEditorModule { return; } - // TODO: Copy the files to a temporary directory and install pack dependencies - + // Create new temporary directory for query files and pack dependencies const queryDir = (await dir({ unsafeCleanup: true })).path; - // TODO: Write both the application mode and framework mode query - const queryFile = join(queryDir, "FetchExternalApis.ql"); - await writeFile(queryFile, query[queryName], "utf8"); + if (!isQueryLanguage(db.language)) { + void showAndLogExceptionWithTelemetry( + extLogger, + telemetryListener, + redactableError`Unsupported database language ${db.language}`, + ); + return; + } + + const query = fetchExternalApiQueries[db.language]; + if (!query) { + void showAndLogExceptionWithTelemetry( + extLogger, + telemetryListener, + redactableError`No external API usage query found for language ${db.language}`, + ); + return; + } + + Object.values(Mode).map(async (mode) => { + const queryFile = join( + queryDir, + `FetchExternalApis${ + mode.charAt(0).toUpperCase() + mode.slice(1) + }Mode.ql`, + ); + await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8"); + }); if (query.dependencies) { for (const [filename, contents] of Object.entries( @@ -120,14 +155,15 @@ export class DataExtensionsEditorModule { name: "codeql/external-api-usage", version: "0.0.0", dependencies: { - [`codeql/${databaseItem.language}-all`]: "*", + [`codeql/${db.language}-all`]: "*", }, }; const qlpackFile = join(queryDir, "codeql-pack.yml"); await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8"); - // TODO: install pack dependencies in temporary file (`codeql pack install`) + // TODO: test dependency installation + await this.cliServer.packInstall(queryDir); const view = new DataExtensionsEditorView( this.ctx, diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts index 79ea5bc16..f2b3e5901 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts @@ -249,19 +249,15 @@ export class DataExtensionsEditorView extends AbstractWebview< async (progress) => { try { const cancellationTokenSource = new CancellationTokenSource(); - const queryResult = await runQuery( - this.mode === Mode.Framework - ? "frameworkModeQuery" - : "applicationModeQuery", - { - cliServer: this.cliServer, - queryRunner: this.queryRunner, - databaseItem: this.databaseItem, - queryStorageDir: this.queryStorageDir, - progress: (update) => progress({ ...update, maxStep: 1500 }), - token: cancellationTokenSource.token, - }, - ); + const queryResult = await runQuery(this.mode, { + cliServer: this.cliServer, + queryRunner: this.queryRunner, + databaseItem: this.databaseItem, + queryStorageDir: this.queryStorageDir, + queryDir: this.queryDir, + progress: (update) => progress({ ...update, maxStep: 1500 }), + token: cancellationTokenSource.token, + }); if (!queryResult) { return; } @@ -433,6 +429,7 @@ export class DataExtensionsEditorView extends AbstractWebview< cliServer: this.cliServer, queryRunner: this.queryRunner, queryStorageDir: this.queryStorageDir, + queryDir: this.queryDir, databaseItem: this.databaseItem, progress: (update) => progress({ ...update, maxStep }), }); @@ -513,6 +510,7 @@ export class DataExtensionsEditorView extends AbstractWebview< this.cliServer, this.queryRunner, this.queryStorageDir, + this.queryDir, addedDatabase, modelFile, Mode.Framework, diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index e12b0eea1..038081788 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -1,41 +1,36 @@ import { CoreCompletedQuery, QueryRunner } from "../query-server"; -import { dir } from "tmp-promise"; -import { writeFile } from "fs-extra"; -import { dump as dumpYaml } from "js-yaml"; import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders"; import { extLogger } from "../common/logging/vscode"; import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging"; -import { isQueryLanguage } from "../common/query-language"; import { CancellationToken } from "vscode"; import { CodeQLCliServer } from "../codeql-cli/cli"; import { DatabaseItem } from "../databases/local-databases"; import { ProgressCallback } from "../common/vscode/progress"; -import { fetchExternalApiQueries } from "./queries"; import { QueryResultType } from "../query-server/new-messages"; -import { join } from "path"; import { redactableError } from "../common/errors"; import { telemetryListener } from "../common/vscode/telemetry"; -import { Query } from "./queries/query"; +import { join } from "path"; type RunQueryOptions = { cliServer: Pick; queryRunner: Pick; databaseItem: Pick; queryStorageDir: string; + queryDir: string; progress: ProgressCallback; token: CancellationToken; }; export async function runQuery( - queryName: keyof Omit, + mode: string, { cliServer, queryRunner, databaseItem, queryStorageDir, - progress, queryDir, + progress, token, }: RunQueryOptions, ): Promise { @@ -45,25 +40,6 @@ export async function runQuery( // For a reference of what this should do in the future, see the previous implementation in // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72 - if (!isQueryLanguage(databaseItem.language)) { - void showAndLogExceptionWithTelemetry( - extLogger, - telemetryListener, - redactableError`Unsupported database language ${databaseItem.language}`, - ); - return; - } - - const query = fetchExternalApiQueries[databaseItem.language]; - if (!query) { - void showAndLogExceptionWithTelemetry( - extLogger, - telemetryListener, - redactableError`No external API usage query found for language ${databaseItem.language}`, - ); - return; - } - // TODO: install dependencies const additionalPacks = getOnDiskWorkspaceFolders(); @@ -71,10 +47,14 @@ export async function runQuery( await cliServer.resolveQlpacks(additionalPacks, true), ); + const queryFile = join( + queryDir, + `FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`, + ); + const queryRun = queryRunner.createQueryRun( databaseItem.databaseUri.fsPath, { - // TODO: select correct query file based on the `queryName` and `queryDir` queryPath: queryFile, quickEvalPosition: undefined, quickEvalCountOnly: false, From 7c10447bb5fffabd045ee3e56147c712dedcc44b Mon Sep 17 00:00:00 2001 From: Nora Date: Fri, 28 Jul 2023 08:10:18 +0000 Subject: [PATCH 03/13] Add .github folder manually --- .../src/data-extensions-editor/extension-pack-picker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts b/extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts index c5188a88d..0f9da8554 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts @@ -43,6 +43,10 @@ export async function pickExtensionPack( // Get all existing extension packs in the workspace const additionalPacks = getOnDiskWorkspaceFolders(); + // the CLI doesn't check packs in the .github folder, so we need to add it manually + if (additionalPacks.length === 1) { + additionalPacks.push(`${additionalPacks[0]}/.github`); + } const extensionPacksInfo = await cliServer.resolveQlpacks( additionalPacks, true, From 558a70e3c846d0a4097efd67406f65a78dab4145 Mon Sep 17 00:00:00 2001 From: Nora Date: Mon, 31 Jul 2023 09:41:02 +0000 Subject: [PATCH 04/13] Adjust external api query test --- .../external-api-usage-query.test.ts | 55 ++++--------------- 1 file changed, 10 insertions(+), 45 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index e6449592e..fde06f387 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -4,11 +4,8 @@ import { } from "../../../../src/data-extensions-editor/external-api-usage-query"; import { createMockLogger } from "../../../__mocks__/loggerMock"; import { DatabaseKind } from "../../../../src/databases/local-databases"; -import { file } from "tmp-promise"; +import { dirSync, file } from "tmp-promise"; import { QueryResultType } from "../../../../src/query-server/new-messages"; -import { readdir, readFile } from "fs-extra"; -import { load } from "js-yaml"; -import { dirname, join } from "path"; import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries"; import * as log from "../../../../src/common/logging/notifications"; import { RedactableError } from "../../../../src/common/errors"; @@ -18,7 +15,11 @@ import { Query } from "../../../../src/data-extensions-editor/queries/query"; import { mockedUri } from "../../utils/mocking.helpers"; describe("runQuery", () => { - const cases = Object.keys(fetchExternalApiQueries).flatMap((lang) => { + const languages = Object.keys(fetchExternalApiQueries); + + const cases = languages.flatMap((lang) => { + const queryDir = dirSync({ unsafeCleanup: true }).name; + const query = fetchExternalApiQueries[lang as QueryLanguage]; if (!query) { return []; @@ -30,12 +31,13 @@ describe("runQuery", () => { return Array.from(keys).map((name) => ({ language: lang as QueryLanguage, queryName: name as keyof Omit, + queryDir, })); }); test.each(cases)( "should run $queryName for $language", - async ({ language, queryName }) => { + async ({ language, queryName, queryDir }) => { const logPath = (await file()).path; const query = fetchExternalApiQueries[language]; @@ -70,6 +72,7 @@ describe("runQuery", () => { language, }, queryStorageDir: "/tmp/queries", + queryDir, progress: jest.fn(), token: { isCancellationRequested: false, @@ -86,7 +89,7 @@ describe("runQuery", () => { expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( "/a/b/c/src.zip", { - queryPath: expect.stringMatching(/FetchExternalApis\.ql/), + queryPath: expect.stringMatching(/FetchExternalApis\S*\.ql/), quickEvalPosition: undefined, quickEvalCountOnly: false, }, @@ -97,44 +100,6 @@ describe("runQuery", () => { undefined, undefined, ); - - const queryPath = - options.queryRunner.createQueryRun.mock.calls[0][1].queryPath; - const queryDirectory = dirname(queryPath); - - const queryFiles = await readdir(queryDirectory); - expect(queryFiles.sort()).toEqual( - [ - "codeql-pack.yml", - "FetchExternalApis.ql", - "AutomodelVsCode.qll", - ].sort(), - ); - - const suiteFileContents = await readFile( - join(queryDirectory, "codeql-pack.yml"), - "utf8", - ); - const suiteYaml = load(suiteFileContents); - expect(suiteYaml).toEqual({ - name: "codeql/external-api-usage", - version: "0.0.0", - dependencies: { - [`codeql/${language}-all`]: "*", - }, - }); - - expect( - await readFile(join(queryDirectory, "FetchExternalApis.ql"), "utf8"), - ).toEqual(query[queryName]); - - for (const [filename, contents] of Object.entries( - query.dependencies ?? {}, - )) { - expect(await readFile(join(queryDirectory, filename), "utf8")).toEqual( - contents, - ); - } }, ); }); From 098437b463e3d57c4e389f32f857784b7f6ab213 Mon Sep 17 00:00:00 2001 From: Nora Date: Mon, 31 Jul 2023 09:41:12 +0000 Subject: [PATCH 05/13] Cleanup --- extensions/ql-vscode/AutomodelVsCode.qll | 0 .../src/data-extensions-editor/data-extensions-editor-module.ts | 1 - .../src/data-extensions-editor/external-api-usage-query.ts | 2 -- 3 files changed, 3 deletions(-) create mode 100644 extensions/ql-vscode/AutomodelVsCode.qll diff --git a/extensions/ql-vscode/AutomodelVsCode.qll b/extensions/ql-vscode/AutomodelVsCode.qll new file mode 100644 index 000000000..e69de29bb diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 77f5141aa..8082be63b 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -162,7 +162,6 @@ export class DataExtensionsEditorModule { const qlpackFile = join(queryDir, "codeql-pack.yml"); await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8"); - // TODO: test dependency installation await this.cliServer.packInstall(queryDir); const view = new DataExtensionsEditorView( diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index 038081788..d527dcf60 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -40,8 +40,6 @@ export async function runQuery( // For a reference of what this should do in the future, see the previous implementation in // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72 - // TODO: install dependencies - const additionalPacks = getOnDiskWorkspaceFolders(); const extensionPacks = Object.keys( await cliServer.resolveQlpacks(additionalPacks, true), From 1ee9cdaaddbdac36eb5b4abc1cfa4591c666db7a Mon Sep 17 00:00:00 2001 From: Nora Date: Mon, 31 Jul 2023 10:15:08 +0000 Subject: [PATCH 06/13] Add test for error log --- .../external-api-usage-query.test.ts | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index fde06f387..50cacb897 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -35,6 +35,61 @@ describe("runQuery", () => { })); }); + it("should log an error", async () => { + const showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction< + typeof showAndLogExceptionWithTelemetry + > = jest.spyOn(log, "showAndLogExceptionWithTelemetry"); + + const logPath = (await file()).path; + + const query = fetchExternalApiQueries[cases[0].language]; + if (!query) { + throw new Error(`No query found for language ${cases[0].language}`); + } + + const options = { + cliServer: { + resolveQlpacks: jest.fn().mockResolvedValue({ + "my/extensions": "/a/b/c/", + }), + }, + queryRunner: { + createQueryRun: jest.fn().mockReturnValue({ + evaluate: jest.fn().mockResolvedValue({ + resultType: QueryResultType.CANCELLATION, + }), + outputDir: { + logPath, + }, + }), + logger: createMockLogger(), + }, + databaseItem: { + databaseUri: mockedUri("/a/b/c/src.zip"), + contents: { + kind: DatabaseKind.Database, + name: "foo", + datasetUri: mockedUri(), + }, + language: cases[0].language, + }, + queryStorageDir: "/tmp/queries", + queryDir: cases[0].queryDir, + progress: jest.fn(), + token: { + isCancellationRequested: false, + onCancellationRequested: jest.fn(), + }, + }; + + expect(await runQuery(cases[0].queryName, options)).toBeUndefined(); + expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.any(RedactableError), + ); + }); + test.each(cases)( "should run $queryName for $language", async ({ language, queryName, queryDir }) => { From 4c0f68f193e54d07cd84b064bda7cba652dcd0c8 Mon Sep 17 00:00:00 2001 From: Nora Date: Mon, 31 Jul 2023 12:04:18 +0000 Subject: [PATCH 07/13] Adjust picker test --- .../data-extensions-editor/extension-pack-picker.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts index 481acb9e7..211ad07e4 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts @@ -99,7 +99,10 @@ describe("pickExtensionPack", () => { name: "codeql-custom-queries-java", index: 0, }; - additionalPacks = [Uri.file(tmpDir).fsPath]; + additionalPacks = [ + Uri.file(tmpDir).fsPath, + `${Uri.file(tmpDir).fsPath}/.github`, + ]; workspaceFoldersSpy = jest .spyOn(workspace, "workspaceFolders", "get") .mockReturnValue([workspaceFolder]); From 553e5cb4a10d68e667c49c8affc7e908127ab04b Mon Sep 17 00:00:00 2001 From: Nora Date: Tue, 1 Aug 2023 08:32:27 +0000 Subject: [PATCH 08/13] Remove automodel qll file --- extensions/ql-vscode/AutomodelVsCode.qll | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 extensions/ql-vscode/AutomodelVsCode.qll diff --git a/extensions/ql-vscode/AutomodelVsCode.qll b/extensions/ql-vscode/AutomodelVsCode.qll deleted file mode 100644 index e69de29bb..000000000 From 6e4641f2c123ca2a2c0ba4b08de3db5a7faf4ffc Mon Sep 17 00:00:00 2001 From: Nora Date: Tue, 1 Aug 2023 09:51:51 +0000 Subject: [PATCH 09/13] Clean up runQyery test --- .../external-api-usage-query.test.ts | 149 ++++++++---------- 1 file changed, 66 insertions(+), 83 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index 50cacb897..907e1cde4 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -11,29 +11,15 @@ import * as log from "../../../../src/common/logging/notifications"; import { RedactableError } from "../../../../src/common/errors"; import { showAndLogExceptionWithTelemetry } from "../../../../src/common/logging"; import { QueryLanguage } from "../../../../src/common/query-language"; -import { Query } from "../../../../src/data-extensions-editor/queries/query"; import { mockedUri } from "../../utils/mocking.helpers"; +import { Mode } from "../../../../src/data-extensions-editor/shared/mode"; describe("runQuery", () => { - const languages = Object.keys(fetchExternalApiQueries); + const language = Object.keys(fetchExternalApiQueries)[ + Math.floor(Math.random() * Object.keys(fetchExternalApiQueries).length) + ] as QueryLanguage; - const cases = languages.flatMap((lang) => { - const queryDir = dirSync({ unsafeCleanup: true }).name; - - const query = fetchExternalApiQueries[lang as QueryLanguage]; - if (!query) { - return []; - } - - const keys = new Set(Object.keys(query)); - keys.delete("dependencies"); - - return Array.from(keys).map((name) => ({ - language: lang as QueryLanguage, - queryName: name as keyof Omit, - queryDir, - })); - }); + const queryDir = dirSync({ unsafeCleanup: true }).name; it("should log an error", async () => { const showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction< @@ -42,9 +28,9 @@ describe("runQuery", () => { const logPath = (await file()).path; - const query = fetchExternalApiQueries[cases[0].language]; + const query = fetchExternalApiQueries[language]; if (!query) { - throw new Error(`No query found for language ${cases[0].language}`); + throw new Error(`No query found for language ${language}`); } const options = { @@ -71,10 +57,10 @@ describe("runQuery", () => { name: "foo", datasetUri: mockedUri(), }, - language: cases[0].language, + language, }, queryStorageDir: "/tmp/queries", - queryDir: cases[0].queryDir, + queryDir, progress: jest.fn(), token: { isCancellationRequested: false, @@ -82,7 +68,7 @@ describe("runQuery", () => { }, }; - expect(await runQuery(cases[0].queryName, options)).toBeUndefined(); + expect(await runQuery(Mode.Application, options)).toBeUndefined(); expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( expect.anything(), undefined, @@ -90,73 +76,70 @@ describe("runQuery", () => { ); }); - test.each(cases)( - "should run $queryName for $language", - async ({ language, queryName, queryDir }) => { - const logPath = (await file()).path; + it("should run query for random language", async () => { + const logPath = (await file()).path; - const query = fetchExternalApiQueries[language]; - if (!query) { - throw new Error(`No query found for language ${language}`); - } + const query = fetchExternalApiQueries[language]; + if (!query) { + throw new Error(`No query found for language ${language}`); + } - const options = { - cliServer: { - resolveQlpacks: jest.fn().mockResolvedValue({ - "my/extensions": "/a/b/c/", + const options = { + cliServer: { + resolveQlpacks: jest.fn().mockResolvedValue({ + "my/extensions": "/a/b/c/", + }), + }, + queryRunner: { + createQueryRun: jest.fn().mockReturnValue({ + evaluate: jest.fn().mockResolvedValue({ + resultType: QueryResultType.SUCCESS, }), - }, - queryRunner: { - createQueryRun: jest.fn().mockReturnValue({ - evaluate: jest.fn().mockResolvedValue({ - resultType: QueryResultType.SUCCESS, - }), - outputDir: { - logPath, - }, - }), - logger: createMockLogger(), - }, - databaseItem: { - databaseUri: mockedUri("/a/b/c/src.zip"), - contents: { - kind: DatabaseKind.Database, - name: "foo", - datasetUri: mockedUri(), + outputDir: { + logPath, }, - language, + }), + logger: createMockLogger(), + }, + databaseItem: { + databaseUri: mockedUri("/a/b/c/src.zip"), + contents: { + kind: DatabaseKind.Database, + name: "foo", + datasetUri: mockedUri(), }, - queryStorageDir: "/tmp/queries", - queryDir, - progress: jest.fn(), - token: { - isCancellationRequested: false, - onCancellationRequested: jest.fn(), - }, - }; + language, + }, + queryStorageDir: "/tmp/queries", + queryDir, + progress: jest.fn(), + token: { + isCancellationRequested: false, + onCancellationRequested: jest.fn(), + }, + }; - const result = await runQuery(queryName, options); + const result = await runQuery(Mode.Framework, options); - expect(result?.resultType).toEqual(QueryResultType.SUCCESS); + expect(result?.resultType).toEqual(QueryResultType.SUCCESS); - expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1); - expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true); - expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( - "/a/b/c/src.zip", - { - queryPath: expect.stringMatching(/FetchExternalApis\S*\.ql/), - quickEvalPosition: undefined, - quickEvalCountOnly: false, - }, - false, - [], - ["my/extensions"], - "/tmp/queries", - undefined, - undefined, - ); - }, - ); + expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1); + expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true); + expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( + "/a/b/c/src.zip", + { + queryPath: expect.stringMatching(/FetchExternalApis\S*\.ql/), + quickEvalPosition: undefined, + quickEvalCountOnly: false, + }, + false, + [], + ["my/extensions"], + "/tmp/queries", + undefined, + undefined, + ); + }); }); describe("readQueryResults", () => { From a39e55590a81629d6d50b28033eddb664f3059e4 Mon Sep 17 00:00:00 2001 From: Nora Date: Tue, 1 Aug 2023 09:52:16 +0000 Subject: [PATCH 10/13] Move query setup to query file --- .../data-extensions-editor-module.ts | 41 +++---------------- .../external-api-usage-query.ts | 39 +++++++++++++++++- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 8082be63b..3f0028afb 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -4,7 +4,7 @@ import { DataExtensionsEditorCommands } from "../common/commands"; import { CliVersionConstraint, CodeQLCliServer } from "../codeql-cli/cli"; import { QueryRunner } from "../query-server"; import { DatabaseManager } from "../databases/local-databases"; -import { ensureDir, writeFile } from "fs-extra"; +import { ensureDir } from "fs-extra"; import { join } from "path"; import { App } from "../common/app"; import { withProgress } from "../common/vscode/progress"; @@ -14,13 +14,12 @@ import { showAndLogExceptionWithTelemetry, } from "../common/logging"; import { dir } from "tmp-promise"; -import { dump as dumpYaml } from "js-yaml"; import { fetchExternalApiQueries } from "./queries"; import { telemetryListener } from "../common/vscode/telemetry"; import { redactableError } from "../common/errors"; import { extLogger } from "../common/logging/vscode"; import { isQueryLanguage } from "../common/query-language"; -import { Mode } from "./shared/mode"; +import { setUpPack } from "./external-api-usage-query"; const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"]; @@ -110,9 +109,6 @@ export class DataExtensionsEditorModule { return; } - // Create new temporary directory for query files and pack dependencies - const queryDir = (await dir({ unsafeCleanup: true })).path; - if (!isQueryLanguage(db.language)) { void showAndLogExceptionWithTelemetry( extLogger, @@ -132,36 +128,9 @@ export class DataExtensionsEditorModule { return; } - Object.values(Mode).map(async (mode) => { - const queryFile = join( - queryDir, - `FetchExternalApis${ - mode.charAt(0).toUpperCase() + mode.slice(1) - }Mode.ql`, - ); - await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8"); - }); - - if (query.dependencies) { - for (const [filename, contents] of Object.entries( - query.dependencies, - )) { - const dependencyFile = join(queryDir, filename); - await writeFile(dependencyFile, contents, "utf8"); - } - } - - const syntheticQueryPack = { - name: "codeql/external-api-usage", - version: "0.0.0", - dependencies: { - [`codeql/${db.language}-all`]: "*", - }, - }; - - const qlpackFile = join(queryDir, "codeql-pack.yml"); - await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8"); - + // Create new temporary directory for query files and pack dependencies + const queryDir = (await dir({ unsafeCleanup: true })).path; + await setUpPack(queryDir, query, db.language); await this.cliServer.packInstall(queryDir); const view = new DataExtensionsEditorView( diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts index d527dcf60..ae7f5e02e 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts @@ -10,6 +10,11 @@ import { QueryResultType } from "../query-server/new-messages"; import { redactableError } from "../common/errors"; import { telemetryListener } from "../common/vscode/telemetry"; import { join } from "path"; +import { Mode } from "./shared/mode"; +import { writeFile } from "fs-extra"; +import { Query } from "./queries/query"; +import { QueryLanguage } from "../common/query-language"; +import { dump } from "js-yaml"; type RunQueryOptions = { cliServer: Pick; @@ -22,8 +27,40 @@ type RunQueryOptions = { token: CancellationToken; }; +export async function setUpPack( + queryDir: string, + query: Query, + language: QueryLanguage, +) { + Object.values(Mode).map(async (mode) => { + const queryFile = join( + queryDir, + `FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`, + ); + await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8"); + }); + + if (query.dependencies) { + for (const [filename, contents] of Object.entries(query.dependencies)) { + const dependencyFile = join(queryDir, filename); + await writeFile(dependencyFile, contents, "utf8"); + } + } + + const syntheticQueryPack = { + name: "codeql/external-api-usage", + version: "0.0.0", + dependencies: { + [`codeql/${language}-all`]: "*", + }, + }; + + const qlpackFile = join(queryDir, "codeql-pack.yml"); + await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8"); +} + export async function runQuery( - mode: string, + mode: Mode, { cliServer, queryRunner, From fea45ea04d727d019950b8de6a978980364be8a6 Mon Sep 17 00:00:00 2001 From: Nora Date: Tue, 1 Aug 2023 09:53:47 +0000 Subject: [PATCH 11/13] wrap all describe blocks in external api usage test --- .../external-api-usage-query.test.ts | 468 +++++++++--------- 1 file changed, 235 insertions(+), 233 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index 907e1cde4..60d7b9ff6 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -14,249 +14,251 @@ import { QueryLanguage } from "../../../../src/common/query-language"; import { mockedUri } from "../../utils/mocking.helpers"; import { Mode } from "../../../../src/data-extensions-editor/shared/mode"; -describe("runQuery", () => { - const language = Object.keys(fetchExternalApiQueries)[ - Math.floor(Math.random() * Object.keys(fetchExternalApiQueries).length) - ] as QueryLanguage; +describe("external api usage query", () => { + describe("runQuery", () => { + const language = Object.keys(fetchExternalApiQueries)[ + Math.floor(Math.random() * Object.keys(fetchExternalApiQueries).length) + ] as QueryLanguage; - const queryDir = dirSync({ unsafeCleanup: true }).name; + const queryDir = dirSync({ unsafeCleanup: true }).name; - it("should log an error", async () => { - const showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction< - typeof showAndLogExceptionWithTelemetry - > = jest.spyOn(log, "showAndLogExceptionWithTelemetry"); + it("should log an error", async () => { + const showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction< + typeof showAndLogExceptionWithTelemetry + > = jest.spyOn(log, "showAndLogExceptionWithTelemetry"); - const logPath = (await file()).path; + const logPath = (await file()).path; - const query = fetchExternalApiQueries[language]; - if (!query) { - throw new Error(`No query found for language ${language}`); - } + const query = fetchExternalApiQueries[language]; + if (!query) { + throw new Error(`No query found for language ${language}`); + } - const options = { - cliServer: { - resolveQlpacks: jest.fn().mockResolvedValue({ - "my/extensions": "/a/b/c/", - }), - }, - queryRunner: { - createQueryRun: jest.fn().mockReturnValue({ - evaluate: jest.fn().mockResolvedValue({ - resultType: QueryResultType.CANCELLATION, + const options = { + cliServer: { + resolveQlpacks: jest.fn().mockResolvedValue({ + "my/extensions": "/a/b/c/", }), - outputDir: { - logPath, - }, - }), - logger: createMockLogger(), - }, - databaseItem: { - databaseUri: mockedUri("/a/b/c/src.zip"), - contents: { - kind: DatabaseKind.Database, - name: "foo", - datasetUri: mockedUri(), }, - language, - }, - queryStorageDir: "/tmp/queries", - queryDir, - progress: jest.fn(), - token: { - isCancellationRequested: false, - onCancellationRequested: jest.fn(), - }, - }; - - expect(await runQuery(Mode.Application, options)).toBeUndefined(); - expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( - expect.anything(), - undefined, - expect.any(RedactableError), - ); - }); - - it("should run query for random language", async () => { - const logPath = (await file()).path; - - const query = fetchExternalApiQueries[language]; - if (!query) { - throw new Error(`No query found for language ${language}`); - } - - const options = { - cliServer: { - resolveQlpacks: jest.fn().mockResolvedValue({ - "my/extensions": "/a/b/c/", - }), - }, - queryRunner: { - createQueryRun: jest.fn().mockReturnValue({ - evaluate: jest.fn().mockResolvedValue({ - resultType: QueryResultType.SUCCESS, - }), - outputDir: { - logPath, - }, - }), - logger: createMockLogger(), - }, - databaseItem: { - databaseUri: mockedUri("/a/b/c/src.zip"), - contents: { - kind: DatabaseKind.Database, - name: "foo", - datasetUri: mockedUri(), - }, - language, - }, - queryStorageDir: "/tmp/queries", - queryDir, - progress: jest.fn(), - token: { - isCancellationRequested: false, - onCancellationRequested: jest.fn(), - }, - }; - - const result = await runQuery(Mode.Framework, options); - - expect(result?.resultType).toEqual(QueryResultType.SUCCESS); - - expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1); - expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true); - expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( - "/a/b/c/src.zip", - { - queryPath: expect.stringMatching(/FetchExternalApis\S*\.ql/), - quickEvalPosition: undefined, - quickEvalCountOnly: false, - }, - false, - [], - ["my/extensions"], - "/tmp/queries", - undefined, - undefined, - ); - }); -}); - -describe("readQueryResults", () => { - const options = { - cliServer: { - bqrsInfo: jest.fn(), - bqrsDecode: jest.fn(), - }, - bqrsPath: "/tmp/results.bqrs", - }; - - let showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction< - typeof showAndLogExceptionWithTelemetry - >; - - beforeEach(() => { - showAndLogExceptionWithTelemetrySpy = jest.spyOn( - log, - "showAndLogExceptionWithTelemetry", - ); - }); - - it("returns undefined when there are no results", async () => { - options.cliServer.bqrsInfo.mockResolvedValue({ - "result-sets": [], - }); - - expect(await readQueryResults(options)).toBeUndefined(); - expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( - expect.anything(), - undefined, - expect.any(RedactableError), - ); - }); - - it("returns undefined when there are multiple result sets", async () => { - options.cliServer.bqrsInfo.mockResolvedValue({ - "result-sets": [ - { - name: "#select", - rows: 10, - columns: [ - { name: "usage", kind: "e" }, - { name: "apiName", kind: "s" }, - { kind: "s" }, - { kind: "s" }, - ], - }, - { - name: "#select2", - rows: 10, - columns: [ - { name: "usage", kind: "e" }, - { name: "apiName", kind: "s" }, - { kind: "s" }, - { kind: "s" }, - ], - }, - ], - }); - - expect(await readQueryResults(options)).toBeUndefined(); - expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( - expect.anything(), - undefined, - expect.any(RedactableError), - ); - }); - - it("gets the result set", async () => { - options.cliServer.bqrsInfo.mockResolvedValue({ - "result-sets": [ - { - name: "#select", - rows: 10, - columns: [ - { name: "usage", kind: "e" }, - { name: "apiName", kind: "s" }, - { kind: "s" }, - { kind: "s" }, - ], - }, - ], - "compatible-query-kinds": ["Table", "Tree", "Graph"], - }); - const decodedResultSet = { - columns: [ - { name: "usage", kind: "e" }, - { name: "apiName", kind: "s" }, - { kind: "s" }, - { kind: "s" }, - ], - tuples: [ - [ - "java.io.PrintStream#println(String)", - true, - { - label: "println(...)", - url: { - uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", - startLine: 29, - startColumn: 9, - endLine: 29, - endColumn: 49, + queryRunner: { + createQueryRun: jest.fn().mockReturnValue({ + evaluate: jest.fn().mockResolvedValue({ + resultType: QueryResultType.CANCELLATION, + }), + outputDir: { + logPath, }, + }), + logger: createMockLogger(), + }, + databaseItem: { + databaseUri: mockedUri("/a/b/c/src.zip"), + contents: { + kind: DatabaseKind.Database, + name: "foo", + datasetUri: mockedUri(), + }, + language, + }, + queryStorageDir: "/tmp/queries", + queryDir, + progress: jest.fn(), + token: { + isCancellationRequested: false, + onCancellationRequested: jest.fn(), + }, + }; + + expect(await runQuery(Mode.Application, options)).toBeUndefined(); + expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.any(RedactableError), + ); + }); + + it("should run query for random language", async () => { + const logPath = (await file()).path; + + const query = fetchExternalApiQueries[language]; + if (!query) { + throw new Error(`No query found for language ${language}`); + } + + const options = { + cliServer: { + resolveQlpacks: jest.fn().mockResolvedValue({ + "my/extensions": "/a/b/c/", + }), + }, + queryRunner: { + createQueryRun: jest.fn().mockReturnValue({ + evaluate: jest.fn().mockResolvedValue({ + resultType: QueryResultType.SUCCESS, + }), + outputDir: { + logPath, + }, + }), + logger: createMockLogger(), + }, + databaseItem: { + databaseUri: mockedUri("/a/b/c/src.zip"), + contents: { + kind: DatabaseKind.Database, + name: "foo", + datasetUri: mockedUri(), + }, + language, + }, + queryStorageDir: "/tmp/queries", + queryDir, + progress: jest.fn(), + token: { + isCancellationRequested: false, + onCancellationRequested: jest.fn(), + }, + }; + + const result = await runQuery(Mode.Framework, options); + + expect(result?.resultType).toEqual(QueryResultType.SUCCESS); + + expect(options.cliServer.resolveQlpacks).toHaveBeenCalledTimes(1); + expect(options.cliServer.resolveQlpacks).toHaveBeenCalledWith([], true); + expect(options.queryRunner.createQueryRun).toHaveBeenCalledWith( + "/a/b/c/src.zip", + { + queryPath: expect.stringMatching(/FetchExternalApis\S*\.ql/), + quickEvalPosition: undefined, + quickEvalCountOnly: false, + }, + false, + [], + ["my/extensions"], + "/tmp/queries", + undefined, + undefined, + ); + }); + }); + + describe("readQueryResults", () => { + const options = { + cliServer: { + bqrsInfo: jest.fn(), + bqrsDecode: jest.fn(), + }, + bqrsPath: "/tmp/results.bqrs", + }; + + let showAndLogExceptionWithTelemetrySpy: jest.SpiedFunction< + typeof showAndLogExceptionWithTelemetry + >; + + beforeEach(() => { + showAndLogExceptionWithTelemetrySpy = jest.spyOn( + log, + "showAndLogExceptionWithTelemetry", + ); + }); + + it("returns undefined when there are no results", async () => { + options.cliServer.bqrsInfo.mockResolvedValue({ + "result-sets": [], + }); + + expect(await readQueryResults(options)).toBeUndefined(); + expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.any(RedactableError), + ); + }); + + it("returns undefined when there are multiple result sets", async () => { + options.cliServer.bqrsInfo.mockResolvedValue({ + "result-sets": [ + { + name: "#select", + rows: 10, + columns: [ + { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, + ], + }, + { + name: "#select2", + rows: 10, + columns: [ + { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, + ], }, ], - ], - }; - options.cliServer.bqrsDecode.mockResolvedValue(decodedResultSet); + }); - const result = await readQueryResults(options); - expect(result).toEqual(decodedResultSet); - expect(options.cliServer.bqrsInfo).toHaveBeenCalledWith(options.bqrsPath); - expect(options.cliServer.bqrsDecode).toHaveBeenCalledWith( - options.bqrsPath, - "#select", - ); + expect(await readQueryResults(options)).toBeUndefined(); + expect(showAndLogExceptionWithTelemetrySpy).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.any(RedactableError), + ); + }); + + it("gets the result set", async () => { + options.cliServer.bqrsInfo.mockResolvedValue({ + "result-sets": [ + { + name: "#select", + rows: 10, + columns: [ + { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, + ], + }, + ], + "compatible-query-kinds": ["Table", "Tree", "Graph"], + }); + const decodedResultSet = { + columns: [ + { name: "usage", kind: "e" }, + { name: "apiName", kind: "s" }, + { kind: "s" }, + { kind: "s" }, + ], + tuples: [ + [ + "java.io.PrintStream#println(String)", + true, + { + label: "println(...)", + url: { + uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java", + startLine: 29, + startColumn: 9, + endLine: 29, + endColumn: 49, + }, + }, + ], + ], + }; + options.cliServer.bqrsDecode.mockResolvedValue(decodedResultSet); + + const result = await readQueryResults(options); + expect(result).toEqual(decodedResultSet); + expect(options.cliServer.bqrsInfo).toHaveBeenCalledWith(options.bqrsPath); + expect(options.cliServer.bqrsDecode).toHaveBeenCalledWith( + options.bqrsPath, + "#select", + ); + }); }); }); From 5f489212d4f201e2ab05b512f103319cf2514ce1 Mon Sep 17 00:00:00 2001 From: Nora Date: Tue, 1 Aug 2023 12:33:46 +0000 Subject: [PATCH 12/13] Add test for setup --- .../external-api-usage-query.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts index 60d7b9ff6..758e7c75e 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts @@ -1,6 +1,7 @@ import { readQueryResults, runQuery, + setUpPack, } from "../../../../src/data-extensions-editor/external-api-usage-query"; import { createMockLogger } from "../../../__mocks__/loggerMock"; import { DatabaseKind } from "../../../../src/databases/local-databases"; @@ -13,8 +14,75 @@ import { showAndLogExceptionWithTelemetry } from "../../../../src/common/logging import { QueryLanguage } from "../../../../src/common/query-language"; import { mockedUri } from "../../utils/mocking.helpers"; import { Mode } from "../../../../src/data-extensions-editor/shared/mode"; +import { readFile, readFileSync, readdir } from "fs-extra"; +import { join } from "path"; +import { load } from "js-yaml"; describe("external api usage query", () => { + describe("setUpPack", () => { + const languages = Object.keys(fetchExternalApiQueries).flatMap((lang) => { + const queryDir = dirSync({ unsafeCleanup: true }).name; + const query = fetchExternalApiQueries[lang as QueryLanguage]; + if (!query) { + return []; + } + + return { language: lang as QueryLanguage, queryDir, query }; + }); + + test.each(languages)( + "should create files for $language", + async ({ language, queryDir, query }) => { + await setUpPack(queryDir, query, language); + + const queryFiles = await readdir(queryDir); + expect(queryFiles.sort()).toEqual( + [ + "codeql-pack.yml", + "FetchExternalApisApplicationMode.ql", + "FetchExternalApisFrameworkMode.ql", + "AutomodelVsCode.qll", + ].sort(), + ); + + const suiteFileContents = await readFile( + join(queryDir, "codeql-pack.yml"), + "utf8", + ); + const suiteYaml = load(suiteFileContents); + expect(suiteYaml).toEqual({ + name: "codeql/external-api-usage", + version: "0.0.0", + dependencies: { + [`codeql/${language}-all`]: "*", + }, + }); + + Object.values(Mode).forEach((mode) => { + expect( + readFileSync( + join( + queryDir, + `FetchExternalApis${ + mode.charAt(0).toUpperCase() + mode.slice(1) + }Mode.ql`, + ), + "utf8", + ), + ).toEqual(query[`${mode}ModeQuery`]); + }); + + for (const [filename, contents] of Object.entries( + query.dependencies ?? {}, + )) { + expect(await readFile(join(queryDir, filename), "utf8")).toEqual( + contents, + ); + } + }, + ); + }); + describe("runQuery", () => { const language = Object.keys(fetchExternalApiQueries)[ Math.floor(Math.random() * Object.keys(fetchExternalApiQueries).length) From 868fae093d652f9d37ba5f75fb93a20f723e8483 Mon Sep 17 00:00:00 2001 From: Nora Date: Wed, 2 Aug 2023 14:25:39 +0000 Subject: [PATCH 13/13] Reorder db language check --- .../data-extensions-editor-module.ts | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts index 3f0028afb..cfe30c7dd 100644 --- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts +++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts @@ -70,10 +70,14 @@ export class DataExtensionsEditorModule { return; } - if (!SUPPORTED_LANGUAGES.includes(db.language)) { + const language = db.language; + if ( + !SUPPORTED_LANGUAGES.includes(language) || + !isQueryLanguage(language) + ) { void showAndLogErrorMessage( this.app.logger, - `The data extensions editor is not supported for ${db.language} databases.`, + `The data extensions editor is not supported for ${language} databases.`, ); return; } @@ -109,28 +113,19 @@ export class DataExtensionsEditorModule { return; } - if (!isQueryLanguage(db.language)) { - void showAndLogExceptionWithTelemetry( - extLogger, - telemetryListener, - redactableError`Unsupported database language ${db.language}`, - ); - return; - } - - const query = fetchExternalApiQueries[db.language]; + const query = fetchExternalApiQueries[language]; if (!query) { void showAndLogExceptionWithTelemetry( extLogger, telemetryListener, - redactableError`No external API usage query found for language ${db.language}`, + redactableError`No external API usage query found for language ${language}`, ); return; } // Create new temporary directory for query files and pack dependencies const queryDir = (await dir({ unsafeCleanup: true })).path; - await setUpPack(queryDir, query, db.language); + await setUpPack(queryDir, query, language); await this.cliServer.packInstall(queryDir); const view = new DataExtensionsEditorView(