From a9c36ea6990d92b21da69f81798f265789eef260 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 8 Feb 2023 11:52:49 +0000 Subject: [PATCH 001/132] Create new activated-extension test suite --- extensions/ql-vscode/jest.config.js | 1 + extensions/ql-vscode/package.json | 1 + .../databases/db-panel.test.ts | 0 .../jest-runner-vscode.config.js | 29 +++++++ .../activated-extension/jest.config.ts | 11 +++ .../activated-extension/jest.setup.ts | 1 + .../cli-integration/databaseFetcher.test.ts | 2 +- .../cli-integration/jest.config.ts | 2 +- .../cli-integration/jest.setup.ts | 83 +------------------ .../cli-integration/new-query.test.ts | 2 +- .../cli-integration/queries.test.ts | 2 +- .../remote-queries-manager.test.ts | 2 +- .../variant-analysis-manager.test.ts | 2 +- .../variant-analysis-results-manager.test.ts | 2 +- .../{cli-integration => }/global.helper.ts | 8 +- ...ts => jest-runner-installed-extensions.ts} | 4 +- .../jest.activated-extension.setup.ts | 82 ++++++++++++++++++ 17 files changed, 141 insertions(+), 93 deletions(-) rename extensions/ql-vscode/test/vscode-tests/{cli-integration => activated-extension}/databases/db-panel.test.ts (100%) create mode 100644 extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js create mode 100644 extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts create mode 100644 extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts rename extensions/ql-vscode/test/vscode-tests/{cli-integration => }/global.helper.ts (90%) rename extensions/ql-vscode/test/vscode-tests/{cli-integration/jest-runner-cli-integration.ts => jest-runner-installed-extensions.ts} (94%) create mode 100644 extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts diff --git a/extensions/ql-vscode/jest.config.js b/extensions/ql-vscode/jest.config.js index 68933e683..f06906060 100644 --- a/extensions/ql-vscode/jest.config.js +++ b/extensions/ql-vscode/jest.config.js @@ -8,6 +8,7 @@ module.exports = { projects: [ "/src/view", "/test/unit-tests", + "/test/vscode-tests/activated-extension", "/test/vscode-tests/cli-integration", "/test/vscode-tests/no-workspace", "/test/vscode-tests/minimal-workspace", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 2ac1f5d0e..35e43ae36 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1330,6 +1330,7 @@ "test:unit": "cross-env TZ=UTC LANG=en-US jest --projects test/unit-tests", "test:view": "jest --projects src/view", "integration": "npm-run-all integration:*", + "integration:activated-extension": "jest --projects test/vscode-tests/activated-extension", "integration:no-workspace": "jest --projects test/vscode-tests/no-workspace", "integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace", "cli-integration": "jest --projects test/vscode-tests/cli-integration", diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/databases/db-panel.test.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/databases/db-panel.test.ts similarity index 100% rename from extensions/ql-vscode/test/vscode-tests/cli-integration/databases/db-panel.test.ts rename to extensions/ql-vscode/test/vscode-tests/activated-extension/databases/db-panel.test.ts diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js new file mode 100644 index 000000000..9f95a10f0 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest-runner-vscode.config.js @@ -0,0 +1,29 @@ +const path = require("path"); + +const { + config: baseConfig, + rootDir, +} = require("../jest-runner-vscode.config.base"); + +/** @type import("jest-runner-vscode").RunnerOptions */ +const config = { + ...baseConfig, + launchArgs: [ + ...(baseConfig.launchArgs ?? []), + // explicitly disable extensions that are known to interfere with the CLI integration tests + "--disable-extension", + "eamodio.gitlens", + "--disable-extension", + "github.codespaces", + "--disable-extension", + "github.copilot", + path.resolve(rootDir, "test/data"), + ], + extensionTestsEnv: { + ...baseConfig.extensionTestsEnv, + INTEGRATION_TEST_MODE: "true", + }, + retries: 3, +}; + +module.exports = config; diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts new file mode 100644 index 000000000..3bd6d399c --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts @@ -0,0 +1,11 @@ +import type { Config } from "jest"; + +import baseConfig from "../jest.config.base"; + +const config: Config = { + ...baseConfig, + runner: "/../jest-runner-installed-extensions.ts", + setupFilesAfterEnv: ["/jest.setup.ts"], +}; + +export default config; diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts new file mode 100644 index 000000000..4dadcdf52 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts @@ -0,0 +1 @@ +import "../jest.activated-extension.setup"; diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/databaseFetcher.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/databaseFetcher.test.ts index c5f0ad8cb..6296fced5 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/databaseFetcher.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/databaseFetcher.test.ts @@ -8,7 +8,7 @@ import { importArchiveDatabase, promptImportInternetDatabase, } from "../../../src/databaseFetcher"; -import { cleanDatabases, dbLoc, DB_URL, storagePath } from "./global.helper"; +import { cleanDatabases, dbLoc, DB_URL, storagePath } from "../global.helper"; jest.setTimeout(60_000); diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts index 3a419bc65..3bd6d399c 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts @@ -4,7 +4,7 @@ import baseConfig from "../jest.config.base"; const config: Config = { ...baseConfig, - runner: "/jest-runner-cli-integration.ts", + runner: "/../jest-runner-installed-extensions.ts", setupFilesAfterEnv: ["/jest.setup.ts"], }; diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts index ed9ce1e25..e4a76d03f 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts @@ -1,59 +1,8 @@ -import { - mkdirpSync, - existsSync, - createWriteStream, - realpathSync, -} from "fs-extra"; -import { dirname } from "path"; -import fetch from "node-fetch"; -import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper"; -import * as tmp from "tmp"; -import { CUSTOM_CODEQL_PATH_SETTING } from "../../../src/config"; -import { ConfigurationTarget, env, extensions, workspace } from "vscode"; -import { beforeEachAction } from "../test-config"; +import { workspace } from "vscode"; -// create an extension storage location -let removeStorage: tmp.DirResult["removeCallback"] | undefined; - -beforeAll(async () => { - // Set the CLI version here before activation to ensure we don't accidentally try to download a cli - await beforeEachAction(); - await CUSTOM_CODEQL_PATH_SETTING.updateValue( - process.env.CLI_PATH, - ConfigurationTarget.Workspace, - ); - - // ensure the test database is downloaded - mkdirpSync(dirname(dbLoc)); - if (!existsSync(dbLoc)) { - console.log(`Downloading test database to ${dbLoc}`); - - await new Promise((resolve, reject) => { - return fetch(DB_URL).then((response) => { - const dest = createWriteStream(dbLoc); - response.body.pipe(dest); - - response.body.on("error", reject); - dest.on("error", reject); - dest.on("close", () => { - resolve(dbLoc); - }); - }); - }); - } - - // Create the temp directory to be used as extension local storage. - const dir = tmp.dirSync(); - let storagePath = realpathSync(dir.name); - if (storagePath.substring(0, 2).match(/[A-Z]:/)) { - storagePath = - storagePath.substring(0, 1).toLocaleLowerCase() + - storagePath.substring(1); - } - setStoragePath(storagePath); - - removeStorage = dir.removeCallback; +import "../jest.activated-extension.setup"; +beforeAll(() => { // check that the codeql folder is found in the workspace const folders = workspace.workspaceFolders; if (!folders) { @@ -70,30 +19,4 @@ beforeAll(async () => { ); } } - - // Activate the extension - await extensions.getExtension("GitHub.vscode-codeql")?.activate(); -}); - -beforeEach(async () => { - jest.spyOn(env, "openExternal").mockResolvedValue(false); - - await beforeEachAction(); - - await CUSTOM_CODEQL_PATH_SETTING.updateValue( - process.env.CLI_PATH, - ConfigurationTarget.Workspace, - ); -}); - -// ensure extension is cleaned up. -afterAll(async () => { - // ensure temp directory is cleaned up. - try { - removeStorage?.(); - } catch (e) { - // we are exiting anyway so don't worry about it. - // most likely the directory this is a test on Windows and some files are locked. - console.warn(`Failed to remove storage directory '${storagePath}': ${e}`); - } }); diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts index ea448f931..e5cfd07c6 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts @@ -11,7 +11,7 @@ import { describeWithCodeQL } from "../cli"; import { QueryServerClient } from "../../../src/query-server/queryserver-client"; import { extLogger, ProgressReporter } from "../../../src/common"; import { QueryResultType } from "../../../src/pure/new-messages"; -import { cleanDatabases, dbLoc, storagePath } from "./global.helper"; +import { cleanDatabases, dbLoc, storagePath } from "../global.helper"; import { importArchiveDatabase } from "../../../src/databaseFetcher"; const baseDir = join(__dirname, "../../../test/data"); diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts index 889f09b02..d4f328c75 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts @@ -17,7 +17,7 @@ import { load, dump } from "js-yaml"; import { DatabaseItem, DatabaseManager } from "../../../src/databases"; import { CodeQLExtensionInterface } from "../../../src/extension"; -import { cleanDatabases, dbLoc, storagePath } from "./global.helper"; +import { cleanDatabases, dbLoc, storagePath } from "../global.helper"; import { importArchiveDatabase } from "../../../src/databaseFetcher"; import { CodeQLCliServer } from "../../../src/cli"; import { describeWithCodeQL } from "../cli"; diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/remote-queries-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/remote-queries-manager.test.ts index f298b266a..ecbb785d8 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/remote-queries-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/remote-queries-manager.test.ts @@ -28,7 +28,7 @@ import { RemoteQueriesManager } from "../../../../src/remote-queries/remote-quer import { fixWorkspaceReferences, restoreWorkspaceReferences, -} from "../global.helper"; +} from "../../global.helper"; import { createMockApp } from "../../../__mocks__/appMock"; import { App } from "../../../../src/common/app"; diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts index 55aaf00b2..adf7c133e 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts @@ -31,7 +31,7 @@ import { fixWorkspaceReferences, restoreWorkspaceReferences, storagePath, -} from "../global.helper"; +} from "../../global.helper"; import { VariantAnalysisResultsManager } from "../../../../src/remote-queries/variant-analysis-results-manager"; import { createMockVariantAnalysis } from "../../../factories/remote-queries/shared/variant-analysis"; import * as VariantAnalysisModule from "../../../../src/remote-queries/shared/variant-analysis"; diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-results-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-results-manager.test.ts index ef52edb14..43deb4a40 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-results-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/remote-queries/variant-analysis-results-manager.test.ts @@ -9,7 +9,7 @@ import * as fetchModule from "node-fetch"; import { VariantAnalysisResultsManager } from "../../../../src/remote-queries/variant-analysis-results-manager"; import { CodeQLCliServer } from "../../../../src/cli"; -import { storagePath } from "../global.helper"; +import { storagePath } from "../../global.helper"; import { faker } from "@faker-js/faker"; import { createMockVariantAnalysisRepositoryTask } from "../../../factories/remote-queries/shared/variant-analysis-repo-tasks"; import { diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/global.helper.ts b/extensions/ql-vscode/test/vscode-tests/global.helper.ts similarity index 90% rename from extensions/ql-vscode/test/vscode-tests/cli-integration/global.helper.ts rename to extensions/ql-vscode/test/vscode-tests/global.helper.ts index 73a7a69a8..93b47e66f 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/global.helper.ts +++ b/extensions/ql-vscode/test/vscode-tests/global.helper.ts @@ -2,11 +2,11 @@ import { join } from "path"; import { load, dump } from "js-yaml"; import { realpathSync, readFileSync, writeFileSync } from "fs-extra"; import { commands } from "vscode"; -import { DatabaseManager } from "../../../src/databases"; -import { CodeQLCliServer } from "../../../src/cli"; -import { removeWorkspaceRefs } from "../../../src/remote-queries/run-remote-query"; +import { DatabaseManager } from "../../src/databases"; +import { CodeQLCliServer } from "../../src/cli"; +import { removeWorkspaceRefs } from "../../src/remote-queries/run-remote-query"; -// This file contains helpers shared between actual tests. +// This file contains helpers shared between tests that work with an activated extension. export const DB_URL = "https://github.com/github/vscode-codeql/files/5586722/simple-db.zip"; diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-cli-integration.ts b/extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts similarity index 94% rename from extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-cli-integration.ts rename to extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts index 729624cb8..0c031b72d 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest-runner-cli-integration.ts +++ b/extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts @@ -8,9 +8,9 @@ import { downloadAndUnzipVSCode, resolveCliArgsFromVSCodeExecutablePath, } from "@vscode/test-electron"; -import { ensureCli } from "../ensureCli"; +import { ensureCli } from "./ensureCli"; -export default class JestRunnerCliIntegration extends VSCodeTestRunner { +export default class JestRunnerInstalledExtensions extends VSCodeTestRunner { async runTests( tests: JestRunner.Test[], watcher: JestRunner.TestWatcher, diff --git a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts new file mode 100644 index 000000000..8db9b5d38 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts @@ -0,0 +1,82 @@ +import { + mkdirpSync, + existsSync, + createWriteStream, + realpathSync, +} from "fs-extra"; +import { dirname } from "path"; +import fetch from "node-fetch"; +import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper"; +import * as tmp from "tmp"; +import { CUSTOM_CODEQL_PATH_SETTING } from "../../src/config"; +import { ConfigurationTarget, env, extensions } from "vscode"; +import { beforeEachAction } from "./test-config"; + +// create an extension storage location +let removeStorage: tmp.DirResult["removeCallback"] | undefined; + +beforeAll(async () => { + // Set the CLI version here before activation to ensure we don't accidentally try to download a cli + await beforeEachAction(); + await CUSTOM_CODEQL_PATH_SETTING.updateValue( + process.env.CLI_PATH, + ConfigurationTarget.Workspace, + ); + + // ensure the test database is downloaded + mkdirpSync(dirname(dbLoc)); + if (!existsSync(dbLoc)) { + console.log(`Downloading test database to ${dbLoc}`); + + await new Promise((resolve, reject) => { + return fetch(DB_URL).then((response) => { + const dest = createWriteStream(dbLoc); + response.body.pipe(dest); + + response.body.on("error", reject); + dest.on("error", reject); + dest.on("close", () => { + resolve(dbLoc); + }); + }); + }); + } + + // Create the temp directory to be used as extension local storage. + const dir = tmp.dirSync(); + let storagePath = realpathSync(dir.name); + if (storagePath.substring(0, 2).match(/[A-Z]:/)) { + storagePath = + storagePath.substring(0, 1).toLocaleLowerCase() + + storagePath.substring(1); + } + setStoragePath(storagePath); + + removeStorage = dir.removeCallback; + + // Activate the extension + await extensions.getExtension("GitHub.vscode-codeql")?.activate(); +}); + +beforeEach(async () => { + jest.spyOn(env, "openExternal").mockResolvedValue(false); + + await beforeEachAction(); + + await CUSTOM_CODEQL_PATH_SETTING.updateValue( + process.env.CLI_PATH, + ConfigurationTarget.Workspace, + ); +}); + +// ensure extension is cleaned up. +afterAll(async () => { + // ensure temp directory is cleaned up. + try { + removeStorage?.(); + } catch (e) { + // we are exiting anyway so don't worry about it. + // most likely the directory this is a test on Windows and some files are locked. + console.warn(`Failed to remove storage directory '${storagePath}': ${e}`); + } +}); From 6b8467fc4e7f8e7525405e1aea7bc641538f907f Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 8 Feb 2023 13:14:00 +0000 Subject: [PATCH 002/132] Update test script names --- .github/workflows/main.yml | 15 ++++++++++----- extensions/ql-vscode/package.json | 10 +++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75bc523b3..15d1499c6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,7 +132,12 @@ jobs: - name: Run unit tests working-directory: extensions/ql-vscode run: | - npm run test + npm run test:unit + + - name: Run view tests + working-directory: extensions/ql-vscode + run: | + npm run test:view test: name: Test @@ -173,7 +178,7 @@ jobs: VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' run: | unset DBUS_SESSION_BUS_ADDRESS - /usr/bin/xvfb-run npm run integration + /usr/bin/xvfb-run npm run test:vscode-integration - name: Run integration tests (Windows) if: matrix.os == 'windows-latest' @@ -181,7 +186,7 @@ jobs: env: VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' run: | - npm run integration + npm run test:vscode-integration set-matrix: name: Set Matrix for cli-test @@ -254,10 +259,10 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | unset DBUS_SESSION_BUS_ADDRESS - /usr/bin/xvfb-run npm run cli-integration + /usr/bin/xvfb-run npm run test:cli-integration - name: Run CLI tests (Windows) working-directory: extensions/ql-vscode if: matrix.os == 'windows-latest' run: | - npm run cli-integration + npm run test:cli-integration diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 35e43ae36..da307f1c9 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1329,11 +1329,11 @@ "test": "npm-run-all -p test:*", "test:unit": "cross-env TZ=UTC LANG=en-US jest --projects test/unit-tests", "test:view": "jest --projects src/view", - "integration": "npm-run-all integration:*", - "integration:activated-extension": "jest --projects test/vscode-tests/activated-extension", - "integration:no-workspace": "jest --projects test/vscode-tests/no-workspace", - "integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace", - "cli-integration": "jest --projects test/vscode-tests/cli-integration", + "test:vscode-integration": "npm-run-all test:vscode-integration:*", + "test:vscode-integration:activated-extension": "jest --projects test/vscode-tests/activated-extension", + "test:vscode-integration:no-workspace": "jest --projects test/vscode-tests/no-workspace", + "test:vscode-integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace", + "test:cli-integration": "jest --projects test/vscode-tests/cli-integration", "update-vscode": "node ./node_modules/vscode/bin/install", "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix", "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0", From 642b8e59603b360d4d9df43cf36b723153153c2d Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 8 Feb 2023 13:28:17 +0000 Subject: [PATCH 003/132] Add documentation for new test suite and scripts --- CONTRIBUTING.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8854e269..ba706c2e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,10 +98,11 @@ We have several types of tests: * Unit tests: these live in the `tests/unit-tests/` directory * View tests: these live in `src/view/variant-analysis/__tests__/` * VSCode integration tests: - * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality that is meant to work before you even have a workspace open. + * `test/vscode-tests/activated-extension` tests: These are intended to cover functionality that require the full extension to be activated but don't require the CLI. This suite is not run against multiple versions of the CLI in CI. + * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality that is meant to work before you even have a workspace open but don't require the extension to be activated. * `test/vscode-tests/minimal-workspace` tests: These are intended to cover functionality that need a workspace but don't require the full extension to be activated. * CLI integration tests: these live in `test/vscode-tests/cli-integration` - * These tests are intendended to be cover functionality that is related to the integration between the CodeQL CLI and the extension. + * These tests are intended to be cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against all supported versions of the CLI in CI. The CLI integration tests require an instance of the CodeQL CLI to run so they will require some extra setup steps. When adding new tests to our test suite, please be mindful of whether they need to be in the cli-integration folder. If the tests don't depend on the CLI, they are better suited to being a VSCode integration test. @@ -119,7 +120,7 @@ Then, from the `extensions/ql-vscode` directory, use the appropriate command to * Unit tests: `npm run test:unit` * View Tests: `npm test:view` -* VSCode integration tests: `npm run integration` +* VSCode integration tests: `npm run test:vscode-integration` ###### CLI integration tests @@ -130,7 +131,7 @@ The CLI integration tests require the CodeQL standard libraries in order to run 2. Run your test command: ```shell -cd extensions/ql-vscode && npm run cli-integration +cd extensions/ql-vscode && npm run test:cli-integration ``` ##### 2. From VSCode @@ -161,13 +162,13 @@ The easiest way to run a single test is to change the `it` of the test to `it.on to only run tests for this specific file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`: ```shell -npm run cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts +npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts ``` You can also use the `--testNamePattern` option to run a specific test within a file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`: ```shell -npm run cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo" +npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo" ``` ##### 2. From VSCode From ce56e6f038e802a9eeb411034237cb4ce5b8cdaa Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Fri, 10 Feb 2023 10:43:48 +0000 Subject: [PATCH 004/132] Retry VSCode integration tests This will call `jest.retryTimes` to retry the VSCode integration tests up to 3 times. This should help with the flaky tests. --- .../test/vscode-tests/jest.activated-extension.setup.ts | 4 ++++ extensions/ql-vscode/test/vscode-tests/jest.setup.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts index 8db9b5d38..02e49ecca 100644 --- a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts @@ -15,6 +15,10 @@ import { beforeEachAction } from "./test-config"; // create an extension storage location let removeStorage: tmp.DirResult["removeCallback"] | undefined; +jest.retryTimes(3, { + logErrorsBeforeRetry: true, +}); + beforeAll(async () => { // Set the CLI version here before activation to ensure we don't accidentally try to download a cli await beforeEachAction(); diff --git a/extensions/ql-vscode/test/vscode-tests/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/jest.setup.ts index 55a546471..fe58b7f48 100644 --- a/extensions/ql-vscode/test/vscode-tests/jest.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/jest.setup.ts @@ -1,6 +1,10 @@ import { env } from "vscode"; import { beforeEachAction } from "./test-config"; +jest.retryTimes(3, { + logErrorsBeforeRetry: true, +}); + beforeEach(async () => { jest.spyOn(env, "openExternal").mockResolvedValue(false); From 14eb6b4f89dd09e988ba30db886baa4e8649f365 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 13 Feb 2023 11:37:11 +0100 Subject: [PATCH 005/132] Only download test DB for CLI integration tests The test database is not required for activated extension tests, so we only need to download the test database for CLI integration tests. --- .../activated-extension/jest.setup.ts | 13 +++- .../cli-integration/jest.setup.ts | 69 +++++++++++++++++- .../jest.activated-extension.setup.ts | 70 ++----------------- 3 files changed, 87 insertions(+), 65 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts index 4dadcdf52..69c4f39f1 100644 --- a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.setup.ts @@ -1 +1,12 @@ -import "../jest.activated-extension.setup"; +import { + beforeAllAction, + beforeEachAction, +} from "../jest.activated-extension.setup"; + +beforeAll(async () => { + await beforeAllAction(); +}); + +beforeEach(async () => { + await beforeEachAction(); +}); diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts index e4a76d03f..416ca1ea9 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts @@ -1,6 +1,61 @@ import { workspace } from "vscode"; -import "../jest.activated-extension.setup"; +import { + beforeAllAction, + beforeEachAction, +} from "../jest.activated-extension.setup"; +import * as tmp from "tmp"; +import { + createWriteStream, + existsSync, + mkdirpSync, + realpathSync, +} from "fs-extra"; +import { dirname } from "path"; +import { DB_URL, dbLoc, setStoragePath, storagePath } from "../global.helper"; +import fetch from "node-fetch"; + +// create an extension storage location +let removeStorage: tmp.DirResult["removeCallback"] | undefined; + +beforeAll(async () => { + // ensure the test database is downloaded + mkdirpSync(dirname(dbLoc)); + if (!existsSync(dbLoc)) { + console.log(`Downloading test database to ${dbLoc}`); + + await new Promise((resolve, reject) => { + return fetch(DB_URL).then((response) => { + const dest = createWriteStream(dbLoc); + response.body.pipe(dest); + + response.body.on("error", reject); + dest.on("error", reject); + dest.on("close", () => { + resolve(dbLoc); + }); + }); + }); + } + + // Create the temp directory to be used as extension local storage. + const dir = tmp.dirSync(); + let storagePath = realpathSync(dir.name); + if (storagePath.substring(0, 2).match(/[A-Z]:/)) { + storagePath = + storagePath.substring(0, 1).toLocaleLowerCase() + + storagePath.substring(1); + } + setStoragePath(storagePath); + + removeStorage = dir.removeCallback; + + await beforeAllAction(); +}); + +beforeEach(async () => { + await beforeEachAction(); +}); beforeAll(() => { // check that the codeql folder is found in the workspace @@ -20,3 +75,15 @@ beforeAll(() => { } } }); + +// ensure extension is cleaned up. +afterAll(async () => { + // ensure temp directory is cleaned up. + try { + removeStorage?.(); + } catch (e) { + // we are exiting anyway so don't worry about it. + // most likely the directory this is a test on Windows and some files are locked. + console.warn(`Failed to remove storage directory '${storagePath}': ${e}`); + } +}); diff --git a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts index 8db9b5d38..ec7d1e8f9 100644 --- a/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/jest.activated-extension.setup.ts @@ -1,82 +1,26 @@ -import { - mkdirpSync, - existsSync, - createWriteStream, - realpathSync, -} from "fs-extra"; -import { dirname } from "path"; -import fetch from "node-fetch"; -import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper"; -import * as tmp from "tmp"; import { CUSTOM_CODEQL_PATH_SETTING } from "../../src/config"; import { ConfigurationTarget, env, extensions } from "vscode"; -import { beforeEachAction } from "./test-config"; +import { beforeEachAction as testConfigBeforeEachAction } from "./test-config"; -// create an extension storage location -let removeStorage: tmp.DirResult["removeCallback"] | undefined; - -beforeAll(async () => { +export async function beforeAllAction() { // Set the CLI version here before activation to ensure we don't accidentally try to download a cli - await beforeEachAction(); + await testConfigBeforeEachAction(); await CUSTOM_CODEQL_PATH_SETTING.updateValue( process.env.CLI_PATH, ConfigurationTarget.Workspace, ); - // ensure the test database is downloaded - mkdirpSync(dirname(dbLoc)); - if (!existsSync(dbLoc)) { - console.log(`Downloading test database to ${dbLoc}`); - - await new Promise((resolve, reject) => { - return fetch(DB_URL).then((response) => { - const dest = createWriteStream(dbLoc); - response.body.pipe(dest); - - response.body.on("error", reject); - dest.on("error", reject); - dest.on("close", () => { - resolve(dbLoc); - }); - }); - }); - } - - // Create the temp directory to be used as extension local storage. - const dir = tmp.dirSync(); - let storagePath = realpathSync(dir.name); - if (storagePath.substring(0, 2).match(/[A-Z]:/)) { - storagePath = - storagePath.substring(0, 1).toLocaleLowerCase() + - storagePath.substring(1); - } - setStoragePath(storagePath); - - removeStorage = dir.removeCallback; - // Activate the extension await extensions.getExtension("GitHub.vscode-codeql")?.activate(); -}); +} -beforeEach(async () => { +export async function beforeEachAction() { jest.spyOn(env, "openExternal").mockResolvedValue(false); - await beforeEachAction(); + await testConfigBeforeEachAction(); await CUSTOM_CODEQL_PATH_SETTING.updateValue( process.env.CLI_PATH, ConfigurationTarget.Workspace, ); -}); - -// ensure extension is cleaned up. -afterAll(async () => { - // ensure temp directory is cleaned up. - try { - removeStorage?.(); - } catch (e) { - // we are exiting anyway so don't worry about it. - // most likely the directory this is a test on Windows and some files are locked. - console.warn(`Failed to remove storage directory '${storagePath}': ${e}`); - } -}); +} From b412896d544cd135176a25f56c59ad928af22c9e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 13 Feb 2023 11:39:07 +0100 Subject: [PATCH 006/132] Make purpose of `no-workspace` tests clearer --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90785bb64..fe5a29214 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,7 @@ We have several types of tests: * View tests: these live in `src/view/variant-analysis/__tests__/` * VSCode integration tests: * `test/vscode-tests/activated-extension` tests: These are intended to cover functionality that require the full extension to be activated but don't require the CLI. This suite is not run against multiple versions of the CLI in CI. - * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality that is meant to work before you even have a workspace open but don't require the extension to be activated. + * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality around not having a workspace. The extension is not activated in these tests. * `test/vscode-tests/minimal-workspace` tests: These are intended to cover functionality that need a workspace but don't require the full extension to be activated. * CLI integration tests: these live in `test/vscode-tests/cli-integration` * These tests are intended to be cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against all supported versions of the CLI in CI. From 46670cc7d610a8a2df85f406ec4906aafd568c41 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 13 Feb 2023 11:39:38 +0100 Subject: [PATCH 007/132] Clarify CLI integration test CI runs --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe5a29214..3115d05e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ We have several types of tests: * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality around not having a workspace. The extension is not activated in these tests. * `test/vscode-tests/minimal-workspace` tests: These are intended to cover functionality that need a workspace but don't require the full extension to be activated. * CLI integration tests: these live in `test/vscode-tests/cli-integration` - * These tests are intended to be cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against all supported versions of the CLI in CI. + * These tests are intended to cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against each supported versions of the CLI in CI. The CLI integration tests require an instance of the CodeQL CLI to run so they will require some extra setup steps. When adding new tests to our test suite, please be mindful of whether they need to be in the cli-integration folder. If the tests don't depend on the CLI, they are better suited to being a VSCode integration test. From 930def851d4853768c1a9112f846db3e525358bb Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Mon, 13 Feb 2023 12:02:21 +0100 Subject: [PATCH 008/132] Disable parallelization of tests This would run the unit, view, integration and CLI integration tests in parallel, which would cause problems with multiple VSCode instances and use a lot of memory. --- extensions/ql-vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index b86ac9b1e..1e2f5511c 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1326,7 +1326,7 @@ "scripts": { "build": "gulp", "watch": "gulp watch", - "test": "npm-run-all -p test:*", + "test": "npm-run-all test:*", "test:unit": "cross-env TZ=UTC LANG=en-US jest --projects test/unit-tests", "test:view": "jest --projects src/view", "test:vscode-integration": "npm-run-all test:vscode-integration:*", From 7508ef2e260c4d7c29c4564e1880990164fd6f18 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 14 Feb 2023 10:27:47 +0100 Subject: [PATCH 009/132] Move components used by MRVA out of remote-queries --- .../stories/{remote-queries => common}/TextButton.stories.tsx | 2 +- .../{remote-queries => }/data/analysesResultsMessage.json | 0 .../src/stories/{remote-queries => }/data/rawResults.json | 0 .../{remote-queries => }/data/remoteQueryResultMessage.json | 0 .../src/stories/remote-queries/RemoteQueries.stories.tsx | 4 ++-- .../AnalysisAlertResult.stories.tsx | 4 ++-- .../src/stories/variant-analysis/RepoRow.stories.tsx | 4 ++-- .../variant-analysis/VariantAnalysisAnalyzedRepos.stories.tsx | 2 +- .../src/view/{remote-queries => common}/TextButton.tsx | 0 .../ql-vscode/src/view/remote-queries/RemoteQueries.tsx | 4 ++-- .../AnalysisAlertResult.tsx | 0 .../src/view/variant-analysis/AnalyzedRepoItemContent.tsx | 4 ++-- .../{remote-queries => variant-analysis}/RawResultsTable.tsx | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) rename extensions/ql-vscode/src/stories/{remote-queries => common}/TextButton.stories.tsx (88%) rename extensions/ql-vscode/src/stories/{remote-queries => }/data/analysesResultsMessage.json (100%) rename extensions/ql-vscode/src/stories/{remote-queries => }/data/rawResults.json (100%) rename extensions/ql-vscode/src/stories/{remote-queries => }/data/remoteQueryResultMessage.json (100%) rename extensions/ql-vscode/src/stories/{remote-queries => variant-analysis}/AnalysisAlertResult.stories.tsx (90%) rename extensions/ql-vscode/src/view/{remote-queries => common}/TextButton.tsx (100%) rename extensions/ql-vscode/src/view/{remote-queries => variant-analysis}/AnalysisAlertResult.tsx (100%) rename extensions/ql-vscode/src/view/{remote-queries => variant-analysis}/RawResultsTable.tsx (98%) diff --git a/extensions/ql-vscode/src/stories/remote-queries/TextButton.stories.tsx b/extensions/ql-vscode/src/stories/common/TextButton.stories.tsx similarity index 88% rename from extensions/ql-vscode/src/stories/remote-queries/TextButton.stories.tsx rename to extensions/ql-vscode/src/stories/common/TextButton.stories.tsx index fd3a6e376..6a4332e69 100644 --- a/extensions/ql-vscode/src/stories/remote-queries/TextButton.stories.tsx +++ b/extensions/ql-vscode/src/stories/common/TextButton.stories.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import { ComponentStory, ComponentMeta } from "@storybook/react"; -import TextButtonComponent from "../../view/remote-queries/TextButton"; +import TextButtonComponent from "../../view/common/TextButton"; export default { title: "Text Button", diff --git a/extensions/ql-vscode/src/stories/remote-queries/data/analysesResultsMessage.json b/extensions/ql-vscode/src/stories/data/analysesResultsMessage.json similarity index 100% rename from extensions/ql-vscode/src/stories/remote-queries/data/analysesResultsMessage.json rename to extensions/ql-vscode/src/stories/data/analysesResultsMessage.json diff --git a/extensions/ql-vscode/src/stories/remote-queries/data/rawResults.json b/extensions/ql-vscode/src/stories/data/rawResults.json similarity index 100% rename from extensions/ql-vscode/src/stories/remote-queries/data/rawResults.json rename to extensions/ql-vscode/src/stories/data/rawResults.json diff --git a/extensions/ql-vscode/src/stories/remote-queries/data/remoteQueryResultMessage.json b/extensions/ql-vscode/src/stories/data/remoteQueryResultMessage.json similarity index 100% rename from extensions/ql-vscode/src/stories/remote-queries/data/remoteQueryResultMessage.json rename to extensions/ql-vscode/src/stories/data/remoteQueryResultMessage.json diff --git a/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx b/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx index b3ecd2667..caafcacc9 100644 --- a/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx +++ b/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx @@ -5,8 +5,8 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; import { RemoteQueries } from "../../view/remote-queries/RemoteQueries"; -import * as remoteQueryResult from "./data/remoteQueryResultMessage.json"; -import * as analysesResults from "./data/analysesResultsMessage.json"; +import * as remoteQueryResult from "../data/remoteQueryResultMessage.json"; +import * as analysesResults from "../data/analysesResultsMessage.json"; export default { title: "MRVA/Remote Queries", diff --git a/extensions/ql-vscode/src/stories/remote-queries/AnalysisAlertResult.stories.tsx b/extensions/ql-vscode/src/stories/variant-analysis/AnalysisAlertResult.stories.tsx similarity index 90% rename from extensions/ql-vscode/src/stories/remote-queries/AnalysisAlertResult.stories.tsx rename to extensions/ql-vscode/src/stories/variant-analysis/AnalysisAlertResult.stories.tsx index 219d797a3..8a8e9a683 100644 --- a/extensions/ql-vscode/src/stories/remote-queries/AnalysisAlertResult.stories.tsx +++ b/extensions/ql-vscode/src/stories/variant-analysis/AnalysisAlertResult.stories.tsx @@ -2,11 +2,11 @@ import * as React from "react"; import { ComponentStory, ComponentMeta } from "@storybook/react"; -import AnalysisAlertResult from "../../view/remote-queries/AnalysisAlertResult"; +import AnalysisAlertResult from "../../view/variant-analysis/AnalysisAlertResult"; import type { AnalysisAlert } from "../../remote-queries/shared/analysis-result"; export default { - title: "Analysis Alert Result", + title: "Variant Analysis/Analysis Alert Result", component: AnalysisAlertResult, } as ComponentMeta; diff --git a/extensions/ql-vscode/src/stories/variant-analysis/RepoRow.stories.tsx b/extensions/ql-vscode/src/stories/variant-analysis/RepoRow.stories.tsx index 95bce78bd..a57cbcc24 100644 --- a/extensions/ql-vscode/src/stories/variant-analysis/RepoRow.stories.tsx +++ b/extensions/ql-vscode/src/stories/variant-analysis/RepoRow.stories.tsx @@ -13,8 +13,8 @@ import { } from "../../remote-queries/shared/analysis-result"; import { createMockRepositoryWithMetadata } from "../../../test/factories/remote-queries/shared/repository"; -import * as analysesResults from "../remote-queries/data/analysesResultsMessage.json"; -import * as rawResults from "../remote-queries/data/rawResults.json"; +import * as analysesResults from "../data/analysesResultsMessage.json"; +import * as rawResults from "../data/rawResults.json"; import { RepoRow, RepoRowProps } from "../../view/variant-analysis/RepoRow"; export default { diff --git a/extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisAnalyzedRepos.stories.tsx b/extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisAnalyzedRepos.stories.tsx index 1b18b9942..3405e2f58 100644 --- a/extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisAnalyzedRepos.stories.tsx +++ b/extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisAnalyzedRepos.stories.tsx @@ -16,7 +16,7 @@ import { createMockVariantAnalysis } from "../../../test/factories/remote-querie import { createMockRepositoryWithMetadata } from "../../../test/factories/remote-queries/shared/repository"; import { createMockScannedRepo } from "../../../test/factories/remote-queries/shared/scanned-repositories"; -import * as analysesResults from "../remote-queries/data/analysesResultsMessage.json"; +import * as analysesResults from "../data/analysesResultsMessage.json"; export default { title: "Variant Analysis/Analyzed Repos", diff --git a/extensions/ql-vscode/src/view/remote-queries/TextButton.tsx b/extensions/ql-vscode/src/view/common/TextButton.tsx similarity index 100% rename from extensions/ql-vscode/src/view/remote-queries/TextButton.tsx rename to extensions/ql-vscode/src/view/common/TextButton.tsx diff --git a/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx b/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx index 69c8292a6..08a9dbab6 100644 --- a/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx +++ b/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx @@ -29,8 +29,8 @@ import { RepoIcon, TerminalIcon, } from "@primer/octicons-react"; -import AnalysisAlertResult from "./AnalysisAlertResult"; -import RawResultsTable from "./RawResultsTable"; +import AnalysisAlertResult from "../variant-analysis/AnalysisAlertResult"; +import RawResultsTable from "../variant-analysis/RawResultsTable"; import RepositoriesSearch from "./RepositoriesSearch"; import StarCount from "../common/StarCount"; import SortRepoFilter, { Sort, sorter } from "./SortRepoFilter"; diff --git a/extensions/ql-vscode/src/view/remote-queries/AnalysisAlertResult.tsx b/extensions/ql-vscode/src/view/variant-analysis/AnalysisAlertResult.tsx similarity index 100% rename from extensions/ql-vscode/src/view/remote-queries/AnalysisAlertResult.tsx rename to extensions/ql-vscode/src/view/variant-analysis/AnalysisAlertResult.tsx diff --git a/extensions/ql-vscode/src/view/variant-analysis/AnalyzedRepoItemContent.tsx b/extensions/ql-vscode/src/view/variant-analysis/AnalyzedRepoItemContent.tsx index 085cbd84d..264c96c69 100644 --- a/extensions/ql-vscode/src/view/variant-analysis/AnalyzedRepoItemContent.tsx +++ b/extensions/ql-vscode/src/view/variant-analysis/AnalyzedRepoItemContent.tsx @@ -4,8 +4,8 @@ import { AnalysisAlert, AnalysisRawResults, } from "../../remote-queries/shared/analysis-result"; -import AnalysisAlertResult from "../remote-queries/AnalysisAlertResult"; -import RawResultsTable from "../remote-queries/RawResultsTable"; +import AnalysisAlertResult from "./AnalysisAlertResult"; +import RawResultsTable from "./RawResultsTable"; import { VariantAnalysisRepoStatus, VariantAnalysisScannedRepositoryDownloadStatus, diff --git a/extensions/ql-vscode/src/view/remote-queries/RawResultsTable.tsx b/extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx similarity index 98% rename from extensions/ql-vscode/src/view/remote-queries/RawResultsTable.tsx rename to extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx index 6ad78c299..9078651fc 100644 --- a/extensions/ql-vscode/src/view/remote-queries/RawResultsTable.tsx +++ b/extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx @@ -8,7 +8,7 @@ import { ResultSetSchema, } from "../../pure/bqrs-cli-types"; import { tryGetRemoteLocation } from "../../pure/bqrs-utils"; -import TextButton from "./TextButton"; +import TextButton from "../common/TextButton"; import { convertNonPrintableChars } from "../../text-utils"; import { sendTelemetry, useTelemetryOnChange } from "../common/telemetry"; From c0ffd795e045318f4d1f163dfcbfad32cd20080e Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 14 Feb 2023 10:28:51 +0100 Subject: [PATCH 010/132] Remove remote queries React components --- .../remote-queries/DownloadButton.stories.tsx | 27 - .../DownloadSpinner.stories.tsx | 12 - .../remote-queries/LastUpdated.stories.tsx | 20 - .../remote-queries/RemoteQueries.stories.tsx | 25 - .../RepositoriesSearch.stories.tsx | 29 - .../view/remote-queries/CollapsibleItem.tsx | 50 -- .../view/remote-queries/DownloadButton.tsx | 30 - .../view/remote-queries/DownloadSpinner.tsx | 16 - .../view/remote-queries/FullScreenModal.tsx | 53 -- .../src/view/remote-queries/LastUpdated.tsx | 35 -- .../src/view/remote-queries/RemoteQueries.tsx | 551 ------------------ .../remote-queries/RepoListCopyButton.tsx | 29 - .../remote-queries/RepositoriesSearch.tsx | 31 - .../view/remote-queries/SortRepoFilter.tsx | 94 --- .../src/view/remote-queries/baseStyles.css | 4 - .../src/view/remote-queries/index.tsx | 9 - .../src/view/remote-queries/remoteQueries.css | 53 -- 17 files changed, 1068 deletions(-) delete mode 100644 extensions/ql-vscode/src/stories/remote-queries/DownloadButton.stories.tsx delete mode 100644 extensions/ql-vscode/src/stories/remote-queries/DownloadSpinner.stories.tsx delete mode 100644 extensions/ql-vscode/src/stories/remote-queries/LastUpdated.stories.tsx delete mode 100644 extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx delete mode 100644 extensions/ql-vscode/src/stories/remote-queries/RepositoriesSearch.stories.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/CollapsibleItem.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/DownloadButton.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/DownloadSpinner.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/FullScreenModal.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/LastUpdated.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/RepoListCopyButton.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/RepositoriesSearch.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/SortRepoFilter.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/baseStyles.css delete mode 100644 extensions/ql-vscode/src/view/remote-queries/index.tsx delete mode 100644 extensions/ql-vscode/src/view/remote-queries/remoteQueries.css diff --git a/extensions/ql-vscode/src/stories/remote-queries/DownloadButton.stories.tsx b/extensions/ql-vscode/src/stories/remote-queries/DownloadButton.stories.tsx deleted file mode 100644 index 1ea54caa0..000000000 --- a/extensions/ql-vscode/src/stories/remote-queries/DownloadButton.stories.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import * as React from "react"; - -import { ComponentStory, ComponentMeta } from "@storybook/react"; - -import DownloadButtonComponent from "../../view/remote-queries/DownloadButton"; - -export default { - title: "Download Button", - component: DownloadButtonComponent, - argTypes: { - onClick: { - action: "clicked", - table: { - disable: true, - }, - }, - }, -} as ComponentMeta; - -const Template: ComponentStory = (args) => ( - -); - -export const DownloadButton = Template.bind({}); -DownloadButton.args = { - text: "Download", -}; diff --git a/extensions/ql-vscode/src/stories/remote-queries/DownloadSpinner.stories.tsx b/extensions/ql-vscode/src/stories/remote-queries/DownloadSpinner.stories.tsx deleted file mode 100644 index 358933b77..000000000 --- a/extensions/ql-vscode/src/stories/remote-queries/DownloadSpinner.stories.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import * as React from "react"; - -import { ComponentMeta } from "@storybook/react"; - -import DownloadSpinnerComponent from "../../view/remote-queries/DownloadSpinner"; - -export default { - title: "Download Spinner", - component: DownloadSpinnerComponent, -} as ComponentMeta; - -export const DownloadSpinner = ; diff --git a/extensions/ql-vscode/src/stories/remote-queries/LastUpdated.stories.tsx b/extensions/ql-vscode/src/stories/remote-queries/LastUpdated.stories.tsx deleted file mode 100644 index b60bd0928..000000000 --- a/extensions/ql-vscode/src/stories/remote-queries/LastUpdated.stories.tsx +++ /dev/null @@ -1,20 +0,0 @@ -import * as React from "react"; - -import { ComponentStory, ComponentMeta } from "@storybook/react"; - -import LastUpdatedComponent from "../../view/remote-queries/LastUpdated"; - -export default { - title: "MRVA/Last Updated", - component: LastUpdatedComponent, -} as ComponentMeta; - -const Template: ComponentStory = (args) => ( - -); - -export const LastUpdated = Template.bind({}); - -LastUpdated.args = { - lastUpdated: -3_600_000, // 1 hour ago -}; diff --git a/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx b/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx deleted file mode 100644 index caafcacc9..000000000 --- a/extensions/ql-vscode/src/stories/remote-queries/RemoteQueries.stories.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import * as React from "react"; -import { useEffect } from "react"; - -import { ComponentStory, ComponentMeta } from "@storybook/react"; - -import { RemoteQueries } from "../../view/remote-queries/RemoteQueries"; - -import * as remoteQueryResult from "../data/remoteQueryResultMessage.json"; -import * as analysesResults from "../data/analysesResultsMessage.json"; - -export default { - title: "MRVA/Remote Queries", - component: RemoteQueries, -} as ComponentMeta; - -const Template: ComponentStory = () => { - useEffect(() => { - window.postMessage(remoteQueryResult); - window.postMessage(analysesResults); - }); - - return ; -}; - -export const Top10JavaScript = Template.bind({}); diff --git a/extensions/ql-vscode/src/stories/remote-queries/RepositoriesSearch.stories.tsx b/extensions/ql-vscode/src/stories/remote-queries/RepositoriesSearch.stories.tsx deleted file mode 100644 index 5207d5806..000000000 --- a/extensions/ql-vscode/src/stories/remote-queries/RepositoriesSearch.stories.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import * as React from "react"; -import { useState } from "react"; - -import { ComponentMeta } from "@storybook/react"; - -import RepositoriesSearchComponent from "../../view/remote-queries/RepositoriesSearch"; - -export default { - title: "MRVA/Repositories Search", - component: RepositoriesSearchComponent, - argTypes: { - filterValue: { - control: { - disable: true, - }, - }, - }, -} as ComponentMeta; - -export const RepositoriesSearch = () => { - const [filterValue, setFilterValue] = useState(""); - - return ( - - ); -}; diff --git a/extensions/ql-vscode/src/view/remote-queries/CollapsibleItem.tsx b/extensions/ql-vscode/src/view/remote-queries/CollapsibleItem.tsx deleted file mode 100644 index ad8f55411..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/CollapsibleItem.tsx +++ /dev/null @@ -1,50 +0,0 @@ -import * as React from "react"; -import styled from "styled-components"; -import { ChevronDownIcon, ChevronRightIcon } from "@primer/octicons-react"; -import { useState } from "react"; - -const Container = styled.div` - display: block; - vertical-align: middle; - cursor: pointer; -`; - -const TitleContainer = styled.span` - display: inline-block; -`; - -const Button = styled.button` - display: inline-block; - background-color: transparent; - color: var(--vscode-editor-foreground); - border: none; - padding-left: 0; - padding-right: 0.1em; -`; - -const CollapsibleItem = ({ - title, - children, -}: { - title: React.ReactNode; - children: React.ReactNode; -}) => { - const [isExpanded, setExpanded] = useState(false); - return ( - <> - setExpanded(!isExpanded)}> - - {title} - - {isExpanded && children} - - ); -}; - -export default CollapsibleItem; diff --git a/extensions/ql-vscode/src/view/remote-queries/DownloadButton.tsx b/extensions/ql-vscode/src/view/remote-queries/DownloadButton.tsx deleted file mode 100644 index c7328ab02..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/DownloadButton.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import * as React from "react"; -import styled from "styled-components"; -import { DownloadIcon } from "@primer/octicons-react"; - -const ButtonLink = styled.a` - display: inline-block; - font-size: x-small; - text-decoration: none; - cursor: pointer; - vertical-align: middle; - - svg { - fill: var(--vscode-textLink-foreground); - } -`; - -const DownloadButton = ({ - text, - onClick, -}: { - text: string; - onClick: () => void; -}) => ( - - - {text} - -); - -export default DownloadButton; diff --git a/extensions/ql-vscode/src/view/remote-queries/DownloadSpinner.tsx b/extensions/ql-vscode/src/view/remote-queries/DownloadSpinner.tsx deleted file mode 100644 index 5778de717..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/DownloadSpinner.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { VSCodeProgressRing } from "@vscode/webview-ui-toolkit/react"; -import * as React from "react"; -import styled from "styled-components"; - -const SpinnerContainer = styled.span` - vertical-align: middle; - display: inline-block; -`; - -const DownloadSpinner = () => ( - - - -); - -export default DownloadSpinner; diff --git a/extensions/ql-vscode/src/view/remote-queries/FullScreenModal.tsx b/extensions/ql-vscode/src/view/remote-queries/FullScreenModal.tsx deleted file mode 100644 index 4de087f7f..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/FullScreenModal.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import * as React from "react"; -import { createPortal } from "react-dom"; -import styled from "styled-components"; -import { XCircleIcon } from "@primer/octicons-react"; - -const Container = styled.div` - position: fixed; - top: 0; - left: 0; - height: 100%; - width: 100%; - opacity: 1; - background-color: var(--vscode-editor-background); - z-index: 5000; - padding-top: 1em; -`; - -const CloseButton = styled.button` - position: absolute; - top: 1em; - right: 1em; - background-color: var(--vscode-editor-background); - border: none; -`; - -const FullScreenModal = ({ - setOpen, - containerElementId, - children, -}: { - setOpen: (open: boolean) => void; - containerElementId: string; - children: React.ReactNode; -}) => { - const containerElement = document.getElementById(containerElementId); - if (!containerElement) { - throw Error(`Could not find container element. Id: ${containerElementId}`); - } - - return createPortal( - <> - - setOpen(false)}> - - - {children} - - , - containerElement, - ); -}; - -export default FullScreenModal; diff --git a/extensions/ql-vscode/src/view/remote-queries/LastUpdated.tsx b/extensions/ql-vscode/src/view/remote-queries/LastUpdated.tsx deleted file mode 100644 index eb41ae4a7..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/LastUpdated.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import * as React from "react"; -import { RepoPushIcon } from "@primer/octicons-react"; -import styled from "styled-components"; - -import { humanizeRelativeTime } from "../../pure/time"; - -const IconContainer = styled.span` - flex-grow: 0; - text-align: right; - margin-right: 0; -`; - -const Duration = styled.span` - text-align: left; - width: 8em; - margin-left: 0.5em; -`; - -type Props = { lastUpdated?: number }; - -const LastUpdated = ({ lastUpdated }: Props) => - // lastUpdated will be undefined for older results that were - // created before the lastUpdated field was added. - Number.isFinite(lastUpdated) ? ( - <> - - - - {humanizeRelativeTime(lastUpdated)} - - ) : ( - <> - ); - -export default LastUpdated; diff --git a/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx b/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx deleted file mode 100644 index 08a9dbab6..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/RemoteQueries.tsx +++ /dev/null @@ -1,551 +0,0 @@ -import * as React from "react"; -import { useEffect, useState } from "react"; -import { Flash, ThemeProvider } from "@primer/react"; -import { ToRemoteQueriesMessage } from "../../pure/interface-types"; -import { - AnalysisSummary, - RemoteQueryResult, -} from "../../remote-queries/shared/remote-query-result"; -import { MAX_RAW_RESULTS } from "../../remote-queries/shared/result-limits"; -import { vscode } from "../vscode-api"; -import { VSCodeBadge, VSCodeButton } from "@vscode/webview-ui-toolkit/react"; -import { - HorizontalSpace, - SectionTitle, - VerticalSpace, - ViewTitle, -} from "../common"; -import DownloadButton from "./DownloadButton"; -import { - AnalysisResults, - getAnalysisResultCount, -} from "../../remote-queries/shared/analysis-result"; -import DownloadSpinner from "./DownloadSpinner"; -import CollapsibleItem from "./CollapsibleItem"; -import { - AlertIcon, - CodeSquareIcon, - FileCodeIcon, - RepoIcon, - TerminalIcon, -} from "@primer/octicons-react"; -import AnalysisAlertResult from "../variant-analysis/AnalysisAlertResult"; -import RawResultsTable from "../variant-analysis/RawResultsTable"; -import RepositoriesSearch from "./RepositoriesSearch"; -import StarCount from "../common/StarCount"; -import SortRepoFilter, { Sort, sorter } from "./SortRepoFilter"; -import LastUpdated from "./LastUpdated"; -import RepoListCopyButton from "./RepoListCopyButton"; - -import "./baseStyles.css"; -import "./remoteQueries.css"; - -const numOfReposInContractedMode = 10; - -const emptyQueryResult: RemoteQueryResult = { - queryId: "", - queryTitle: "", - queryFileName: "", - queryFilePath: "", - queryText: "", - language: "", - workflowRunUrl: "", - totalRepositoryCount: 0, - affectedRepositoryCount: 0, - totalResultCount: 0, - executionTimestamp: "", - executionDuration: "", - analysisSummaries: [], - analysisFailures: [], -}; - -const downloadAnalysisResults = (analysisSummary: AnalysisSummary) => { - vscode.postMessage({ - t: "remoteQueryDownloadAnalysisResults", - analysisSummary, - }); -}; - -const downloadAllAnalysesResults = (query: RemoteQueryResult) => { - vscode.postMessage({ - t: "remoteQueryDownloadAllAnalysesResults", - analysisSummaries: query.analysisSummaries, - }); -}; - -const openQueryFile = (queryResult: RemoteQueryResult) => { - vscode.postMessage({ - t: "openFile", - filePath: queryResult.queryFilePath, - }); -}; - -const openQueryTextVirtualFile = (queryResult: RemoteQueryResult) => { - vscode.postMessage({ - t: "openVirtualFile", - queryText: queryResult.queryText, - }); -}; - -function createResultsDescription(queryResult: RemoteQueryResult) { - const reposCount = `${queryResult.totalRepositoryCount} ${ - queryResult.totalRepositoryCount === 1 ? "repository" : "repositories" - }`; - return `${queryResult.totalResultCount} results from running against ${reposCount} (${queryResult.executionDuration}), ${queryResult.executionTimestamp}`; -} - -const sumAnalysesResults = (analysesResults: AnalysisResults[]) => - analysesResults.reduce((acc, curr) => acc + getAnalysisResultCount(curr), 0); - -const QueryInfo = (queryResult: RemoteQueryResult) => ( - <> - - {createResultsDescription(queryResult)} - - - openQueryFile(queryResult)} - > - - {" "} - {" "} - - {queryResult.queryFileName} - - - - openQueryTextVirtualFile(queryResult)} - > - - {" "} - {" "} - - Query - - - - - - {" "} - {" "} - - Logs - - - -); - -const Failures = (queryResult: RemoteQueryResult) => { - if (queryResult.analysisFailures.length === 0) { - return <>; - } - return ( - <> - - - {queryResult.analysisFailures.map((f, i) => ( -
-

- - {f.nwo}: - {f.error} -

- {i === queryResult.analysisFailures.length - 1 ? ( - <> - ) : ( - - )} -
- ))} - - - ); -}; - -const SummaryTitleWithResults = ({ - queryResult, - analysesResults, - sort, - setSort, -}: { - queryResult: RemoteQueryResult; - analysesResults: AnalysisResults[]; - sort: Sort; - setSort: (sort: Sort) => void; -}) => { - const showDownloadButton = - queryResult.totalResultCount !== sumAnalysesResults(analysesResults); - - return ( -
- - Repositories with results ({queryResult.affectedRepositoryCount}): - - {showDownloadButton && ( - downloadAllAnalysesResults(queryResult)} - /> - )} -
- - - -
-
- ); -}; - -const SummaryTitleNoResults = () => ( -
- No results found -
-); - -const SummaryItemDownload = ({ - analysisSummary, - analysisResults, -}: { - analysisSummary: AnalysisSummary; - analysisResults: AnalysisResults | undefined; -}) => { - if (!analysisResults || analysisResults.status === "Failed") { - return ( - downloadAnalysisResults(analysisSummary)} - /> - ); - } - - if (analysisResults.status === "InProgress") { - return ( - <> - - - - ); - } - - return <>; -}; - -const SummaryItem = ({ - analysisSummary, - analysisResults, -}: { - analysisSummary: AnalysisSummary; - analysisResults: AnalysisResults | undefined; -}) => ( - <> - - - - {analysisSummary.nwo} - - - {analysisSummary.resultCount.toString()} - - - - - - - -); - -const Summary = ({ - queryResult, - analysesResults, - sort, - setSort, -}: { - queryResult: RemoteQueryResult; - analysesResults: AnalysisResults[]; - sort: Sort; - setSort: (sort: Sort) => void; -}) => { - const [repoListExpanded, setRepoListExpanded] = useState(false); - const numOfReposToShow = repoListExpanded - ? queryResult.analysisSummaries.length - : numOfReposInContractedMode; - - return ( - <> - {queryResult.affectedRepositoryCount === 0 ? ( - - ) : ( - - )} - -
    - {queryResult.analysisSummaries - .slice(0, numOfReposToShow) - .sort(sorter(sort)) - .map((summary, i) => ( -
  • - a.nwo === summary.nwo, - )} - /> -
    • - ))} -
- {queryResult.analysisSummaries.length > numOfReposInContractedMode && ( - - )} - - ); -}; - -const AnalysesResultsTitle = ({ - totalAnalysesResults, - totalResults, -}: { - totalAnalysesResults: number; - totalResults: number; -}) => { - if (totalAnalysesResults === totalResults) { - return {totalAnalysesResults} results; - } - - return ( - - {totalAnalysesResults}/{totalResults} results - - ); -}; - -const exportResults = (queryResult: RemoteQueryResult) => { - vscode.postMessage({ - t: "remoteQueryExportResults", - queryId: queryResult.queryId, - }); -}; - -const AnalysesResultsDescription = ({ - queryResult, - analysesResults, -}: { - queryResult: RemoteQueryResult; - analysesResults: AnalysisResults[]; -}) => { - const showDownloadsMessage = queryResult.analysisSummaries.some( - (s) => - !analysesResults.some((a) => a.nwo === s.nwo && a.status === "Completed"), - ); - const downloadsMessage = ( - <> - - Some results haven't been downloaded automatically because of their - size or because enough were downloaded already. Download them manually - from the list above if you want to see them here. - - ); - - const showMaxResultsMessage = analysesResults.some( - (a) => a.rawResults?.capped, - ); - const maxRawResultsMessage = ( - <> - - Some repositories have more than {MAX_RAW_RESULTS} results. We will only - show you up to  - {MAX_RAW_RESULTS} results for each repository. - - ); - - return ( - <> - {showDownloadsMessage && downloadsMessage} - {showMaxResultsMessage && maxRawResultsMessage} - - ); -}; - -const RepoAnalysisResults = (analysisResults: AnalysisResults) => { - const numOfResults = getAnalysisResultCount(analysisResults); - const title = ( - <> - {analysisResults.nwo} - - {numOfResults.toString()} - - ); - - return ( - -
    - {analysisResults.interpretedResults.map((r, i) => ( -
  • - - -
    • - ))} -
- {analysisResults.rawResults && ( - - )} - - ); -}; - -const AnalysesResults = ({ - queryResult, - analysesResults, - totalResults, - sort, -}: { - queryResult: RemoteQueryResult; - analysesResults: AnalysisResults[]; - totalResults: number; - sort: Sort; -}) => { - const totalAnalysesResults = sumAnalysesResults(analysesResults); - const [filterValue, setFilterValue] = useState(""); - - if (totalResults === 0) { - return <>; - } - - return ( - <> - -
-
- -
-
- exportResults(queryResult)}> - Export all - -
-
- - - - - -
    - {analysesResults - .filter( - (a) => - a.interpretedResults.length || - a.rawResults?.resultSet?.rows?.length, - ) - .filter((a) => - a.nwo.toLowerCase().includes(filterValue.toLowerCase()), - ) - .sort(sorter(sort)) - .map((r) => ( -
  • - -
    • - ))} -
- - ); -}; - -export function RemoteQueries(): JSX.Element { - const [queryResult, setQueryResult] = - useState(emptyQueryResult); - const [analysesResults, setAnalysesResults] = useState([]); - const [sort, setSort] = useState("name"); - - useEffect(() => { - const listener = (evt: MessageEvent) => { - if (evt.origin === window.origin) { - const msg: ToRemoteQueriesMessage = evt.data; - if (msg.t === "setRemoteQueryResult") { - setQueryResult(msg.queryResult); - } else if (msg.t === "setAnalysesResults") { - setAnalysesResults(msg.analysesResults); - } - } else { - // sanitize origin - const origin = evt.origin.replace(/\n|\r/g, ""); - console.error(`Invalid event origin ${origin}`); - } - }; - window.addEventListener("message", listener); - - return () => { - window.removeEventListener("message", listener); - }; - }, []); - - if (!queryResult) { - return
Waiting for results to load.
; - } - - try { - return ( -
- - {queryResult.queryTitle} - - - - - -
- ); - } catch (err) { - console.error(err); - return
There was an error displaying the view.
; - } -} diff --git a/extensions/ql-vscode/src/view/remote-queries/RepoListCopyButton.tsx b/extensions/ql-vscode/src/view/remote-queries/RepoListCopyButton.tsx deleted file mode 100644 index a241c17a9..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/RepoListCopyButton.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import * as React from "react"; -import { vscode } from "../vscode-api"; -import { RemoteQueryResult } from "../../remote-queries/shared/remote-query-result"; -import { CopyIcon } from "@primer/octicons-react"; -import { IconButton } from "@primer/react"; - -const copyRepositoryList = (queryResult: RemoteQueryResult) => { - vscode.postMessage({ - t: "copyRepoList", - queryId: queryResult.queryId, - }); -}; - -const RepoListCopyButton = ({ - queryResult, -}: { - queryResult: RemoteQueryResult; -}) => ( - copyRepositoryList(queryResult)} - /> -); - -export default RepoListCopyButton; diff --git a/extensions/ql-vscode/src/view/remote-queries/RepositoriesSearch.tsx b/extensions/ql-vscode/src/view/remote-queries/RepositoriesSearch.tsx deleted file mode 100644 index dfcdf14ad..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/RepositoriesSearch.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import * as React from "react"; -import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react"; - -interface RepositoriesSearchProps { - filterValue: string; - setFilterValue: (value: string) => void; -} - -const RepositoriesSearch = ({ - filterValue, - setFilterValue, -}: RepositoriesSearchProps) => { - return ( - <> - - setFilterValue((e.target as HTMLInputElement).value) - } - > - - - - ); -}; - -export default RepositoriesSearch; diff --git a/extensions/ql-vscode/src/view/remote-queries/SortRepoFilter.tsx b/extensions/ql-vscode/src/view/remote-queries/SortRepoFilter.tsx deleted file mode 100644 index 6c63b5c73..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/SortRepoFilter.tsx +++ /dev/null @@ -1,94 +0,0 @@ -import * as React from "react"; -import { FilterIcon } from "@primer/octicons-react"; -import { ActionList, ActionMenu, IconButton } from "@primer/react"; -import styled from "styled-components"; - -const SortWrapper = styled.span` - flex-grow: 2; - text-align: right; - margin-right: 0; -`; - -export type Sort = "name" | "stars" | "results" | "lastUpdated"; -type Props = { - sort: Sort; - setSort: (sort: Sort) => void; -}; - -type Sortable = { - nwo: string; - starCount?: number; - resultCount?: number; - lastUpdated?: number; -}; - -const sortBy = [ - { name: "Sort by Name", sort: "name" }, - { name: "Sort by Results", sort: "results" }, - { name: "Sort by Stars", sort: "stars" }, - { name: "Sort by Last Updated", sort: "lastUpdated" }, -]; - -export function sorter( - sort: Sort, -): (left: Sortable, right: Sortable) => number { - // stars and results are highest to lowest - // name is alphabetical - return (left: Sortable, right: Sortable) => { - if (sort === "stars") { - const stars = (right.starCount || 0) - (left.starCount || 0); - if (stars !== 0) { - return stars; - } - } - if (sort === "lastUpdated") { - const lastUpdated = (right.lastUpdated || 0) - (left.lastUpdated || 0); - if (lastUpdated !== 0) { - return lastUpdated; - } - } - if (sort === "results") { - const results = (right.resultCount || 0) - (left.resultCount || 0); - if (results !== 0) { - return results; - } - } - - // Fall back on name compare if results, stars, or lastUpdated are equal - return left.nwo.localeCompare(right.nwo, undefined, { - sensitivity: "base", - }); - }; -} - -const SortRepoFilter = ({ sort, setSort }: Props) => { - return ( - - - - - - - - - {sortBy.map((type, index) => ( - setSort(type.sort as Sort)} - > - {type.name} - - ))} - - - - - ); -}; - -export default SortRepoFilter; diff --git a/extensions/ql-vscode/src/view/remote-queries/baseStyles.css b/extensions/ql-vscode/src/view/remote-queries/baseStyles.css deleted file mode 100644 index 4d3d4d543..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/baseStyles.css +++ /dev/null @@ -1,4 +0,0 @@ -body { - font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial, - sans-serif, Apple Color Emoji, Segoe UI Emoji; -} diff --git a/extensions/ql-vscode/src/view/remote-queries/index.tsx b/extensions/ql-vscode/src/view/remote-queries/index.tsx deleted file mode 100644 index d7292eeca..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/index.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import * as React from "react"; -import { WebviewDefinition } from "../webview-definition"; -import { RemoteQueries } from "./RemoteQueries"; - -const definition: WebviewDefinition = { - component: , -}; - -export default definition; diff --git a/extensions/ql-vscode/src/view/remote-queries/remoteQueries.css b/extensions/ql-vscode/src/view/remote-queries/remoteQueries.css deleted file mode 100644 index 943de24f4..000000000 --- a/extensions/ql-vscode/src/view/remote-queries/remoteQueries.css +++ /dev/null @@ -1,53 +0,0 @@ -.vscode-codeql__remote-queries { - max-width: 55em; -} - -.vscode-codeql__query-info-link { - text-decoration: none; - padding-right: 1em; - color: var(--vscode-editor-foreground); -} - -.vscode-codeql__query-info-link:hover { - color: var(--vscode-editor-foreground); -} - -.vscode-codeql__query-summary-container { - padding-top: 1.5em; - display: flex; -} - -.vscode-codeql__analysis-summaries-list-item { - margin-top: 0.5em; - display: flex; -} - -.vscode-codeql__analyses-results-list-item { - padding-top: 0.5em; -} - -.vscode-codeql__analysis-item { - padding-right: 0.1em; -} - -.vscode-codeql__expand-button { - background: none; - color: var(--vscode-textLink-foreground); - border: none; - cursor: pointer; - padding-top: 1em; - font-size: x-small; -} - -.vscode-codeql__analysis-failure { - margin: 0; - font-family: ui-monospace, SFMono-Regular, SF Mono, Menlo, Consolas, - Liberation Mono, monospace; - color: var(--vscode-editor-foreground); -} - -.vscode-codeql__flat-list { - list-style-type: none; - margin: 0; - padding: 0.5em 0 0 0; -} From 5a9d12ea606c054e385e550b006281dee34795a9 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 7 Feb 2023 22:16:37 +0000 Subject: [PATCH 011/132] Introduce wrapper for `codeql pack add` CLI command Similar to what we do with `codeql pack install`. Tnis will simulate us running `codeql pack add codeql/-all`. We're going to need in order to: - generate a lock file (codeql-pack.lock.yaml) - install the correct packages for our skeleton QL pack based on the lock file. --- extensions/ql-vscode/src/cli.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index e00f50f77..5fd275e44 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -28,6 +28,7 @@ import { CompilationMessage } from "./pure/legacy-messages"; import { sarifParser } from "./sarif-parser"; import { dbSchemeToLanguage, walkDirectory } from "./helpers"; import { App } from "./common/app"; +import { QueryLanguage } from "./qlpack-generator"; /** * The version of the SARIF format that we are using. @@ -1216,6 +1217,23 @@ export class CodeQLCliServer implements Disposable { ); } + /** + * Adds a list of QL library packs with optional version ranges as dependencies of + * the current package, and then installs them. This command modifies the qlpack.yml + * file of the current package. Formatting and comments will be removed. + * @param dir The directory where QL pack exists. + * @param language The language of the QL pack. + */ + async packAdd(dir: string, queryLanguage: QueryLanguage) { + const args = ["--dir", dir]; + args.push(`codeql/${queryLanguage}-all`); + return this.runJsonCodeQlCliCommandWithAuthentication( + ["pack", "add"], + args, + "Adding pack dependencies and installing them", + ); + } + /** * Downloads a specified pack. * @param packs The `` of the packs to download. From 6cda6534d1c3dba9578fde9f35ecb7c7c1a34852 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 7 Feb 2023 22:17:08 +0000 Subject: [PATCH 012/132] Introduce a QlPackGenerator class This will receive a folder name and language. It will generate: - a `codeql-pack.yml` file - an `example.ql` file - a `codeql-pack.lock.yml` file It will also install dependencies listed in `codeql-pack.lock.yml` file. We were initially planning to call the `packInstall` command once we generate `codeql-pack.yml` in order to install dependencies. However, the `packAdd` command does this for us, as well as generating a lock file. Rather than trying to craft the lock file by hand, we're opting to use the cli command. NB: We're introducing a new `QueryLanguage` type which is identical to the `VariantAnalysisQueryLanguage`. In a subsequent PR we'll unify these two types. --- extensions/ql-vscode/src/qlpack-generator.ts | 85 +++++++++++++++++++ .../qlpack-generator.test.ts | 53 ++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 extensions/ql-vscode/src/qlpack-generator.ts create mode 100644 extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts new file mode 100644 index 000000000..b9b438508 --- /dev/null +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -0,0 +1,85 @@ +import { mkdir, writeFile } from "fs-extra"; +import { dump } from "js-yaml"; +import { join } from "path"; +import { CodeQLCliServer } from "./cli"; + +export type QueryLanguage = + | "csharp" + | "cpp" + | "go" + | "java" + | "javascript" + | "python" + | "ruby" + | "swift"; + +export class QlPackGenerator { + private readonly qlpackName: string; + private readonly qlpackVersion: string; + private readonly header: string; + private readonly qlpackFileName: string; + + constructor( + private readonly folderName: string, + private readonly queryLanguage: QueryLanguage, + private readonly cliServer: CodeQLCliServer, + ) { + this.qlpackName = `getting-started/codeql-extra-queries-${this.queryLanguage}`; + this.qlpackVersion = "1.0.0"; + this.header = "# This is an automatically generated file.\n\n"; + + this.qlpackFileName = "qlpack.yml"; + } + + public async generate() { + await mkdir(this.folderName); + + // create qlpack.yml + await this.createQlPackYaml(); + + // create example.ql + await this.createExampleQlFile(); + + // create codeql-pack.lock.yml + await this.createCodeqlPackLockYaml(); + } + + private async createQlPackYaml() { + const qlPackFile = join(this.folderName, this.qlpackFileName); + + const qlPackYml = { + name: this.qlpackName, + version: this.qlpackVersion, + dependencies: { + [`codeql/${this.queryLanguage}-all`]: "*", + }, + }; + + await writeFile(qlPackFile, this.header + dump(qlPackYml), "utf8"); + } + + private async createExampleQlFile() { + const exampleQlFile = join(this.folderName, "example.ql"); + + const exampleQl = ` +/** + * @name Empty block + * @kind problem + * @problem.severity warning + * @id ${this.queryLanguage}/example/empty-block + */ + +import ${this.queryLanguage} + +from BlockStmt b +where b.getNumStmt() = 0 +select b, "This is an empty block." +`.trim(); + + await writeFile(exampleQlFile, exampleQl, "utf8"); + } + + private async createCodeqlPackLockYaml() { + await this.cliServer.packAdd(this.folderName, this.queryLanguage); + } +} diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts new file mode 100644 index 000000000..5fcc7ccd5 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts @@ -0,0 +1,53 @@ +import { join } from "path"; +import { existsSync, rmdirSync } from "fs"; +import { QlPackGenerator, QueryLanguage } from "../../../src/qlpack-generator"; +import { CodeQLCliServer } from "../../../src/cli"; + +describe("QlPackGenerator", () => { + let packfolderName: string; + let qlPackYamlFilePath: string; + let exampleQlFilePath: string; + let language: string; + let generator: QlPackGenerator; + let packAddSpy: jest.SpyInstance; + + beforeEach(async () => { + language = "ruby"; + packfolderName = `test-ql-pack-${language}`; + qlPackYamlFilePath = join(packfolderName, "qlpack.yml"); + exampleQlFilePath = join(packfolderName, "example.ql"); + + packAddSpy = jest.fn(); + const mockCli = { + packAdd: packAddSpy, + } as unknown as CodeQLCliServer; + + generator = new QlPackGenerator( + packfolderName, + language as QueryLanguage, + mockCli, + ); + }); + + afterEach(async () => { + try { + rmdirSync(packfolderName, { recursive: true }); + } catch (e) { + // ignore + } + }); + + it("should generate a QL pack", async () => { + expect(existsSync(packfolderName)).toBe(false); + expect(existsSync(qlPackYamlFilePath)).toBe(false); + expect(existsSync(exampleQlFilePath)).toBe(false); + + await generator.generate(); + + expect(existsSync(packfolderName)).toBe(true); + expect(existsSync(qlPackYamlFilePath)).toBe(true); + expect(existsSync(exampleQlFilePath)).toBe(true); + + expect(packAddSpy).toHaveBeenCalledWith(packfolderName, language); + }); +}); From a8f36ee9e84a8692eba58121cfa2850ec39aaf68 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 7 Feb 2023 22:17:51 +0000 Subject: [PATCH 013/132] Generate a QL pack when you add a new database, if one is missing --- extensions/ql-vscode/src/databases.ts | 22 ++++++- .../minimal-workspace/databases.test.ts | 64 +++++++++++++++++-- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/extensions/ql-vscode/src/databases.ts b/extensions/ql-vscode/src/databases.ts index e6651b8e0..1624d43cd 100644 --- a/extensions/ql-vscode/src/databases.ts +++ b/extensions/ql-vscode/src/databases.ts @@ -26,6 +26,7 @@ import { QueryRunner } from "./queryRunner"; import { pathsEqual } from "./pure/files"; import { redactableError } from "./pure/errors"; import { isCodespacesTemplate } from "./config"; +import { QlPackGenerator, QueryLanguage } from "./qlpack-generator"; /** * databases.ts @@ -655,9 +656,26 @@ export class DatabaseManager extends DisposableObject { return; } - await showBinaryChoiceDialog( - `We've noticed you don't have a QL pack downloaded to analyze this database. Can we set up a ${databaseItem.language} query pack for you`, + const answer = await showBinaryChoiceDialog( + `We've noticed you don't have a QL pack downloaded to analyze this database. Can we set up a query pack for you?`, ); + + if (!answer) { + return; + } + + try { + const qlPackGenerator = new QlPackGenerator( + folderName, + databaseItem.language as QueryLanguage, + this.cli, + ); + await qlPackGenerator.generate(); + } catch (e: unknown) { + void this.logger.log( + `Could not create skeleton QL pack: ${getErrorMessage(e)}`, + ); + } } private async reregisterDatabases( diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts index 068be7385..f3f13d451 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts @@ -23,6 +23,7 @@ import { testDisposeHandler } from "../test-dispose-handler"; import { QueryRunner } from "../../../src/queryRunner"; import * as helpers from "../../../src/helpers"; import { Setting } from "../../../src/config"; +import { QlPackGenerator } from "../../../src/qlpack-generator"; describe("databases", () => { const MOCK_DB_OPTIONS: FullDatabaseOptions = { @@ -37,6 +38,7 @@ describe("databases", () => { let registerSpy: jest.Mock, []>; let deregisterSpy: jest.Mock, []>; let resolveDatabaseSpy: jest.Mock, []>; + let packAddSpy: jest.Mock; let logSpy: jest.Mock; let showBinaryChoiceDialogSpy: jest.SpiedFunction< @@ -52,6 +54,7 @@ describe("databases", () => { registerSpy = jest.fn(() => Promise.resolve(undefined)); deregisterSpy = jest.fn(() => Promise.resolve(undefined)); resolveDatabaseSpy = jest.fn(() => Promise.resolve({} as DbInfo)); + packAddSpy = jest.fn(); logSpy = jest.fn(() => { /* */ }); @@ -79,6 +82,7 @@ describe("databases", () => { } as unknown as QueryRunner, { resolveDatabase: resolveDatabaseSpy, + packAdd: packAddSpy, } as unknown as CodeQLCliServer, { log: logSpy, @@ -589,20 +593,66 @@ describe("databases", () => { describe("createSkeletonPacks", () => { let mockDbItem: DatabaseItemImpl; + let packfolderName: string; + let qlPackYamlFilePath: string; + let exampleQlFilePath: string; + let language: string; + + beforeEach(() => { + language = "ruby"; + + const options: FullDatabaseOptions = { + dateAdded: 123, + ignoreSourceArchive: false, + language, + }; + mockDbItem = createMockDB(options); + + packfolderName = `codeql-custom-queries-${mockDbItem.language}`; + qlPackYamlFilePath = join(packfolderName, "qlpack.yml"); + exampleQlFilePath = join(packfolderName, "example.ql"); + }); + + afterEach(async () => { + try { + fs.rmdirSync(packfolderName, { recursive: true }); + } catch (e) { + // ignore + } + }); describe("when the language is set", () => { it("should offer the user to set up a skeleton QL pack", async () => { - const options: FullDatabaseOptions = { - dateAdded: 123, - ignoreSourceArchive: false, - language: "ruby", - }; - mockDbItem = createMockDB(options); - await (databaseManager as any).createSkeletonPacks(mockDbItem); expect(showBinaryChoiceDialogSpy).toBeCalledTimes(1); }); + + it("should return early if the user refuses help", async () => { + showBinaryChoiceDialogSpy = jest + .spyOn(helpers, "showBinaryChoiceDialog") + .mockResolvedValue(false); + + const generateSpy = jest.spyOn(QlPackGenerator.prototype, "generate"); + + await (databaseManager as any).createSkeletonPacks(mockDbItem); + + expect(generateSpy).not.toBeCalled(); + }); + + it("should create the skeleton QL pack for the user", async () => { + expect(fs.existsSync(packfolderName)).toBe(false); + expect(fs.existsSync(qlPackYamlFilePath)).toBe(false); + expect(fs.existsSync(exampleQlFilePath)).toBe(false); + + await (databaseManager as any).createSkeletonPacks(mockDbItem); + + expect(fs.existsSync(packfolderName)).toBe(true); + expect(fs.existsSync(qlPackYamlFilePath)).toBe(true); + expect(fs.existsSync(exampleQlFilePath)).toBe(true); + + expect(packAddSpy).toHaveBeenCalledWith(packfolderName, language); + }); }); describe("when the language is not set", () => { From b04b3bf33e169058a6d314c6ff428f9b9d734ba3 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Thu, 9 Feb 2023 14:51:09 +0000 Subject: [PATCH 014/132] Reword `packAdd` output Co-authored-by: Andrew Eisenberg --- extensions/ql-vscode/src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index 5fd275e44..ec0999645 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -1230,7 +1230,7 @@ export class CodeQLCliServer implements Disposable { return this.runJsonCodeQlCliCommandWithAuthentication( ["pack", "add"], args, - "Adding pack dependencies and installing them", + "Adding and installing pack dependencies.", ); } From 3464cd0cda95d00c95821aaf3807cdcd783f5eb3 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Thu, 9 Feb 2023 14:54:11 +0000 Subject: [PATCH 015/132] Reword dialog box prompt Co-authored-by: Andrew Eisenberg --- extensions/ql-vscode/src/databases.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/databases.ts b/extensions/ql-vscode/src/databases.ts index 1624d43cd..c9fee8836 100644 --- a/extensions/ql-vscode/src/databases.ts +++ b/extensions/ql-vscode/src/databases.ts @@ -657,7 +657,7 @@ export class DatabaseManager extends DisposableObject { } const answer = await showBinaryChoiceDialog( - `We've noticed you don't have a QL pack downloaded to analyze this database. Can we set up a query pack for you?`, + `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`, ); if (!answer) { From c4bed4e8aae40ab2f6f9d2f55c2fff06f96f2359 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Fri, 10 Feb 2023 12:56:12 +0000 Subject: [PATCH 016/132] Simplify example query to make it work with all languages --- extensions/ql-vscode/src/qlpack-generator.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index b9b438508..6fbfbe8ca 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -71,9 +71,7 @@ export class QlPackGenerator { import ${this.queryLanguage} -from BlockStmt b -where b.getNumStmt() = 0 -select b, "This is an empty block." +select "Hello, world!" `.trim(); await writeFile(exampleQlFile, exampleQl, "utf8"); From ce3e19a2d7a7e3050db76c511a10bb45e8790ab1 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Fri, 10 Feb 2023 13:03:24 +0000 Subject: [PATCH 017/132] Mention that the query file is automatically generated --- extensions/ql-vscode/src/qlpack-generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index 6fbfbe8ca..6f8109783 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -63,6 +63,7 @@ export class QlPackGenerator { const exampleQl = ` /** + * This is an automatically generated file * @name Empty block * @kind problem * @problem.severity warning From 24eb8fd307d5adc8d439767447982565778f71b4 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 12:41:22 +0000 Subject: [PATCH 018/132] Store QL pack in workspace instead of VSCode storage We're checking that the skeleton QL pack doesn't exist as a workspace folder, so we should be creating this folder in the workspace as well. Initially this was being created in VSCode's local storage. --- extensions/ql-vscode/src/databases.ts | 1 + extensions/ql-vscode/src/qlpack-generator.ts | 33 ++++++++++++++---- .../qlpack-generator.test.ts | 34 +++++++++++++------ 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/extensions/ql-vscode/src/databases.ts b/extensions/ql-vscode/src/databases.ts index c9fee8836..995157883 100644 --- a/extensions/ql-vscode/src/databases.ts +++ b/extensions/ql-vscode/src/databases.ts @@ -669,6 +669,7 @@ export class DatabaseManager extends DisposableObject { folderName, databaseItem.language as QueryLanguage, this.cli, + this.ctx.storageUri?.fsPath, ); await qlPackGenerator.generate(); } catch (e: unknown) { diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index 6f8109783..57fc809ea 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -1,6 +1,7 @@ -import { mkdir, writeFile } from "fs-extra"; +import { writeFile } from "fs-extra"; import { dump } from "js-yaml"; import { join } from "path"; +import { Uri, workspace } from "vscode"; import { CodeQLCliServer } from "./cli"; export type QueryLanguage = @@ -18,21 +19,28 @@ export class QlPackGenerator { private readonly qlpackVersion: string; private readonly header: string; private readonly qlpackFileName: string; + private readonly folderUri: Uri; constructor( private readonly folderName: string, private readonly queryLanguage: QueryLanguage, private readonly cliServer: CodeQLCliServer, + private readonly storagePath: string | undefined, ) { + if (this.storagePath === undefined) { + throw new Error("Workspace storage path is undefined"); + } this.qlpackName = `getting-started/codeql-extra-queries-${this.queryLanguage}`; this.qlpackVersion = "1.0.0"; this.header = "# This is an automatically generated file.\n\n"; this.qlpackFileName = "qlpack.yml"; + this.folderUri = Uri.parse(join(this.storagePath, this.folderName)); } public async generate() { - await mkdir(this.folderName); + // create QL pack folder and add to workspace + await this.createWorkspaceFolder(); // create qlpack.yml await this.createQlPackYaml(); @@ -44,8 +52,19 @@ export class QlPackGenerator { await this.createCodeqlPackLockYaml(); } + private async createWorkspaceFolder() { + await workspace.fs.createDirectory(this.folderUri); + + const end = (workspace.workspaceFolders || []).length; + + await workspace.updateWorkspaceFolders(end, 0, { + name: this.folderName, + uri: this.folderUri, + }); + } + private async createQlPackYaml() { - const qlPackFile = join(this.folderName, this.qlpackFileName); + const qlPackFilePath = join(this.folderUri.path, this.qlpackFileName); const qlPackYml = { name: this.qlpackName, @@ -55,11 +74,11 @@ export class QlPackGenerator { }, }; - await writeFile(qlPackFile, this.header + dump(qlPackYml), "utf8"); + await writeFile(qlPackFilePath, this.header + dump(qlPackYml), "utf8"); } private async createExampleQlFile() { - const exampleQlFile = join(this.folderName, "example.ql"); + const exampleQlFilePath = join(this.folderUri.path, "example.ql"); const exampleQl = ` /** @@ -75,10 +94,10 @@ import ${this.queryLanguage} select "Hello, world!" `.trim(); - await writeFile(exampleQlFile, exampleQl, "utf8"); + await writeFile(exampleQlFilePath, exampleQl, "utf8"); } private async createCodeqlPackLockYaml() { - await this.cliServer.packAdd(this.folderName, this.queryLanguage); + await this.cliServer.packAdd(this.folderUri.path, this.queryLanguage); } } diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts index 5fcc7ccd5..b51e6c432 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts @@ -1,10 +1,14 @@ import { join } from "path"; -import { existsSync, rmdirSync } from "fs"; +import { existsSync, rmSync } from "fs"; import { QlPackGenerator, QueryLanguage } from "../../../src/qlpack-generator"; import { CodeQLCliServer } from "../../../src/cli"; +import { isFolderAlreadyInWorkspace } from "../../../src/helpers"; +import { workspace } from "vscode"; +import { getErrorMessage } from "../../../src/pure/helpers-pure"; describe("QlPackGenerator", () => { - let packfolderName: string; + let packFolderName: string; + let packFolderPath: string; let qlPackYamlFilePath: string; let exampleQlFilePath: string; let language: string; @@ -13,9 +17,11 @@ describe("QlPackGenerator", () => { beforeEach(async () => { language = "ruby"; - packfolderName = `test-ql-pack-${language}`; - qlPackYamlFilePath = join(packfolderName, "qlpack.yml"); - exampleQlFilePath = join(packfolderName, "example.ql"); + packFolderName = `test-ql-pack-${language}`; + packFolderPath = join(__dirname, packFolderName); + + qlPackYamlFilePath = join(packFolderPath, "qlpack.yml"); + exampleQlFilePath = join(packFolderPath, "example.ql"); packAddSpy = jest.fn(); const mockCli = { @@ -23,31 +29,37 @@ describe("QlPackGenerator", () => { } as unknown as CodeQLCliServer; generator = new QlPackGenerator( - packfolderName, + packFolderName, language as QueryLanguage, mockCli, + __dirname, ); }); afterEach(async () => { try { - rmdirSync(packfolderName, { recursive: true }); + rmSync(packFolderPath, { recursive: true }); + + const end = (workspace.workspaceFolders || []).length; + workspace.updateWorkspaceFolders(end - 1, 1); } catch (e) { - // ignore + console.log( + `Could not remove folder from workspace: ${getErrorMessage(e)}`, + ); } }); it("should generate a QL pack", async () => { - expect(existsSync(packfolderName)).toBe(false); + expect(isFolderAlreadyInWorkspace(packFolderName)).toBe(false); expect(existsSync(qlPackYamlFilePath)).toBe(false); expect(existsSync(exampleQlFilePath)).toBe(false); await generator.generate(); - expect(existsSync(packfolderName)).toBe(true); + expect(isFolderAlreadyInWorkspace(packFolderName)).toBe(true); expect(existsSync(qlPackYamlFilePath)).toBe(true); expect(existsSync(exampleQlFilePath)).toBe(true); - expect(packAddSpy).toHaveBeenCalledWith(packfolderName, language); + expect(packAddSpy).toHaveBeenCalledWith(packFolderPath, language); }); }); From d6ccc1113548e6d6700c63741bb1878e9bc59998 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 12:41:46 +0000 Subject: [PATCH 019/132] Update tests to check we call the generator We don't need to repeat the tests for the generator functionality here. All we want to check is that the generator is triggered correctly. --- .../minimal-workspace/databases.test.ts | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts index f3f13d451..71acfe65e 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts @@ -593,10 +593,8 @@ describe("databases", () => { describe("createSkeletonPacks", () => { let mockDbItem: DatabaseItemImpl; - let packfolderName: string; - let qlPackYamlFilePath: string; - let exampleQlFilePath: string; let language: string; + let generateSpy: jest.SpyInstance; beforeEach(() => { language = "ruby"; @@ -608,17 +606,9 @@ describe("databases", () => { }; mockDbItem = createMockDB(options); - packfolderName = `codeql-custom-queries-${mockDbItem.language}`; - qlPackYamlFilePath = join(packfolderName, "qlpack.yml"); - exampleQlFilePath = join(packfolderName, "example.ql"); - }); - - afterEach(async () => { - try { - fs.rmdirSync(packfolderName, { recursive: true }); - } catch (e) { - // ignore - } + generateSpy = jest + .spyOn(QlPackGenerator.prototype, "generate") + .mockImplementation(() => Promise.resolve()); }); describe("when the language is set", () => { @@ -633,25 +623,15 @@ describe("databases", () => { .spyOn(helpers, "showBinaryChoiceDialog") .mockResolvedValue(false); - const generateSpy = jest.spyOn(QlPackGenerator.prototype, "generate"); - await (databaseManager as any).createSkeletonPacks(mockDbItem); expect(generateSpy).not.toBeCalled(); }); it("should create the skeleton QL pack for the user", async () => { - expect(fs.existsSync(packfolderName)).toBe(false); - expect(fs.existsSync(qlPackYamlFilePath)).toBe(false); - expect(fs.existsSync(exampleQlFilePath)).toBe(false); - await (databaseManager as any).createSkeletonPacks(mockDbItem); - expect(fs.existsSync(packfolderName)).toBe(true); - expect(fs.existsSync(qlPackYamlFilePath)).toBe(true); - expect(fs.existsSync(exampleQlFilePath)).toBe(true); - - expect(packAddSpy).toHaveBeenCalledWith(packfolderName, language); + expect(generateSpy).toBeCalled(); }); }); From 1ae52ef1cc8aff8a1be82f8d5562e78a01527170 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 15:13:51 +0000 Subject: [PATCH 020/132] Mock `storageUri` for workspace in tests So that we can provide this to the generator. --- .../minimal-workspace/databases.test.ts | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts index 71acfe65e..cd81841cb 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases.test.ts @@ -33,6 +33,7 @@ describe("databases", () => { }; let databaseManager: DatabaseManager; + let extensionContext: ExtensionContext; let updateSpy: jest.Mock, []>; let registerSpy: jest.Mock, []>; @@ -63,16 +64,19 @@ describe("databases", () => { .spyOn(helpers, "showBinaryChoiceDialog") .mockResolvedValue(true); + extensionContext = { + workspaceState: { + update: updateSpy, + get: () => [], + }, + // pretend like databases added in the temp dir are controlled by the extension + // so that they are deleted upon removal + storagePath: dir.name, + storageUri: Uri.parse(dir.name), + } as unknown as ExtensionContext; + databaseManager = new DatabaseManager( - { - workspaceState: { - update: updateSpy, - get: () => [], - }, - // pretend like databases added in the temp dir are controlled by the extension - // so that they are deleted upon removal - storagePath: dir.name, - } as unknown as ExtensionContext, + extensionContext, { registerDatabase: registerSpy, deregisterDatabase: deregisterSpy, From 950a218c2964dc347a0df2a957ffac4963357453 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 14 Feb 2023 17:10:16 +0000 Subject: [PATCH 021/132] Enable eslint rule to disallow awaiting a non-thenable type --- extensions/ql-vscode/.eslintrc.js | 1 + extensions/ql-vscode/src/extension.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/ql-vscode/.eslintrc.js b/extensions/ql-vscode/.eslintrc.js index 77ab05e13..a8463cac1 100644 --- a/extensions/ql-vscode/.eslintrc.js +++ b/extensions/ql-vscode/.eslintrc.js @@ -29,6 +29,7 @@ const baseConfig = { "plugin:@typescript-eslint/recommended", ], rules: { + "@typescript-eslint/await-thenable": "error", "@typescript-eslint/no-use-before-define": 0, "@typescript-eslint/no-unused-vars": [ "warn", diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index ea68753fa..fce563b93 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -462,7 +462,7 @@ export async function activate( ) { registerErrorStubs([checkForUpdatesCommand], (command) => async () => { const installActionName = "Install CodeQL CLI"; - const chosenAction = await void showAndLogErrorMessage( + const chosenAction = await showAndLogErrorMessage( `Can't execute ${command}: missing CodeQL CLI.`, { items: [installActionName], From a59a008d8e0ef3a178a8b0b4ac5d142a402d125d Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 17:12:31 +0000 Subject: [PATCH 022/132] Use temporary directory for generator tests This will hopefully work with Windows tests as well. --- .../minimal-workspace/qlpack-generator.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts index b51e6c432..f015f442c 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts @@ -1,10 +1,11 @@ import { join } from "path"; -import { existsSync, rmSync } from "fs"; +import { existsSync } from "fs"; import { QlPackGenerator, QueryLanguage } from "../../../src/qlpack-generator"; import { CodeQLCliServer } from "../../../src/cli"; import { isFolderAlreadyInWorkspace } from "../../../src/helpers"; import { workspace } from "vscode"; import { getErrorMessage } from "../../../src/pure/helpers-pure"; +import * as tmp from "tmp"; describe("QlPackGenerator", () => { let packFolderName: string; @@ -14,11 +15,14 @@ describe("QlPackGenerator", () => { let language: string; let generator: QlPackGenerator; let packAddSpy: jest.SpyInstance; + let dir: tmp.DirResult; beforeEach(async () => { + dir = tmp.dirSync(); + language = "ruby"; packFolderName = `test-ql-pack-${language}`; - packFolderPath = join(__dirname, packFolderName); + packFolderPath = join(dir.name, packFolderName); qlPackYamlFilePath = join(packFolderPath, "qlpack.yml"); exampleQlFilePath = join(packFolderPath, "example.ql"); @@ -32,13 +36,13 @@ describe("QlPackGenerator", () => { packFolderName, language as QueryLanguage, mockCli, - __dirname, + dir.name, ); }); afterEach(async () => { try { - rmSync(packFolderPath, { recursive: true }); + dir.removeCallback(); const end = (workspace.workspaceFolders || []).length; workspace.updateWorkspaceFolders(end - 1, 1); From f1227dd2ebffd1c3b4850c44cc456aae3b464b82 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 18:01:05 +0000 Subject: [PATCH 023/132] Build directory URI using `Uri.file` instead of `Uri.parse` `Uri.parse` will not work with Windows paths as it will consider `C:\path` to indicate a file scheme (the "C:" part) and will complain about it. With `Uri.file` we can build the URI without hitting this complication. --- extensions/ql-vscode/src/qlpack-generator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index 57fc809ea..1fa0f61e1 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -35,7 +35,7 @@ export class QlPackGenerator { this.header = "# This is an automatically generated file.\n\n"; this.qlpackFileName = "qlpack.yml"; - this.folderUri = Uri.parse(join(this.storagePath, this.folderName)); + this.folderUri = Uri.file(join(this.storagePath, this.folderName)); } public async generate() { From 3e87a2d53cfad94fee3ce7d6890e551a0fa0f503 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 18:02:22 +0000 Subject: [PATCH 024/132] Remove directory from workspace by index We were initially always removing the last folder in the workspace as we assumed that would be the directory we use. Now that we've switched to using a temporary directory, this is no longer the case so we need to find the index of the directory in the list of workspace folders and then use that index to remove the directory. --- .../minimal-workspace/qlpack-generator.test.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts index f015f442c..c21c280d6 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts @@ -44,8 +44,14 @@ describe("QlPackGenerator", () => { try { dir.removeCallback(); - const end = (workspace.workspaceFolders || []).length; - workspace.updateWorkspaceFolders(end - 1, 1); + const workspaceFolders = workspace.workspaceFolders || []; + const folderIndex = workspaceFolders.findIndex( + (workspaceFolder) => workspaceFolder.name === dir.name, + ); + + if (folderIndex !== undefined) { + workspace.updateWorkspaceFolders(folderIndex, 1); + } } catch (e) { console.log( `Could not remove folder from workspace: ${getErrorMessage(e)}`, @@ -63,7 +69,6 @@ describe("QlPackGenerator", () => { expect(isFolderAlreadyInWorkspace(packFolderName)).toBe(true); expect(existsSync(qlPackYamlFilePath)).toBe(true); expect(existsSync(exampleQlFilePath)).toBe(true); - expect(packAddSpy).toHaveBeenCalledWith(packFolderPath, language); }); }); From 086df15357f1dbe2cd94fc612c694f5639a184db Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Tue, 14 Feb 2023 18:36:25 +0000 Subject: [PATCH 025/132] Use file system path On windows, the `Uri.path` will return an extra folder, as we can see in the tests: ``` ENOENT: no such file or directory, open 'D:\C:\Users\RUNNER~1\AppData\Local\Temp\tmp-4784XPDQPb5jM6IW\test-ql-pack-ruby\qlpack.yml' ``` Let's use `Uri.fsPath` instead. --- extensions/ql-vscode/src/qlpack-generator.ts | 6 +++--- .../minimal-workspace/qlpack-generator.test.ts | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index 1fa0f61e1..a88e6a410 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -64,7 +64,7 @@ export class QlPackGenerator { } private async createQlPackYaml() { - const qlPackFilePath = join(this.folderUri.path, this.qlpackFileName); + const qlPackFilePath = join(this.folderUri.fsPath, this.qlpackFileName); const qlPackYml = { name: this.qlpackName, @@ -78,7 +78,7 @@ export class QlPackGenerator { } private async createExampleQlFile() { - const exampleQlFilePath = join(this.folderUri.path, "example.ql"); + const exampleQlFilePath = join(this.folderUri.fsPath, "example.ql"); const exampleQl = ` /** @@ -98,6 +98,6 @@ select "Hello, world!" } private async createCodeqlPackLockYaml() { - await this.cliServer.packAdd(this.folderUri.path, this.queryLanguage); + await this.cliServer.packAdd(this.folderUri.fsPath, this.queryLanguage); } } diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts index c21c280d6..504b7801b 100644 --- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/qlpack-generator.test.ts @@ -2,8 +2,7 @@ import { join } from "path"; import { existsSync } from "fs"; import { QlPackGenerator, QueryLanguage } from "../../../src/qlpack-generator"; import { CodeQLCliServer } from "../../../src/cli"; -import { isFolderAlreadyInWorkspace } from "../../../src/helpers"; -import { workspace } from "vscode"; +import { Uri, workspace } from "vscode"; import { getErrorMessage } from "../../../src/pure/helpers-pure"; import * as tmp from "tmp"; @@ -22,7 +21,7 @@ describe("QlPackGenerator", () => { language = "ruby"; packFolderName = `test-ql-pack-${language}`; - packFolderPath = join(dir.name, packFolderName); + packFolderPath = Uri.file(join(dir.name, packFolderName)).fsPath; qlPackYamlFilePath = join(packFolderPath, "qlpack.yml"); exampleQlFilePath = join(packFolderPath, "example.ql"); @@ -60,15 +59,16 @@ describe("QlPackGenerator", () => { }); it("should generate a QL pack", async () => { - expect(isFolderAlreadyInWorkspace(packFolderName)).toBe(false); + expect(existsSync(packFolderPath)).toBe(false); expect(existsSync(qlPackYamlFilePath)).toBe(false); expect(existsSync(exampleQlFilePath)).toBe(false); await generator.generate(); - expect(isFolderAlreadyInWorkspace(packFolderName)).toBe(true); + expect(existsSync(packFolderPath)).toBe(true); expect(existsSync(qlPackYamlFilePath)).toBe(true); expect(existsSync(exampleQlFilePath)).toBe(true); + expect(packAddSpy).toHaveBeenCalledWith(packFolderPath, language); }); }); From ab29fb759f9fb7c21a67fa23fbfb932768775c29 Mon Sep 17 00:00:00 2001 From: Elena Tanasoiu Date: Wed, 15 Feb 2023 09:57:00 +0000 Subject: [PATCH 026/132] Copy changes and remove extra line --- extensions/ql-vscode/src/cli.ts | 6 +++--- extensions/ql-vscode/src/qlpack-generator.ts | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index ec0999645..4b8be9de9 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -1218,8 +1218,8 @@ export class CodeQLCliServer implements Disposable { } /** - * Adds a list of QL library packs with optional version ranges as dependencies of - * the current package, and then installs them. This command modifies the qlpack.yml + * Adds a core language QL library pack for the given query language as a dependency + * of the current package, and then installs them. This command modifies the qlpack.yml * file of the current package. Formatting and comments will be removed. * @param dir The directory where QL pack exists. * @param language The language of the QL pack. @@ -1230,7 +1230,7 @@ export class CodeQLCliServer implements Disposable { return this.runJsonCodeQlCliCommandWithAuthentication( ["pack", "add"], args, - "Adding and installing pack dependencies.", + `Adding and installing ${queryLanguage} pack dependency.`, ); } diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index a88e6a410..d4505cc3e 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -69,9 +69,7 @@ export class QlPackGenerator { const qlPackYml = { name: this.qlpackName, version: this.qlpackVersion, - dependencies: { - [`codeql/${this.queryLanguage}-all`]: "*", - }, + dependencies: {}, }; await writeFile(qlPackFilePath, this.header + dump(qlPackYml), "utf8"); From c0dc710b30e69656d450669f8071b7ccdccc53b8 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 15 Feb 2023 11:17:09 +0100 Subject: [PATCH 027/132] Rename markdown-generation file --- extensions/ql-vscode/src/remote-queries/export-results.ts | 2 +- ...te-queries-markdown-generation.ts => markdown-generation.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename extensions/ql-vscode/src/remote-queries/{remote-queries-markdown-generation.ts => markdown-generation.ts} (100%) diff --git a/extensions/ql-vscode/src/remote-queries/export-results.ts b/extensions/ql-vscode/src/remote-queries/export-results.ts index afeed5c5a..76400e5b7 100644 --- a/extensions/ql-vscode/src/remote-queries/export-results.ts +++ b/extensions/ql-vscode/src/remote-queries/export-results.ts @@ -18,7 +18,7 @@ import { generateVariantAnalysisMarkdown, MarkdownFile, RepositorySummary, -} from "./remote-queries-markdown-generation"; +} from "./markdown-generation"; import { pluralize } from "../pure/word"; import { VariantAnalysisManager } from "./variant-analysis-manager"; import { assertNever } from "../pure/helpers-pure"; diff --git a/extensions/ql-vscode/src/remote-queries/remote-queries-markdown-generation.ts b/extensions/ql-vscode/src/remote-queries/markdown-generation.ts similarity index 100% rename from extensions/ql-vscode/src/remote-queries/remote-queries-markdown-generation.ts rename to extensions/ql-vscode/src/remote-queries/markdown-generation.ts From 47c7e9e101e838b7f9782aa9d8c16d5cf80a72dd Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 15 Feb 2023 11:18:47 +0100 Subject: [PATCH 028/132] Move text-utils to pure --- .../ql-vscode/src/databases/config/db-config-validator.ts | 2 +- extensions/ql-vscode/src/{ => pure}/text-utils.ts | 0 .../ql-vscode/src/remote-queries/markdown-generation.ts | 8 ++++---- .../ql-vscode/src/view/results/result-table-utils.tsx | 2 +- .../src/view/variant-analysis/RawResultsTable.tsx | 2 +- extensions/ql-vscode/test/unit-tests/text-utils.test.ts | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename extensions/ql-vscode/src/{ => pure}/text-utils.ts (100%) diff --git a/extensions/ql-vscode/src/databases/config/db-config-validator.ts b/extensions/ql-vscode/src/databases/config/db-config-validator.ts index dbfe1789c..a3401c99e 100644 --- a/extensions/ql-vscode/src/databases/config/db-config-validator.ts +++ b/extensions/ql-vscode/src/databases/config/db-config-validator.ts @@ -2,7 +2,7 @@ import { readJsonSync } from "fs-extra"; import { resolve } from "path"; import Ajv from "ajv"; import { clearLocalDbConfig, DbConfig } from "./db-config"; -import { findDuplicateStrings } from "../../text-utils"; +import { findDuplicateStrings } from "../../pure/text-utils"; import { DbConfigValidationError, DbConfigValidationErrorKind, diff --git a/extensions/ql-vscode/src/text-utils.ts b/extensions/ql-vscode/src/pure/text-utils.ts similarity index 100% rename from extensions/ql-vscode/src/text-utils.ts rename to extensions/ql-vscode/src/pure/text-utils.ts diff --git a/extensions/ql-vscode/src/remote-queries/markdown-generation.ts b/extensions/ql-vscode/src/remote-queries/markdown-generation.ts index fd64fc3c7..5387ab342 100644 --- a/extensions/ql-vscode/src/remote-queries/markdown-generation.ts +++ b/extensions/ql-vscode/src/remote-queries/markdown-generation.ts @@ -2,20 +2,20 @@ import { CellValue } from "../pure/bqrs-cli-types"; import { tryGetRemoteLocation } from "../pure/bqrs-utils"; import { createRemoteFileRef } from "../pure/location-link-utils"; import { parseHighlightedLine, shouldHighlightLine } from "../pure/sarif-utils"; -import { convertNonPrintableChars } from "../text-utils"; -import { +import { convertNonPrintableChars } from "../pure/text-utils"; +import type { AnalysisAlert, AnalysisRawResults, CodeSnippet, FileLink, HighlightedRegion, } from "./shared/analysis-result"; -import { +import type { VariantAnalysis, VariantAnalysisScannedRepository, VariantAnalysisScannedRepositoryResult, } from "./shared/variant-analysis"; -import { RepositoryWithMetadata } from "./shared/repository"; +import type { RepositoryWithMetadata } from "./shared/repository"; export type MarkdownLinkType = "local" | "gist"; diff --git a/extensions/ql-vscode/src/view/results/result-table-utils.tsx b/extensions/ql-vscode/src/view/results/result-table-utils.tsx index af1037dd1..7c75d4900 100644 --- a/extensions/ql-vscode/src/view/results/result-table-utils.tsx +++ b/extensions/ql-vscode/src/view/results/result-table-utils.tsx @@ -9,7 +9,7 @@ import { } from "../../pure/interface-types"; import { assertNever } from "../../pure/helpers-pure"; import { vscode } from "../vscode-api"; -import { convertNonPrintableChars } from "../../text-utils"; +import { convertNonPrintableChars } from "../../pure/text-utils"; import { sendTelemetry } from "../common/telemetry"; export interface ResultTableProps { diff --git a/extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx b/extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx index 9078651fc..3ebbde553 100644 --- a/extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx +++ b/extensions/ql-vscode/src/view/variant-analysis/RawResultsTable.tsx @@ -9,7 +9,7 @@ import { } from "../../pure/bqrs-cli-types"; import { tryGetRemoteLocation } from "../../pure/bqrs-utils"; import TextButton from "../common/TextButton"; -import { convertNonPrintableChars } from "../../text-utils"; +import { convertNonPrintableChars } from "../../pure/text-utils"; import { sendTelemetry, useTelemetryOnChange } from "../common/telemetry"; const numOfResultsInContractedMode = 5; diff --git a/extensions/ql-vscode/test/unit-tests/text-utils.test.ts b/extensions/ql-vscode/test/unit-tests/text-utils.test.ts index bc2499d42..d3d12983f 100644 --- a/extensions/ql-vscode/test/unit-tests/text-utils.test.ts +++ b/extensions/ql-vscode/test/unit-tests/text-utils.test.ts @@ -1,4 +1,4 @@ -import { findDuplicateStrings } from "../../src/text-utils"; +import { findDuplicateStrings } from "../../src/pure/text-utils"; describe("findDuplicateStrings", () => { it("should find duplicates strings in an array of strings", () => { From f69a6c523bb5fe91367dbe3a788f13ca6470b7e0 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 15 Feb 2023 10:24:02 +0000 Subject: [PATCH 029/132] More unnecessary awaits --- extensions/ql-vscode/scripts/lint-scenarios.ts | 2 +- extensions/ql-vscode/src/cli.ts | 10 +++++----- .../ql-vscode/src/mocks/mock-gh-api-server.ts | 4 ++-- .../src/mocks/vscode-mock-gh-api-server.ts | 4 ++-- extensions/ql-vscode/src/test-adapter.ts | 14 +++++--------- .../cli-integration/legacy-query.test.ts | 4 ++-- .../vscode-tests/cli-integration/new-query.test.ts | 4 ++-- 7 files changed, 19 insertions(+), 23 deletions(-) diff --git a/extensions/ql-vscode/scripts/lint-scenarios.ts b/extensions/ql-vscode/scripts/lint-scenarios.ts index 2ce4f0b4d..46739e0e5 100644 --- a/extensions/ql-vscode/scripts/lint-scenarios.ts +++ b/extensions/ql-vscode/scripts/lint-scenarios.ts @@ -28,7 +28,7 @@ async function lintScenarios() { throw new Error(`Invalid schema: ${ajv.errorsText()}`); } - const validate = await ajv.compile(schema); + const validate = ajv.compile(schema); let invalidFiles = 0; diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index e00f50f77..751aa374f 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -295,7 +295,7 @@ export class CodeQLCliServer implements Disposable { ); } - return await spawnServer( + return spawnServer( codeQlPath, "CodeQL CLI Server", ["execute", "cli-server"], @@ -455,7 +455,7 @@ export class CodeQLCliServer implements Disposable { void logStream(child.stderr!, logger); } - for await (const event of await splitStreamAtSeparators(child.stdout!, [ + for await (const event of splitStreamAtSeparators(child.stdout!, [ "\0", ])) { yield event; @@ -487,7 +487,7 @@ export class CodeQLCliServer implements Disposable { cancellationToken?: CancellationToken, logger?: Logger, ): AsyncGenerator { - for await (const event of await this.runAsyncCodeQlCliCommandInternal( + for await (const event of this.runAsyncCodeQlCliCommandInternal( command, commandArgs, cancellationToken, @@ -750,7 +750,7 @@ export class CodeQLCliServer implements Disposable { ...testPaths, ]); - for await (const event of await this.runAsyncCodeQlCliCommand( + for await (const event of this.runAsyncCodeQlCliCommand( ["test", "run"], subcommandArgs, "Run CodeQL Tests", @@ -1543,7 +1543,7 @@ const lineEndings = ["\r\n", "\r", "\n"]; * @param logger The logger that will consume the stream output. */ async function logStream(stream: Readable, logger: Logger): Promise { - for await (const line of await splitStreamAtSeparators(stream, lineEndings)) { + for await (const line of splitStreamAtSeparators(stream, lineEndings)) { // Await the result of log here in order to ensure the logs are written in the correct order. await logger.log(line); } diff --git a/extensions/ql-vscode/src/mocks/mock-gh-api-server.ts b/extensions/ql-vscode/src/mocks/mock-gh-api-server.ts index 5bf5f0299..1d6bbd3ae 100644 --- a/extensions/ql-vscode/src/mocks/mock-gh-api-server.ts +++ b/extensions/ql-vscode/src/mocks/mock-gh-api-server.ts @@ -96,8 +96,8 @@ export class MockGitHubApiServer extends DisposableObject { } public async stopRecording(): Promise { - await this.recorder.stop(); - await this.recorder.clear(); + this.recorder.stop(); + this.recorder.clear(); } public async getScenarioNames(scenariosPath?: string): Promise { diff --git a/extensions/ql-vscode/src/mocks/vscode-mock-gh-api-server.ts b/extensions/ql-vscode/src/mocks/vscode-mock-gh-api-server.ts index c52543d48..fbe43c2c9 100644 --- a/extensions/ql-vscode/src/mocks/vscode-mock-gh-api-server.ts +++ b/extensions/ql-vscode/src/mocks/vscode-mock-gh-api-server.ts @@ -35,11 +35,11 @@ export class VSCodeMockGitHubApiServer extends DisposableObject { } public async startServer(): Promise { - await this.server.startServer(); + this.server.startServer(); } public async stopServer(): Promise { - await this.server.stopServer(); + this.server.stopServer(); await commands.executeCommand( "setContext", diff --git a/extensions/ql-vscode/src/test-adapter.ts b/extensions/ql-vscode/src/test-adapter.ts index b050a9ba2..c8b0f620c 100644 --- a/extensions/ql-vscode/src/test-adapter.ts +++ b/extensions/ql-vscode/src/test-adapter.ts @@ -356,15 +356,11 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter { tests: string[], cancellationToken: CancellationToken, ): Promise { - const workspacePaths = await getOnDiskWorkspaceFolders(); - for await (const event of await this.cliServer.runTests( - tests, - workspacePaths, - { - cancellationToken, - logger: testLogger, - }, - )) { + const workspacePaths = getOnDiskWorkspaceFolders(); + for await (const event of this.cliServer.runTests(tests, workspacePaths, { + cancellationToken, + logger: testLogger, + })) { const state = event.pass ? "passed" : event.messages?.length diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/legacy-query.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/legacy-query.test.ts index 5c0a35741..6c014b609 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/legacy-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/legacy-query.test.ts @@ -50,11 +50,11 @@ class Checkpoint { } async resolve(): Promise { - await this.res(); + this.res(); } async reject(e: Error): Promise { - await this.rej(e); + this.rej(e); } } diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts index ea448f931..bb58940cb 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/new-query.test.ts @@ -50,11 +50,11 @@ class Checkpoint { } async resolve(): Promise { - await this.res(); + this.res(); } async reject(e: Error): Promise { - await this.rej(e); + this.rej(e); } } From 5f20ef3df1da57277d1d7c0d44aebbda9aa5d8cd Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 15 Feb 2023 10:30:21 +0000 Subject: [PATCH 030/132] Fix extra alert from merging in main --- extensions/ql-vscode/src/qlpack-generator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/qlpack-generator.ts b/extensions/ql-vscode/src/qlpack-generator.ts index d4505cc3e..2d7a800f9 100644 --- a/extensions/ql-vscode/src/qlpack-generator.ts +++ b/extensions/ql-vscode/src/qlpack-generator.ts @@ -57,7 +57,7 @@ export class QlPackGenerator { const end = (workspace.workspaceFolders || []).length; - await workspace.updateWorkspaceFolders(end, 0, { + workspace.updateWorkspaceFolders(end, 0, { name: this.folderName, uri: this.folderUri, }); From 38eb9bbd899130ac8fa6a45ea9731203c30e0385 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Wed, 15 Feb 2023 11:46:09 +0100 Subject: [PATCH 031/132] Add tests for markdown generation --- .../src/remote-queries/markdown-generation.ts | 14 +- .../path-problem/analyses-results.json | 756 ++++++++++++++++++ .../path-problem/expected/_summary.md | 49 ++ .../expected/result-1-github-codeql.md | 195 +++++ .../expected/result-2-meteor-meteor.md | 144 ++++ .../problem/analyses-results.json | 198 +++++ .../problem/expected/_summary.md | 44 + .../expected/result-1-github-codeql.md | 17 + .../expected/result-2-meteor-meteor.md | 71 ++ .../raw-results/analyses-results.json | 409 ++++++++++ .../raw-results/expected/_summary.md | 41 + .../expected/result-1-github-codeql.md | 26 + .../expected/result-2-meteor-meteor.md | 6 + .../markdown-generation.test.ts | 157 ++++ 14 files changed, 2118 insertions(+), 9 deletions(-) create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/analyses-results.json create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/_summary.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-1-github-codeql.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-2-meteor-meteor.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/analyses-results.json create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/expected/_summary.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/expected/result-1-github-codeql.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/expected/result-2-meteor-meteor.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/raw-results/analyses-results.json create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/raw-results/expected/_summary.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/raw-results/expected/result-1-github-codeql.md create mode 100644 extensions/ql-vscode/test/unit-tests/data/markdown-generation/raw-results/expected/result-2-meteor-meteor.md create mode 100644 extensions/ql-vscode/test/unit-tests/remote-queries/markdown-generation.test.ts diff --git a/extensions/ql-vscode/src/remote-queries/markdown-generation.ts b/extensions/ql-vscode/src/remote-queries/markdown-generation.ts index 5387ab342..272ee6f52 100644 --- a/extensions/ql-vscode/src/remote-queries/markdown-generation.ts +++ b/extensions/ql-vscode/src/remote-queries/markdown-generation.ts @@ -39,7 +39,7 @@ export interface VariantAnalysisMarkdown { * Generates markdown files with variant analysis results. */ export async function generateVariantAnalysisMarkdown( - variantAnalysis: VariantAnalysis, + variantAnalysis: Pick, results: AsyncIterable< [VariantAnalysisScannedRepository, VariantAnalysisScannedRepositoryResult] >, @@ -91,7 +91,7 @@ export async function generateVariantAnalysisMarkdown( // Generate summary file with links to individual files const summaryFile: MarkdownFile = generateVariantAnalysisMarkdownSummary( - variantAnalysis, + variantAnalysis.query, summaries, linkType, ); @@ -103,20 +103,16 @@ export async function generateVariantAnalysisMarkdown( } export function generateVariantAnalysisMarkdownSummary( - variantAnalysis: VariantAnalysis, + query: VariantAnalysis["query"], summaries: RepositorySummary[], linkType: MarkdownLinkType, ): MarkdownFile { const lines: string[] = []; // Title - lines.push(`### Results for "${variantAnalysis.query.name}"`, ""); + lines.push(`### Results for "${query.name}"`, ""); // Expandable section containing query text - const queryCodeBlock = [ - "```ql", - ...variantAnalysis.query.text.split("\n"), - "```", - ]; + const queryCodeBlock = ["```ql", ...query.text.split("\n"), "```"]; lines.push(...buildExpandableMarkdownSection("Query", queryCodeBlock)); // Padding between sections diff --git a/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/analyses-results.json b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/analyses-results.json new file mode 100644 index 000000000..d5a339a58 --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/analyses-results.json @@ -0,0 +1,756 @@ +[ + { + "repository": { + "id": 143040428, + "fullName": "github/codeql", + "private": false, + "stargazersCount": 5703, + "updatedAt": "2023-02-15T10:11:45Z" + }, + "analysisStatus": "succeeded", + "resultCount": 4, + "artifactSizeInBytes": 3785, + "interpretedResults": [ + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This shell command depends on an uncontrolled " + }, + { + "t": "location", + "text": "absolute path", + "location": { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "highlightedRegion": { + "startLine": 4, + "startColumn": 35, + "endLine": 4, + "endColumn": 44 + } + } + }, + { "t": "text", "text": "." } + ] + }, + "shortDescription": "This shell command depends on an uncontrolled ,absolute path,.", + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 3, + "endLine": 6, + "text": "function cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n" + }, + "highlightedRegion": { + "startLine": 5, + "startColumn": 15, + "endLine": 5, + "endColumn": 18 + }, + "codeFlows": [ + { + "threadFlows": [ + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 2, + "endLine": 6, + "text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n" + }, + "highlightedRegion": { + "startLine": 4, + "startColumn": 35, + "endLine": 4, + "endColumn": 44 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 2, + "endLine": 6, + "text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n" + }, + "highlightedRegion": { + "startLine": 4, + "startColumn": 25, + "endLine": 4, + "endColumn": 53 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 2, + "endLine": 6, + "text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n" + }, + "highlightedRegion": { + "startLine": 4, + "startColumn": 13, + "endLine": 4, + "endColumn": 53 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 2, + "endLine": 6, + "text": " path = require(\"path\");\nfunction cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n" + }, + "highlightedRegion": { + "startLine": 4, + "startColumn": 7, + "endLine": 4, + "endColumn": 53 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 3, + "endLine": 6, + "text": "function cleanupTemp() {\n let cmd = \"rm -rf \" + path.join(__dirname, \"temp\");\n cp.execSync(cmd); // BAD\n}\n" + }, + "highlightedRegion": { + "startLine": 5, + "startColumn": 15, + "endLine": 5, + "endColumn": 18 + } + } + ] + } + ] + }, + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This shell command depends on an uncontrolled " + }, + { + "t": "location", + "text": "absolute path", + "location": { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "highlightedRegion": { + "startLine": 6, + "startColumn": 36, + "endLine": 6, + "endColumn": 45 + } + } + }, + { "t": "text", "text": "." } + ] + }, + "shortDescription": "This shell command depends on an uncontrolled ,absolute path,.", + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 4, + "endLine": 8, + "text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n" + }, + "highlightedRegion": { + "startLine": 6, + "startColumn": 14, + "endLine": 6, + "endColumn": 54 + }, + "codeFlows": [ + { + "threadFlows": [ + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 4, + "endLine": 8, + "text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n" + }, + "highlightedRegion": { + "startLine": 6, + "startColumn": 36, + "endLine": 6, + "endColumn": 45 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 4, + "endLine": 8, + "text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n" + }, + "highlightedRegion": { + "startLine": 6, + "startColumn": 26, + "endLine": 6, + "endColumn": 54 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 4, + "endLine": 8, + "text": "(function() {\n\tcp.execFileSync('rm', ['-rf', path.join(__dirname, \"temp\")]); // GOOD\n\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n" + }, + "highlightedRegion": { + "startLine": 6, + "startColumn": 14, + "endLine": 6, + "endColumn": 54 + } + } + ] + } + ] + }, + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This shell command depends on an uncontrolled " + }, + { + "t": "location", + "text": "absolute path", + "location": { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "highlightedRegion": { + "startLine": 8, + "startColumn": 36, + "endLine": 8, + "endColumn": 45 + } + } + }, + { "t": "text", "text": "." } + ] + }, + "shortDescription": "This shell command depends on an uncontrolled ,absolute path,.", + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 6, + "endLine": 10, + "text": "\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n" + }, + "highlightedRegion": { + "startLine": 8, + "startColumn": 14, + "endLine": 8, + "endColumn": 54 + }, + "codeFlows": [ + { + "threadFlows": [ + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 6, + "endLine": 10, + "text": "\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n" + }, + "highlightedRegion": { + "startLine": 8, + "startColumn": 36, + "endLine": 8, + "endColumn": 45 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 6, + "endLine": 10, + "text": "\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n" + }, + "highlightedRegion": { + "startLine": 8, + "startColumn": 26, + "endLine": 8, + "endColumn": 54 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 6, + "endLine": 10, + "text": "\tcp.execSync('rm -rf ' + path.join(__dirname, \"temp\")); // BAD\n\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n" + }, + "highlightedRegion": { + "startLine": 8, + "startColumn": 14, + "endLine": 8, + "endColumn": 54 + } + } + ] + } + ] + }, + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This shell command depends on an uncontrolled " + }, + { + "t": "location", + "text": "absolute path", + "location": { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "highlightedRegion": { + "startLine": 9, + "startColumn": 40, + "endLine": 9, + "endColumn": 49 + } + } + }, + { "t": "text", "text": "." } + ] + }, + "shortDescription": "This shell command depends on an uncontrolled ,absolute path,.", + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 7, + "endLine": 11, + "text": "\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n\tconst safe = \"\\\"\" + path.join(__dirname, \"temp\") + \"\\\"\";\n" + }, + "highlightedRegion": { + "startLine": 9, + "startColumn": 18, + "endLine": 9, + "endColumn": 58 + }, + "codeFlows": [ + { + "threadFlows": [ + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 7, + "endLine": 11, + "text": "\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n\tconst safe = \"\\\"\" + path.join(__dirname, \"temp\") + \"\\\"\";\n" + }, + "highlightedRegion": { + "startLine": 9, + "startColumn": 40, + "endLine": 9, + "endColumn": 49 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 7, + "endLine": 11, + "text": "\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n\tconst safe = \"\\\"\" + path.join(__dirname, \"temp\") + \"\\\"\";\n" + }, + "highlightedRegion": { + "startLine": 9, + "startColumn": 30, + "endLine": 9, + "endColumn": 58 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b", + "filePath": "javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js" + }, + "codeSnippet": { + "startLine": 7, + "endLine": 11, + "text": "\n\texeca.shell('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\texeca.shellSync('rm -rf ' + path.join(__dirname, \"temp\")); // NOT OK\n\n\tconst safe = \"\\\"\" + path.join(__dirname, \"temp\") + \"\\\"\";\n" + }, + "highlightedRegion": { + "startLine": 9, + "startColumn": 18, + "endLine": 9, + "endColumn": 58 + } + } + ] + } + ] + } + ] + }, + { + "repository": { + "id": 23578923, + "fullName": "test/no-results", + "private": false, + "stargazersCount": 7289, + "updatedAt": "2023-01-01T00:00:00Z" + }, + "analysisStatus": "succeeded", + "resultCount": 0, + "artifactSizeInBytes": 100, + "interpretedResults": [] + }, + { + "repository": { + "id": 3214406, + "fullName": "meteor/meteor", + "private": false, + "stargazersCount": 43274, + "updatedAt": "2023-02-14T21:06:55Z" + }, + "analysisStatus": "succeeded", + "resultCount": 1, + "artifactSizeInBytes": 2378, + "interpretedResults": [ + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This shell command depends on an uncontrolled " + }, + { + "t": "location", + "text": "absolute path", + "location": { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/config.js" + }, + "highlightedRegion": { + "startLine": 39, + "startColumn": 20, + "endLine": 39, + "endColumn": 61 + } + } + }, + { "t": "text", "text": "." } + ] + }, + "shortDescription": "This shell command depends on an uncontrolled ,absolute path,.", + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 28, + "endLine": 259, + "endColumn": 62 + }, + "codeFlows": [ + { + "threadFlows": [ + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/config.js" + }, + "codeSnippet": { + "startLine": 37, + "endLine": 41, + "text": "\nconst meteorLocalFolder = '.meteor';\nconst meteorPath = path.resolve(rootPath, meteorLocalFolder);\n\nmodule.exports = {\n" + }, + "highlightedRegion": { + "startLine": 39, + "startColumn": 20, + "endLine": 39, + "endColumn": 61 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/config.js" + }, + "codeSnippet": { + "startLine": 37, + "endLine": 41, + "text": "\nconst meteorLocalFolder = '.meteor';\nconst meteorPath = path.resolve(rootPath, meteorLocalFolder);\n\nmodule.exports = {\n" + }, + "highlightedRegion": { + "startLine": 39, + "startColumn": 7, + "endLine": 39, + "endColumn": 61 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/config.js" + }, + "codeSnippet": { + "startLine": 42, + "endLine": 46, + "text": " METEOR_LATEST_VERSION,\n extractPath: rootPath,\n meteorPath,\n release: process.env.INSTALL_METEOR_VERSION || METEOR_LATEST_VERSION,\n rootPath,\n" + }, + "highlightedRegion": { + "startLine": 44, + "startColumn": 3, + "endLine": 44, + "endColumn": 13 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 10, + "endLine": 14, + "text": "const os = require('os');\nconst {\n meteorPath,\n release,\n startedPath,\n" + }, + "highlightedRegion": { + "startLine": 12, + "startColumn": 3, + "endLine": 12, + "endColumn": 13 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 9, + "endLine": 25, + "text": "const tmp = require('tmp');\nconst os = require('os');\nconst {\n meteorPath,\n release,\n startedPath,\n extractPath,\n isWindows,\n rootPath,\n sudoUser,\n isSudo,\n isMac,\n METEOR_LATEST_VERSION,\n shouldSetupExecPath,\n} = require('./config.js');\nconst { uninstall } = require('./uninstall');\nconst {\n" + }, + "highlightedRegion": { + "startLine": 11, + "startColumn": 7, + "endLine": 23, + "endColumn": 27 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 42, + "endLine": 259, + "endColumn": 52 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 42, + "endLine": 259, + "endColumn": 52 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 42, + "endLine": 259, + "endColumn": 52 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 42, + "endLine": 259, + "endColumn": 52 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 42, + "endLine": 259, + "endColumn": 52 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 28, + "endLine": 259, + "endColumn": 62 + } + } + ] + }, + { + "threadFlows": [ + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/config.js" + }, + "codeSnippet": { + "startLine": 37, + "endLine": 41, + "text": "\nconst meteorLocalFolder = '.meteor';\nconst meteorPath = path.resolve(rootPath, meteorLocalFolder);\n\nmodule.exports = {\n" + }, + "highlightedRegion": { + "startLine": 39, + "startColumn": 20, + "endLine": 39, + "endColumn": 61 + } + }, + { + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec", + "filePath": "npm-packages/meteor-installer/install.js" + }, + "codeSnippet": { + "startLine": 257, + "endLine": 261, + "text": " if (isWindows()) {\n //set for the current session and beyond\n child_process.execSync(`setx path \"${meteorPath}/;%path%`);\n return;\n }\n" + }, + "highlightedRegion": { + "startLine": 259, + "startColumn": 28, + "endLine": 259, + "endColumn": 62 + } + } + ] + } + ] + } + ] + } +] diff --git a/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/_summary.md b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/_summary.md new file mode 100644 index 000000000..ad1eca260 --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/_summary.md @@ -0,0 +1,49 @@ +### Results for "Shell command built from environment values" + +
+Query + +```ql +/** + * @name Shell command built from environment values + * @description Building a shell command string with values from the enclosing + * environment may cause subtle bugs or vulnerabilities. + * @kind path-problem + * @problem.severity warning + * @security-severity 6.3 + * @precision high + * @id js/shell-command-injection-from-environment + * @tags correctness + * security + * external/cwe/cwe-078 + * external/cwe/cwe-088 + */ + +import javascript +import DataFlow::PathGraph +import semmle.javascript.security.dataflow.ShellCommandInjectionFromEnvironmentQuery + +from + Configuration cfg, DataFlow::PathNode source, DataFlow::PathNode sink, DataFlow::Node highlight, + Source sourceNode +where + sourceNode = source.getNode() and + cfg.hasFlowPath(source, sink) and + if cfg.isSinkWithHighlight(sink.getNode(), _) + then cfg.isSinkWithHighlight(sink.getNode(), highlight) + else highlight = sink.getNode() +select highlight, source, sink, "This shell command depends on an uncontrolled $@.", sourceNode, + sourceNode.getSourceType() + +``` + +
+ +
+ +### Summary + +| Repository | Results | +| --- | --- | +| github/codeql | [4 result(s)](#file-result-1-github-codeql-md) | +| meteor/meteor | [1 result(s)](#file-result-2-meteor-meteor-md) | diff --git a/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-1-github-codeql.md b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-1-github-codeql.md new file mode 100644 index 000000000..cd8eeac28 --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-1-github-codeql.md @@ -0,0 +1,195 @@ +### github/codeql + +[javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L5-L5) + +
function cleanupTemp() {
+  let cmd = "rm -rf " + path.join(__dirname, "temp");
+  cp.execSync(cmd); // BAD
+}
+
+ +*This shell command depends on an uncontrolled [absolute path](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L4-L4).* + +#### Paths + +
+Path with 5 steps + +1. [javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L4-L4) + 
  path = require("path");
+   function cleanupTemp() {
+     let cmd = "rm -rf " + path.join(__dirname, "temp");
+     cp.execSync(cmd); // BAD
+   }
+
+ +2. [javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L4-L4) + 
  path = require("path");
+   function cleanupTemp() {
+     let cmd = "rm -rf " + path.join(__dirname, "temp");
+     cp.execSync(cmd); // BAD
+   }
+
+ +3. [javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L4-L4) + 
  path = require("path");
+   function cleanupTemp() {
+     let cmd = "rm -rf " + path.join(__dirname, "temp");
+     cp.execSync(cmd); // BAD
+   }
+
+ +4. [javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L4-L4) + 
  path = require("path");
+   function cleanupTemp() {
+     let cmd = "rm -rf " + path.join(__dirname, "temp");
+     cp.execSync(cmd); // BAD
+   }
+
+ +5. [javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/src/Security/CWE-078/examples/shell-command-injection-from-environment.js#L5-L5) + 
function cleanupTemp() {
+     let cmd = "rm -rf " + path.join(__dirname, "temp");
+     cp.execSync(cmd); // BAD
+   }
+
+ + +
+ +---------------------------------------- + +[javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L6-L6) + +
(function() {
+	cp.execFileSync('rm',  ['-rf', path.join(__dirname, "temp")]); // GOOD
+	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+
+	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+
+ +*This shell command depends on an uncontrolled [absolute path](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L6-L6).* + +#### Paths + +
+Path with 3 steps + +1. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L6-L6) + 
(function() {
+   	cp.execFileSync('rm',  ['-rf', path.join(__dirname, "temp")]); // GOOD
+   	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+   
+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+
+ +2. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L6-L6) + 
(function() {
+   	cp.execFileSync('rm',  ['-rf', path.join(__dirname, "temp")]); // GOOD
+   	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+   
+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+
+ +3. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L6-L6) + 
(function() {
+   	cp.execFileSync('rm',  ['-rf', path.join(__dirname, "temp")]); // GOOD
+   	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+   
+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+
+ + +
+ +---------------------------------------- + +[javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L8-L8) + +
	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+
+	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+
+
+ +*This shell command depends on an uncontrolled [absolute path](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L8-L8).* + +#### Paths + +
+Path with 3 steps + +1. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L8-L8) + 
	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+   
+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   
+
+ +2. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L8-L8) + 
	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+   
+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   
+
+ +3. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L8-L8) + 
	cp.execSync('rm -rf ' + path.join(__dirname, "temp")); // BAD
+   
+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   
+
+ + +
+ +---------------------------------------- + +[javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L9-L9) + +

+	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+
+	const safe = "\"" + path.join(__dirname, "temp") + "\"";
+
+ +*This shell command depends on an uncontrolled [absolute path](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L9-L9).* + +#### Paths + +
+Path with 3 steps + +1. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L9-L9) + 

+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   
+   	const safe = "\"" + path.join(__dirname, "temp") + "\"";
+
+ +2. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L9-L9) + 

+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   
+   	const safe = "\"" + path.join(__dirname, "temp") + "\"";
+
+ +3. [javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js](https://github.com/github/codeql/blob/48015e5a2e6202131f2d1062cc066dc33ed69a9b/javascript/ql/test/query-tests/Security/CWE-078/tst_shell-command-injection-from-environment.js#L9-L9) + 

+   	execa.shell('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   	execa.shellSync('rm -rf ' + path.join(__dirname, "temp")); // NOT OK
+   
+   	const safe = "\"" + path.join(__dirname, "temp") + "\"";
+
+ + +
+ +---------------------------------------- diff --git a/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-2-meteor-meteor.md b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-2-meteor-meteor.md new file mode 100644 index 000000000..6a216f144 --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/path-problem/expected/result-2-meteor-meteor.md @@ -0,0 +1,144 @@ +### meteor/meteor + +[npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + +
  if (isWindows()) {
+    //set for the current session and beyond
+    child_process.execSync(`setx path "${meteorPath}/;%path%`);
+    return;
+  }
+
+ +*This shell command depends on an uncontrolled [absolute path](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/config.js#L39-L39).* + +#### Paths + +
+Path with 11 steps + +1. [npm-packages/meteor-installer/config.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/config.js#L39-L39) + 

+   const meteorLocalFolder = '.meteor';
+   const meteorPath = path.resolve(rootPath, meteorLocalFolder);
+   
+   module.exports = {
+
+ +2. [npm-packages/meteor-installer/config.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/config.js#L39-L39) + 

+   const meteorLocalFolder = '.meteor';
+   const meteorPath = path.resolve(rootPath, meteorLocalFolder);
+   
+   module.exports = {
+
+ +3. [npm-packages/meteor-installer/config.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/config.js#L44-L44) + 
  METEOR_LATEST_VERSION,
+     extractPath: rootPath,
+     meteorPath,
+     release: process.env.INSTALL_METEOR_VERSION || METEOR_LATEST_VERSION,
+     rootPath,
+
+ +4. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L12-L12) + 
const os = require('os');
+   const {
+     meteorPath,
+     release,
+     startedPath,
+
+ +5. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L11-L23) + 
const tmp = require('tmp');
+   const os = require('os');
+   const {
+     meteorPath,
+     release,
+     startedPath,
+     extractPath,
+     isWindows,
+     rootPath,
+     sudoUser,
+     isSudo,
+     isMac,
+     METEOR_LATEST_VERSION,
+     shouldSetupExecPath,
+   } = require('./config.js');
+   const { uninstall } = require('./uninstall');
+   const {
+
+ +6. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+       //set for the current session and beyond
+       child_process.execSync(`setx path "${meteorPath}/;%path%`);
+       return;
+     }
+
+ +7. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+       //set for the current session and beyond
+       child_process.execSync(`setx path "${meteorPath}/;%path%`);
+       return;
+     }
+
+ +8. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+       //set for the current session and beyond
+       child_process.execSync(`setx path "${meteorPath}/;%path%`);
+       return;
+     }
+
+ +9. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+       //set for the current session and beyond
+       child_process.execSync(`setx path "${meteorPath}/;%path%`);
+       return;
+     }
+
+ +10. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+        //set for the current session and beyond
+        child_process.execSync(`setx path "${meteorPath}/;%path%`);
+        return;
+      }
+
+ +11. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+        //set for the current session and beyond
+        child_process.execSync(`setx path "${meteorPath}/;%path%`);
+        return;
+      }
+
+ + +
+ +
+Path with 2 steps + +1. [npm-packages/meteor-installer/config.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/config.js#L39-L39) + 

+   const meteorLocalFolder = '.meteor';
+   const meteorPath = path.resolve(rootPath, meteorLocalFolder);
+   
+   module.exports = {
+
+ +2. [npm-packages/meteor-installer/install.js](https://github.com/meteor/meteor/blob/73b538fe201cbfe89dd0c709689023f9b3eab1ec/npm-packages/meteor-installer/install.js#L259-L259) + 
  if (isWindows()) {
+       //set for the current session and beyond
+       child_process.execSync(`setx path "${meteorPath}/;%path%`);
+       return;
+     }
+
+ + +
+ +---------------------------------------- diff --git a/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/analyses-results.json b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/analyses-results.json new file mode 100644 index 000000000..b8db420bf --- /dev/null +++ b/extensions/ql-vscode/test/unit-tests/data/markdown-generation/interpreted-results/problem/analyses-results.json @@ -0,0 +1,198 @@ +[ + { + "repository": { + "id": 143040428, + "fullName": "github/codeql", + "private": false, + "stargazersCount": 5703, + "updatedAt": "2023-02-15T10:11:45Z" + }, + "analysisStatus": "succeeded", + "resultCount": 1, + "artifactSizeInBytes": 1038, + "interpretedResults": [ + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'aa'." + } + ] + }, + "shortDescription": "This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'aa'.", + "fileLink": { + "fileLinkPrefix": "https://github.com/github/codeql/blob/d094bbc06d063d0da8d0303676943c345e61de53", + "filePath": "javascript/extractor/tests/regexp/input/multipart.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 15, + "endLine": 22, + "text": "\nvar bad95 = new RegExp(\n \"(a\" + \n \"|\" + \n \"aa)*\" + \n \"b$\"\n);\n\n" + }, + "highlightedRegion": { + "startLine": 17, + "startColumn": 6, + "endLine": 20, + "endColumn": 6 + }, + "codeFlows": [] + } + ] + }, + { + "repository": { + "id": 3214406, + "fullName": "meteor/meteor", + "private": false, + "stargazersCount": 43274, + "updatedAt": "2023-02-14T21:06:55Z" + }, + "analysisStatus": "succeeded", + "resultCount": 5, + "artifactSizeInBytes": 3478, + "interpretedResults": [ + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '----'." + } + ] + }, + "shortDescription": "This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '----'.", + "fileLink": { + "fileLinkPrefix": "https://github.com/meteor/meteor/blob/53f3c4442d3542d3d2a012a854472a0d1bef9d12", + "filePath": "packages/deprecated/markdown/showdown.js" + }, + "severity": "Warning", + "codeSnippet": { + "startLine": 413, + "endLine": 417, + "text": "\t\t/g,hashElement);\n\t*/\n\ttext = text.replace(/(\\n\\n[ ]{0,3}[ \\t]*(?=\\n{2,}))/g,hashElement);\n\n\t// PHP and ASP-style processor instructions ( and <%...%>)\n" + }, + "highlightedRegion": { + "startLine": 415, + "startColumn": 41, + "endLine": 415, + "endColumn": 48 + }, + "codeFlows": [] + }, + { + "message": { + "tokens": [ + { + "t": "text", + "text": "This part of the regular expression may cause exponential backtracking on strings starting with '