mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

First pass on converting cli-integration tests to Jest

Browse Source 
This is a first pass on converting the cli-integration tests to Jest. It
uses a custom Jest runner (based on the jest-runner-vscode) to install
required extensions. It will also necessary to move some code for
installating the CLI to ensure it was possible to install the CLI from
outside VSCode.

Most of the conversion has been done by `npx jest-codemods`, but all
migration of Sinon has been done manually.
This commit is contained in:
Koen Vlaswinkel
2022-11-23 16:37:40 +01:00
parent c31635fe90
commit e5967c3851
29 changed files with 1048 additions and 917 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/jest.config.js
View File

@@ -8,6 +8,7 @@ module.exports = {
projects: [
"<rootDir>/src/view",
"<rootDir>/test",
"<rootDir>/out/vscode-tests/cli-integration",
"<rootDir>/out/vscode-tests/no-workspace",
],
};

2
extensions/ql-vscode/package.json
View File

@@ -1275,7 +1275,7 @@
"integration": "npm-run-all integration:*",
"integration:no-workspace": "jest --projects out/vscode-tests/no-workspace",
"integration:minimal-workspace": "node ./out/vscode-tests/run-integration-tests.js minimal-workspace",
"cli-integration": "node ./out/vscode-tests/run-integration-tests.js cli-integration",
"cli-integration": "jest --config=out/vscode-tests/cli-integration/jest.config.js out/vscode-tests/cli-integration",
"update-vscode": "node ./node_modules/vscode/bin/install",
"format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
"lint": "eslint . --ext .ts,.tsx --max-warnings=0",

60
extensions/ql-vscode/src/distribution.ts
View File

@@ -3,7 +3,6 @@ import * as fs from "fs-extra";
import * as os from "os";
import * as path from "path";
import * as semver from "semver";
import * as unzipper from "unzipper";
import * as url from "url";
import { ExtensionContext, Event } from "vscode";
import { DistributionConfig } from "./config";
@@ -16,6 +15,12 @@ import {
import { logger } from "./logging";
import { getCodeQlCliVersion } from "./cli-version";
import { ProgressCallback, reportStreamProgress } from "./commandRunner";
import {
codeQlLauncherName,
deprecatedCodeQlLauncherName,
extractZipArchive,
getRequiredAssetName,
} from "./pure/distribution";
/**
* distribution.ts
@@ -55,22 +60,6 @@ export interface DistributionProvider {
}
export class DistributionManager implements DistributionProvider {
/**
* Get the name of the codeql cli installation we prefer to install, based on our current platform.
*/
public static getRequiredAssetName(): string {
switch (os.platform()) {
case "linux":
return "codeql-linux64.zip";
case "darwin":
return "codeql-osx64.zip";
case "win32":
return "codeql-win64.zip";
default:
return "codeql.zip";
}
}
constructor(
public readonly config: DistributionConfig,
private readonly versionRange: semver.Range,
@@ -361,7 +350,7 @@ class ExtensionSpecificDistributionManager {
}
// Filter assets to the unique one that we require.
const requiredAssetName = DistributionManager.getRequiredAssetName();
const requiredAssetName = getRequiredAssetName();
const assets = release.assets.filter(
(asset) => asset.name === requiredAssetName,
);
@@ -431,7 +420,7 @@ class ExtensionSpecificDistributionManager {
}
private async getLatestRelease(): Promise<Release> {
const requiredAssetName = DistributionManager.getRequiredAssetName();
const requiredAssetName = getRequiredAssetName();
void logger.log(
`Searching for latest release including ${requiredAssetName}.`,
);
@@ -683,39 +672,6 @@ export class ReleasesApiConsumer {
private static readonly _maxRedirects = 20;
}
export async function extractZipArchive(
archivePath: string,
outPath: string,
): Promise<void> {
const archive = await unzipper.Open.file(archivePath);
await archive.extract({
concurrency: 4,
path: outPath,
});
// Set file permissions for extracted files
await Promise.all(
archive.files.map(async (file) => {
// Only change file permissions if within outPath (path.join normalises the path)
const extractedPath = path.join(outPath, file.path);
if (
extractedPath.indexOf(outPath) !== 0 ||
!(await fs.pathExists(extractedPath))
) {
return Promise.resolve();
}
return fs.chmod(extractedPath, file.externalFileAttributes >>> 16);
}),
);
}
export function codeQlLauncherName(): string {
return os.platform() === "win32" ? "codeql.exe" : "codeql";
}
function deprecatedCodeQlLauncherName(): string | undefined {
return os.platform() === "win32" ? "codeql.cmd" : undefined;
}
function isRedirectStatusCode(statusCode: number): boolean {
return (
statusCode === 301 ||

53
extensions/ql-vscode/src/pure/distribution.ts Normal file
View File

@@ -0,0 +1,53 @@
import * as os from "os";
import * as unzipper from "unzipper";
import * as path from "path";
import * as fs from "fs-extra";
/**
* Get the name of the codeql cli installation we prefer to install, based on our current platform.
*/
export function getRequiredAssetName(): string {
switch (os.platform()) {
case "linux":
return "codeql-linux64.zip";
case "darwin":
return "codeql-osx64.zip";
case "win32":
return "codeql-win64.zip";
default:
return "codeql.zip";
}
}
export async function extractZipArchive(
archivePath: string,
outPath: string,
): Promise<void> {
const archive = await unzipper.Open.file(archivePath);
await archive.extract({
concurrency: 4,
path: outPath,
});
// Set file permissions for extracted files
await Promise.all(
archive.files.map(async (file) => {
// Only change file permissions if within outPath (path.join normalises the path)
const extractedPath = path.join(outPath, file.path);
if (
extractedPath.indexOf(outPath) !== 0 ||
!(await fs.pathExists(extractedPath))
) {
return Promise.resolve();
}
return fs.chmod(extractedPath, file.externalFileAttributes >>> 16);
}),
);
}
export function codeQlLauncherName(): string {
return os.platform() === "win32" ? "codeql.exe" : "codeql";
}
export function deprecatedCodeQlLauncherName(): string | undefined {
return os.platform() === "win32" ? "codeql.cmd" : undefined;
}

55
extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts
View File

@@ -1,7 +1,4 @@
import * as sinon from "sinon";
import * as path from "path";
import { fail } from "assert";
import { expect } from "chai";
import { extensions, CancellationToken, Uri, window } from "vscode";
import { CodeQLExtensionInterface } from "../../extension";
@@ -12,33 +9,29 @@ import {
importArchiveDatabase,
promptImportInternetDatabase,
} from "../../databaseFetcher";
import { ProgressCallback } from "../../commandRunner";
import { cleanDatabases, dbLoc, DB_URL, storagePath } from "./global.helper";
jest.setTimeout(60_000);
/**
* Run various integration tests for databases
*/
describe("Databases", function () {
this.timeout(60000);
describe("Databases", () => {
const LGTM_URL =
"https://lgtm.com/projects/g/aeisenberg/angular-bind-notifier/";
let databaseManager: DatabaseManager;
let sandbox: sinon.SinonSandbox;
let inputBoxStub: sinon.SinonStub;
const inputBoxStub = jest.spyOn(window, "showInputBox");
let cli: CodeQLCliServer;
let progressCallback: ProgressCallback;
const progressCallback = jest.fn();
jest.spyOn(window, "showErrorMessage").mockResolvedValue(undefined);
jest.spyOn(window, "showInformationMessage").mockResolvedValue(undefined);
beforeEach(async () => {
try {
sandbox = sinon.createSandbox();
// the uri.fsPath function on windows returns a lowercase drive letter
// so, force the storage path string to be lowercase, too.
progressCallback = sandbox.spy();
inputBoxStub = sandbox.stub(window, "showInputBox");
sandbox.stub(window, "showErrorMessage");
sandbox.stub(window, "showInformationMessage");
inputBoxStub.mockReset().mockResolvedValue(undefined);
progressCallback.mockReset();
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -61,7 +54,6 @@ describe("Databases", function () {
afterEach(async () => {
try {
sandbox.restore();
await cleanDatabases(databaseManager);
} catch (e) {
fail(e as Error);
@@ -69,7 +61,6 @@ describe("Databases", function () {
});
it("should add a database from a folder", async () => {
const progressCallback = sandbox.spy() as ProgressCallback;
const uri = Uri.file(dbLoc);
let dbItem = await importArchiveDatabase(
uri.toString(true),
@@ -79,16 +70,16 @@ describe("Databases", function () {
{} as CancellationToken,
cli,
);
expect(dbItem).to.be.eq(databaseManager.currentDatabaseItem);
expect(dbItem).to.be.eq(databaseManager.databaseItems[0]);
expect(dbItem).not.to.be.undefined;
expect(dbItem).toBe(databaseManager.currentDatabaseItem);
expect(dbItem).toBe(databaseManager.databaseItems[0]);
expect(dbItem).toBeDefined();
dbItem = dbItem!;
expect(dbItem.name).to.eq("db");
expect(dbItem.databaseUri.fsPath).to.eq(path.join(storagePath, "db", "db"));
expect(dbItem.name).toBe("db");
expect(dbItem.databaseUri.fsPath).toBe(path.join(storagePath, "db", "db"));
});
it("should add a database from lgtm with only one language", async () => {
inputBoxStub.resolves(LGTM_URL);
inputBoxStub.mockResolvedValue(LGTM_URL);
let dbItem = await promptImportLgtmDatabase(
databaseManager,
storagePath,
@@ -96,10 +87,10 @@ describe("Databases", function () {
{} as CancellationToken,
cli,
);
expect(dbItem).not.to.be.undefined;
expect(dbItem).toBeDefined();
dbItem = dbItem!;
expect(dbItem.name).to.eq("aeisenberg_angular-bind-notifier_106179a");
expect(dbItem.databaseUri.fsPath).to.eq(
expect(dbItem.name).toBe("aeisenberg_angular-bind-notifier_106179a");
expect(dbItem.databaseUri.fsPath).toBe(
path.join(
storagePath,
"javascript",
@@ -109,7 +100,7 @@ describe("Databases", function () {
});
it("should add a database from a url", async () => {
inputBoxStub.resolves(DB_URL);
inputBoxStub.mockResolvedValue(DB_URL);
let dbItem = await promptImportInternetDatabase(
databaseManager,
@@ -118,10 +109,10 @@ describe("Databases", function () {
{} as CancellationToken,
cli,
);
expect(dbItem).not.to.be.undefined;
expect(dbItem).toBeDefined();
dbItem = dbItem!;
expect(dbItem.name).to.eq("db");
expect(dbItem.databaseUri.fsPath).to.eq(
expect(dbItem.name).toBe("db");
expect(dbItem.databaseUri.fsPath).toBe(
path.join(storagePath, "simple-db", "db"),
);
});

103
extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts
View File

@@ -1,15 +1,8 @@
import * as path from "path";
import * as tmp from "tmp";
import * as yaml from "js-yaml";
import * as fs from "fs-extra";
import fetch from "node-fetch";
import { fail } from "assert";
import { commands, extensions, workspace } from "vscode";
import { CodeQLExtensionInterface } from "../../extension";
import { commands } from "vscode";
import { DatabaseManager } from "../../databases";
import { getTestSetting } from "../test-config";
import { CUSTOM_CODEQL_PATH_SETTING } from "../../config";
import { CodeQLCliServer } from "../../cli";
import { removeWorkspaceRefs } from "../../remote-queries/run-remote-query";
@@ -26,98 +19,8 @@ export const dbLoc = path.join(
);
export let storagePath: string;
export default function (mocha: Mocha) {
// create an extension storage location
let removeStorage: tmp.DirResult["removeCallback"] | undefined;
// ensure the test database is downloaded
(mocha.options as any).globalSetup.push(async () => {
fs.mkdirpSync(path.dirname(dbLoc));
if (!fs.existsSync(dbLoc)) {
try {
await new Promise((resolve, reject) => {
return fetch(DB_URL).then((response) => {
const dest = fs.createWriteStream(dbLoc);
response.body.pipe(dest);
response.body.on("error", reject);
dest.on("error", reject);
dest.on("close", () => {
resolve(dbLoc);
});
});
});
} catch (e) {
fail("Failed to download test database: " + e);
}
}
});
// Set the CLI version here before activation to ensure we don't accidentally try to download a cli
(mocha.options as any).globalSetup.push(async () => {
await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setInitialTestValue(
process.env.CLI_PATH,
);
});
// Create the temp directory to be used as extension local storage.
(mocha.options as any).globalSetup.push(() => {
const dir = tmp.dirSync();
storagePath = fs.realpathSync(dir.name);
if (storagePath.substring(0, 2).match(/[A-Z]:/)) {
storagePath =
storagePath.substring(0, 1).toLocaleLowerCase() +
storagePath.substring(1);
}
removeStorage = dir.removeCallback;
});
// ensure extension is cleaned up.
(mocha.options as any).globalTeardown.push(async () => {
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
"GitHub.vscode-codeql",
)!
.activate();
// This shuts down the extension and can only be run after all tests have completed.
// If this is not called, then the test process will hang.
if ("dispose" in extension) {
try {
extension.dispose();
} catch (e) {
console.warn("Failed to dispose extension", e);
}
}
});
// ensure temp directory is cleaned up.
(mocha.options as any).globalTeardown.push(() => {
try {
removeStorage?.();
} catch (e) {
// we are exiting anyway so don't worry about it.
// most likely the directory this is a test on Windows and some files are locked.
console.warn(`Failed to remove storage directory '${storagePath}': ${e}`);
}
});
// check that the codeql folder is found in the workspace
(mocha.options as any).globalSetup.push(async () => {
const folders = workspace.workspaceFolders;
if (!folders) {
fail(
'\n\n\nNo workspace folders found.\nYou will need a local copy of the codeql repo.\nMake sure you specify the path to it in launch.json.\nIt should be something along the lines of "${workspaceRoot}/../codeql" depending on where you have your local copy of the codeql repo.\n\n\n',
);
} else {
const codeqlFolder = folders.find((folder) => folder.name === "codeql");
if (!codeqlFolder) {
fail(
'\n\n\nNo workspace folders found.\nYou will need a local copy of the codeql repo.\nMake sure you specify the path to it in launch.json.\nIt should be something along the lines of "${workspaceRoot}/../codeql" depending on where you have your local copy of the codeql repo.\n\n\n',
);
}
}
});
export function setStoragePath(path: string) {
storagePath = path;
}
export async function cleanDatabases(databaseManager: DatabaseManager) {

17
extensions/ql-vscode/src/vscode-tests/cli-integration/helpers.test.ts
View File

@@ -4,17 +4,16 @@ import { extensions } from "vscode";
import { CodeQLCliServer } from "../../cli";
import { CodeQLExtensionInterface } from "../../extension";
import { tryGetQueryMetadata } from "../../helpers";
import { expect } from "chai";
describe("helpers (with CLI)", function () {
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
describe("helpers (with CLI)", () => {
const baseDir = path.join(
__dirname,
"../../../src/vscode-tests/cli-integration",
);
// up to 3 minutes per test
this.timeout(3 * 60 * 1000);
let cli: CodeQLCliServer;
beforeEach(async () => {
@@ -39,9 +38,9 @@ describe("helpers (with CLI)", function () {
path.join(baseDir, "data", "simple-javascript-query.ql"),
);
expect(metadata!.name).to.equal("This is the name");
expect(metadata!.kind).to.equal("problem");
expect(metadata!.id).to.equal("javascript/example/test-query");
expect(metadata!.name).toBe("This is the name");
expect(metadata!.kind).toBe("problem");
expect(metadata!.id).toBe("javascript/example/test-query");
});
it("should handle query with no metadata", async () => {
@@ -51,6 +50,6 @@ describe("helpers (with CLI)", function () {
path.join(baseDir, "data", "simple-query.ql"),
);
expect(noMetadata).to.deep.equal({});
expect(noMetadata).toEqual({});
});
});

13
extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts
View File

@@ -1,13 +0,0 @@
import "source-map-support/register";
import { runTestsInDirectory } from "../index-template";
import "mocha";
import * as sinonChai from "sinon-chai";
import * as chai from "chai";
import "chai/register-should";
import * as chaiAsPromised from "chai-as-promised";
chai.use(chaiAsPromised);
chai.use(sinonChai);
export function run(): Promise<void> {
return runTestsInDirectory(__dirname, true);
}

74
extensions/ql-vscode/src/vscode-tests/cli-integration/jest-runner-cli-integration.ts Normal file
View File

@@ -0,0 +1,74 @@
import * as cp from "child_process";
import * as path from "path";
import type * as JestRunner from "jest-runner";
import VSCodeTestRunner, { RunnerOptions } from "jest-runner-vscode";
import { cosmiconfig } from "cosmiconfig";
import {
downloadAndUnzipVSCode,
resolveCliArgsFromVSCodeExecutablePath,
} from "@vscode/test-electron";
import { ensureCli } from "../ensureCli";
export default class JestRunnerCliIntegration extends VSCodeTestRunner {
async runTests(
tests: JestRunner.Test[],
watcher: JestRunner.TestWatcher,
onStart: JestRunner.OnTestStart,
onResult: JestRunner.OnTestSuccess,
onFailure: JestRunner.OnTestFailure,
): Promise<void> {
// The CLI integration tests require certain extensions to be installed, which needs to happen before the tests are
// actually run. The below code will resolve the path to the VSCode executable, and then use that to install the
// required extensions.
const installedOnVsCodeVersions =
new Set<`${RunnerOptions["version"]}-${RunnerOptions["platform"]}`>();
for (const test of tests) {
const testDir = path.dirname(test.path);
const options: RunnerOptions =
((await cosmiconfig("jest-runner-vscode").search(testDir))
?.config as RunnerOptions) ?? {};
const { version, platform } = options;
const versionKey = `${version}-${platform}`;
if (installedOnVsCodeVersions.has(versionKey)) {
continue;
}
const vscodeExecutablePath = await downloadAndUnzipVSCode(
version,
platform,
);
console.log(`Installing required extensions for ${vscodeExecutablePath}`);
const [cli, ...args] =
resolveCliArgsFromVSCodeExecutablePath(vscodeExecutablePath);
cp.spawnSync(
cli,
[
...args,
"--install-extension",
"hbenl.vscode-test-explorer",
"--install-extension",
"ms-vscode.test-adapter-converter",
],
{
encoding: "utf-8",
stdio: "inherit",
},
);
installedOnVsCodeVersions.add(versionKey);
}
await ensureCli(true);
return super.runTests(tests, watcher, onStart, onResult, onFailure);
}
}

30
extensions/ql-vscode/src/vscode-tests/cli-integration/jest-runner-vscode.config.ts Normal file
View File

@@ -0,0 +1,30 @@
import * as path from "path";
import { RunnerOptions } from "jest-runner-vscode";
import baseConfig, { rootDir } from "../jest-runner-vscode.config.base";
const config: RunnerOptions = {
...baseConfig,
launchArgs: [
...(baseConfig.launchArgs ?? []),
// explicitly disable extensions that are known to interfere with the CLI integration tests
"--disable-extension",
"eamodio.gitlens",
"--disable-extension",
"github.codespaces",
"--disable-extension",
"github.copilot",
path.resolve(rootDir, "test/data"),
// CLI integration tests requires a multi-root workspace so that the data and the QL sources are accessible.
...(process.env.TEST_CODEQL_PATH ? [process.env.TEST_CODEQL_PATH] : []),
],
extensionTestsEnv: {
...baseConfig.extensionTestsEnv,
INTEGRATION_TEST_MODE: "true",
},
};
// We are purposefully not using export default here since that would result in an ESModule, which doesn't seem to be
// supported properly by jest-runner-vscode (cosmiconfig doesn't really seem to support it).
module.exports = config;

11
extensions/ql-vscode/src/vscode-tests/cli-integration/jest.config.ts Normal file
View File

@@ -0,0 +1,11 @@
import type { Config } from "jest";
import baseConfig from "../jest.config.base";
const config: Config = {
...baseConfig,
runner: "<rootDir>/jest-runner-cli-integration.js",
setupFilesAfterEnv: ["<rootDir>/jest.setup.js"],
};
export default config;

101
extensions/ql-vscode/src/vscode-tests/cli-integration/jest.setup.ts Normal file
View File

@@ -0,0 +1,101 @@
import * as fs from "fs-extra";
import * as path from "path";
import fetch from "node-fetch";
import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper";
import * as tmp from "tmp";
import { getTestSetting } from "../test-config";
import { CUSTOM_CODEQL_PATH_SETTING } from "../../config";
import { extensions, workspace } from "vscode";
import { CodeQLExtensionInterface } from "../../extension";
import baseJestSetup from "../jest.setup";
export default baseJestSetup;
// create an extension storage location
let removeStorage: tmp.DirResult["removeCallback"] | undefined;
beforeAll(async () => {
// Set the CLI version here before activation to ensure we don't accidentally try to download a cli
await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setInitialTestValue(
process.env.CLI_PATH,
);
await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setup();
// ensure the test database is downloaded
fs.mkdirpSync(path.dirname(dbLoc));
if (!fs.existsSync(dbLoc)) {
console.log(`Downloading test database to ${dbLoc}`);
try {
await new Promise((resolve, reject) => {
return fetch(DB_URL).then((response) => {
const dest = fs.createWriteStream(dbLoc);
response.body.pipe(dest);
response.body.on("error", reject);
dest.on("error", reject);
dest.on("close", () => {
resolve(dbLoc);
});
});
});
} catch (e) {
fail("Failed to download test database: " + e);
}
}
// Create the temp directory to be used as extension local storage.
const dir = tmp.dirSync();
let storagePath = fs.realpathSync(dir.name);
if (storagePath.substring(0, 2).match(/[A-Z]:/)) {
storagePath =
storagePath.substring(0, 1).toLocaleLowerCase() +
storagePath.substring(1);
}
setStoragePath(storagePath);
removeStorage = dir.removeCallback;
// check that the codeql folder is found in the workspace
const folders = workspace.workspaceFolders;
if (!folders) {
fail(
'\n\n\nNo workspace folders found.\nYou will need a local copy of the codeql repo.\nMake sure you specify the path to it in launch.json.\nIt should be something along the lines of "${workspaceRoot}/../codeql" depending on where you have your local copy of the codeql repo.\n\n\n',
);
} else {
const codeqlFolder = folders.find((folder) => folder.name === "codeql");
if (!codeqlFolder) {
fail(
'\n\n\nNo workspace folders found.\nYou will need a local copy of the codeql repo.\nMake sure you specify the path to it in launch.json.\nIt should be something along the lines of "${workspaceRoot}/../codeql" depending on where you have your local copy of the codeql repo.\n\n\n',
);
}
}
});
// ensure extension is cleaned up.
afterAll(async () => {
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
"GitHub.vscode-codeql",
)!
.activate();
// This shuts down the extension and can only be run after all tests have completed.
// If this is not called, then the test process will hang.
if ("dispose" in extension) {
try {
extension.dispose();
} catch (e) {
console.warn("Failed to dispose extension", e);
}
}
// ensure temp directory is cleaned up.
try {
removeStorage?.();
} catch (e) {
// we are exiting anyway so don't worry about it.
// most likely the directory this is a test on Windows and some files are locked.
console.warn(`Failed to remove storage directory '${storagePath}': ${e}`);
}
});

34
extensions/ql-vscode/src/vscode-tests/cli-integration/legacy-query.test.ts
View File

@@ -1,4 +1,3 @@
import { expect } from "chai";
import * as fs from "fs-extra";
import * as path from "path";
import * as tmp from "tmp";
@@ -11,7 +10,7 @@ import { CellValue } from "../../pure/bqrs-cli-types";
import { extensions } from "vscode";
import { CodeQLExtensionInterface } from "../../extension";
import { fail } from "assert";
import { skipIfNoCodeQL } from "../ensureCli";
import { describeWithCodeQL } from "../cli";
import { QueryServerClient } from "../../legacy-query-server/queryserver-client";
import { logger, ProgressReporter } from "../../logging";
@@ -100,15 +99,9 @@ const db: messages.Dataset = {
workingSet: "default",
};
describe("using the legacy query server", function () {
before(function () {
skipIfNoCodeQL(this);
});
// Note this does not work with arrow functions as the test case bodies:
// ensure they are all written with standard anonymous functions.
this.timeout(20000);
jest.setTimeout(20_000);
describeWithCodeQL()("using the legacy query server", () => {
const nullProgressReporter: ProgressReporter = {
report: () => {
/** ignore */
@@ -118,7 +111,7 @@ describe("using the legacy query server", function () {
let qs: qsClient.QueryServerClient;
let cliServer: cli.CodeQLCliServer;
before(async () => {
beforeAll(async () => {
try {
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -178,8 +171,8 @@ describe("using the legacy query server", function () {
}
});
it(`should be able to compile query ${queryName}`, async function () {
expect(fs.existsSync(queryTestCase.queryPath)).to.be.true;
it(`should be able to compile query ${queryName}`, async () => {
expect(fs.existsSync(queryTestCase.queryPath)).toBe(true);
try {
const qlProgram: messages.QlProgram = {
libraryPath: [],
@@ -210,14 +203,14 @@ describe("using the legacy query server", function () {
/**/
},
);
expect(result.messages!.length).to.equal(0);
expect(result.messages!.length).toBe(0);
await compilationSucceeded.resolve();
} catch (e) {
await compilationSucceeded.reject(e as Error);
}
});
it(`should be able to run query ${queryName}`, async function () {
it(`should be able to run query ${queryName}`, async () => {
try {
await compilationSucceeded.done();
const callbackId = qs.registerCallback((_res) => {
@@ -246,7 +239,7 @@ describe("using the legacy query server", function () {
});
const actualResultSets: ResultSets = {};
it(`should be able to parse results of query ${queryName}`, async function () {
it(`should be able to parse results of query ${queryName}`, async () => {
await evaluationSucceeded.done();
const info = await cliServer.bqrsInfo(RESULTS_PATH);
@@ -260,16 +253,15 @@ describe("using the legacy query server", function () {
await parsedResults.resolve();
});
it(`should have correct results for query ${queryName}`, async function () {
it(`should have correct results for query ${queryName}`, async () => {
await parsedResults.done();
expect(actualResultSets!).not.to.be.empty;
expect(Object.keys(actualResultSets!).sort()).to.eql(
expect(actualResultSets!).not.toHaveLength(0);
expect(Object.keys(actualResultSets!).sort()).toEqual(
Object.keys(queryTestCase.expectedResultSets).sort(),
);
for (const name in queryTestCase.expectedResultSets) {
expect(actualResultSets![name]).to.eql(
expect(actualResultSets![name]).toEqual(
queryTestCase.expectedResultSets[name],
`Results for query predicate ${name} do not match`,
);
}
});

36
extensions/ql-vscode/src/vscode-tests/cli-integration/new-query.test.ts
View File

@@ -1,4 +1,3 @@
import { expect } from "chai";
import * as path from "path";
import * as tmp from "tmp";
import { CancellationTokenSource } from "vscode-jsonrpc";
@@ -9,7 +8,7 @@ import { CellValue } from "../../pure/bqrs-cli-types";
import { extensions, Uri } from "vscode";
import { CodeQLExtensionInterface } from "../../extension";
import { fail } from "assert";
import { skipIfNoCodeQL } from "../ensureCli";
import { describeWithCodeQL } from "../cli";
import { QueryServerClient } from "../../query-server/queryserver-client";
import { logger, ProgressReporter } from "../../logging";
import { QueryResultType } from "../../pure/new-messages";
@@ -101,19 +100,19 @@ const nullProgressReporter: ProgressReporter = {
},
};
describe("using the new query server", function () {
before(function () {
skipIfNoCodeQL(this);
});
jest.setTimeout(20_000);
// Note this does not work with arrow functions as the test case bodies:
// ensure they are all written with standard anonymous functions.
this.timeout(20000);
describeWithCodeQL()("using the new query server", () => {
let testContext: any;
beforeAll(() => {
testContext = {};
});
let qs: qsClient.QueryServerClient;
let cliServer: cli.CodeQLCliServer;
let db: string;
before(async () => {
beforeAll(async () => {
try {
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -127,7 +126,7 @@ describe("using the new query server", function () {
if (
!(await cliServer.cliConstraints.supportsNewQueryServerForTests())
) {
this.ctx.skip();
testContext.ctx.skip();
}
qs = new QueryServerClient(
{
@@ -194,7 +193,7 @@ describe("using the new query server", function () {
);
});
it(`should be able to run query ${queryName}`, async function () {
it(`should be able to run query ${queryName}`, async () => {
try {
const params: messages.RunQueryParams = {
db,
@@ -213,7 +212,7 @@ describe("using the new query server", function () {
/**/
},
);
expect(result.resultType).to.equal(QueryResultType.SUCCESS);
expect(result.resultType).toBe(QueryResultType.SUCCESS);
await evaluationSucceeded.resolve();
} catch (e) {
await evaluationSucceeded.reject(e as Error);
@@ -221,7 +220,7 @@ describe("using the new query server", function () {
});
const actualResultSets: ResultSets = {};
it(`should be able to parse results of query ${queryName}`, async function () {
it(`should be able to parse results of query ${queryName}`, async () => {
await evaluationSucceeded.done();
const info = await cliServer.bqrsInfo(RESULTS_PATH);
@@ -235,16 +234,15 @@ describe("using the new query server", function () {
await parsedResults.resolve();
});
it(`should have correct results for query ${queryName}`, async function () {
it(`should have correct results for query ${queryName}`, async () => {
await parsedResults.done();
expect(actualResultSets!).not.to.be.empty;
expect(Object.keys(actualResultSets!).sort()).to.eql(
expect(actualResultSets!).not.toHaveLength(0);
expect(Object.keys(actualResultSets!).sort()).toEqual(
Object.keys(queryTestCase.expectedResultSets).sort(),
);
for (const name in queryTestCase.expectedResultSets) {
expect(actualResultSets![name]).to.eql(
expect(actualResultSets![name]).toEqual(
queryTestCase.expectedResultSets[name],
`Results for query predicate ${name} do not match`,
);
}
});

122
extensions/ql-vscode/src/vscode-tests/cli-integration/packaging.test.ts
View File

@@ -1,43 +1,39 @@
import * as sinon from "sinon";
import { extensions, window } from "vscode";
import { extensions, QuickPickItem, window } from "vscode";
import * as path from "path";
import * as pq from "proxyquire";
import { CliVersionConstraint, CodeQLCliServer } from "../../cli";
import { CodeQLCliServer } from "../../cli";
import { CodeQLExtensionInterface } from "../../extension";
import { expect } from "chai";
import { getErrorMessage } from "../../pure/helpers-pure";
const proxyquire = pq.noPreserveCache();
import * as helpers from "../../helpers";
import {
handleDownloadPacks,
handleInstallPackDependencies,
} from "../../packaging";
describe("Packaging commands", function () {
let sandbox: sinon.SinonSandbox;
// up to 3 minutes per test
this.timeout(3 * 60 * 1000);
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
describe("Packaging commands", () => {
let cli: CodeQLCliServer;
let progress: sinon.SinonSpy;
let quickPickSpy: sinon.SinonStub;
let inputBoxSpy: sinon.SinonStub;
let showAndLogErrorMessageSpy: sinon.SinonStub;
let showAndLogInformationMessageSpy: sinon.SinonStub;
let mod: any;
const progress = jest.fn();
const quickPickSpy = jest.spyOn(window, "showQuickPick");
const inputBoxSpy = jest.spyOn(window, "showInputBox");
const showAndLogErrorMessageSpy = jest.spyOn(
helpers,
"showAndLogErrorMessage",
);
const showAndLogInformationMessageSpy = jest.spyOn(
helpers,
"showAndLogInformationMessage",
);
beforeEach(async function () {
sandbox = sinon.createSandbox();
progress = sandbox.spy();
quickPickSpy = sandbox.stub(window, "showQuickPick");
inputBoxSpy = sandbox.stub(window, "showInputBox");
showAndLogErrorMessageSpy = sandbox.stub();
showAndLogInformationMessageSpy = sandbox.stub();
mod = proxyquire("../../packaging", {
"./helpers": {
showAndLogErrorMessage: showAndLogErrorMessageSpy,
showAndLogInformationMessage: showAndLogInformationMessageSpy,
},
});
beforeEach(async () => {
progress.mockReset();
quickPickSpy.mockReset().mockResolvedValue(undefined);
inputBoxSpy.mockReset().mockResolvedValue(undefined);
showAndLogErrorMessageSpy.mockReset().mockResolvedValue(undefined);
showAndLogInformationMessageSpy.mockReset().mockResolvedValue(undefined);
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -51,45 +47,41 @@ describe("Packaging commands", function () {
"Extension not initialized. Make sure cli is downloaded and installed properly.",
);
}
if (!(await cli.cliConstraints.supportsPackaging())) {
console.log(
`Packaging commands are not supported on CodeQL CLI v${CliVersionConstraint.CLI_VERSION_WITH_PACKAGING}. Skipping this test.`,
);
this.skip();
}
});
afterEach(() => {
sandbox.restore();
});
it("should download all core query packs", async () => {
quickPickSpy.resolves("Download all core query packs");
quickPickSpy.mockResolvedValue(
"Download all core query packs" as unknown as QuickPickItem,
);
await mod.handleDownloadPacks(cli, progress);
expect(showAndLogInformationMessageSpy.firstCall.args[0]).to.contain(
"Finished downloading packs.",
await handleDownloadPacks(cli, progress);
expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("Finished downloading packs."),
);
});
it("should download valid user-specified pack", async () => {
quickPickSpy.resolves("Download custom specified pack");
inputBoxSpy.resolves("codeql/csharp-solorigate-queries");
quickPickSpy.mockResolvedValue(
"Download custom specified pack" as unknown as QuickPickItem,
);
inputBoxSpy.mockResolvedValue("codeql/csharp-solorigate-queries");
await mod.handleDownloadPacks(cli, progress);
expect(showAndLogInformationMessageSpy.firstCall.args[0]).to.contain(
"Finished downloading packs.",
await handleDownloadPacks(cli, progress);
expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("Finished downloading packs."),
);
});
it("should show error when downloading invalid user-specified pack", async () => {
quickPickSpy.resolves("Download custom specified pack");
inputBoxSpy.resolves("foo/not-a-real-pack@0.0.1");
quickPickSpy.mockResolvedValue(
"Download custom specified pack" as unknown as QuickPickItem,
);
inputBoxSpy.mockResolvedValue("foo/not-a-real-pack@0.0.1");
await mod.handleDownloadPacks(cli, progress);
await handleDownloadPacks(cli, progress);
expect(showAndLogErrorMessageSpy.firstCall.args[0]).to.contain(
"Unable to download all packs.",
expect(showAndLogErrorMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("Unable to download all packs."),
);
});
@@ -98,16 +90,16 @@ describe("Packaging commands", function () {
__dirname,
"../../../src/vscode-tests/cli-integration/data",
);
quickPickSpy.resolves([
quickPickSpy.mockResolvedValue([
{
label: "integration-test-queries-javascript",
packRootDir: [rootDir],
},
]);
] as unknown as QuickPickItem);
await mod.handleInstallPackDependencies(cli, progress);
expect(showAndLogInformationMessageSpy.firstCall.args[0]).to.contain(
"Finished installing pack dependencies.",
await handleInstallPackDependencies(cli, progress);
expect(showAndLogInformationMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("Finished installing pack dependencies."),
);
});
@@ -116,20 +108,20 @@ describe("Packaging commands", function () {
__dirname,
"../../../src/vscode-tests/cli-integration/data-invalid-pack",
);
quickPickSpy.resolves([
quickPickSpy.mockResolvedValue([
{
label: "foo/bar",
packRootDir: [rootDir],
},
]);
] as unknown as QuickPickItem);
try {
// expect this to throw an error
await mod.handleInstallPackDependencies(cli, progress);
await handleInstallPackDependencies(cli, progress);
// This line should not be reached
expect(true).to.be.false;
expect(true).toBe(false);
} catch (e) {
expect(getErrorMessage(e)).to.contain(
expect(getErrorMessage(e)).toContain(
"Unable to install pack dependencies",
);
}

46
extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts
View File

@@ -6,10 +6,8 @@ import {
extensions,
Uri,
} from "vscode";
import * as sinon from "sinon";
import * as path from "path";
import * as fs from "fs-extra";
import { expect } from "chai";
import * as yaml from "js-yaml";
import { DatabaseItem, DatabaseManager } from "../../databases";
@@ -17,27 +15,22 @@ import { CodeQLExtensionInterface } from "../../extension";
import { cleanDatabases, dbLoc, storagePath } from "./global.helper";
import { importArchiveDatabase } from "../../databaseFetcher";
import { CodeQLCliServer } from "../../cli";
import { skipIfNoCodeQL } from "../ensureCli";
import { describeWithCodeQL } from "../cli";
import { tmpDir } from "../../helpers";
import { createInitialQueryInfo } from "../../run-queries-shared";
import { QueryRunner } from "../../queryRunner";
jest.setTimeout(20_000);
/**
* Integration tests for queries
*/
describe("Queries", function () {
this.timeout(20_000);
before(function () {
skipIfNoCodeQL(this);
});
describeWithCodeQL()("Queries", () => {
let dbItem: DatabaseItem;
let databaseManager: DatabaseManager;
let cli: CodeQLCliServer;
let qs: QueryRunner;
let sandbox: sinon.SinonSandbox;
let progress: sinon.SinonSpy;
const progress = jest.fn();
let token: CancellationToken;
let ctx: ExtensionContext;
@@ -46,11 +39,7 @@ describe("Queries", function () {
let oldQlpackLockFile: string; // codeql v2.6.3 and earlier
let qlFile: string;
beforeEach(async function () {
this.timeout(20_000);
sandbox = sinon.createSandbox();
beforeEach(async () => {
try {
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
@@ -77,7 +66,7 @@ describe("Queries", function () {
safeDel(qlFile);
safeDel(qlpackFile);
progress = sandbox.spy();
progress.mockReset();
token = {} as CancellationToken;
// Add a database, but make sure the database manager is empty first
@@ -101,10 +90,8 @@ describe("Queries", function () {
}
});
afterEach(async function () {
this.timeout(20_000);
afterEach(async () => {
try {
sandbox.restore();
safeDel(qlpackFile);
safeDel(qlFile);
await cleanDatabases(databaseManager);
@@ -125,7 +112,7 @@ describe("Queries", function () {
);
// just check that the query was successful
expect((await result).successful).to.eq(true);
expect((await result).successful).toBe(true);
} catch (e) {
console.error("Test Failed");
fail(e as Error);
@@ -145,7 +132,7 @@ describe("Queries", function () {
token,
);
expect(result.successful).to.eq(true);
expect(result.successful).toBe(true);
} catch (e) {
console.error("Test Failed");
fail(e as Error);
@@ -156,14 +143,14 @@ describe("Queries", function () {
await commands.executeCommand("codeQL.quickQuery");
// should have created the quick query file and query pack file
expect(fs.pathExistsSync(qlFile)).to.be.true;
expect(fs.pathExistsSync(qlpackFile)).to.be.true;
expect(fs.pathExistsSync(qlFile)).toBe(true);
expect(fs.pathExistsSync(qlpackFile)).toBe(true);
const qlpackContents: any = await yaml.load(
fs.readFileSync(qlpackFile, "utf8"),
);
// Should have chosen the js libraries
expect(qlpackContents.dependencies["codeql/javascript-all"]).to.eq("*");
expect(qlpackContents.dependencies["codeql/javascript-all"]).toBe("*");
// Should also have a codeql-pack.lock.yml file
const packFileToUse = fs.pathExistsSync(qlpackLockFile)
@@ -172,8 +159,9 @@ describe("Queries", function () {
const qlpackLock: any = await yaml.load(
fs.readFileSync(packFileToUse, "utf8"),
);
expect(!!qlpackLock.dependencies["codeql/javascript-all"].version).to.be
.true;
expect(!!qlpackLock.dependencies["codeql/javascript-all"].version).toBe(
true,
);
});
it("should avoid creating a quick query", async () => {
@@ -192,7 +180,7 @@ describe("Queries", function () {
await commands.executeCommand("codeQL.quickQuery");
// should not have created the quick query file because database schema hasn't changed
expect(fs.readFileSync(qlFile, "utf8")).to.eq("xxx");
expect(fs.readFileSync(qlFile, "utf8")).toBe("xxx");
});
function safeDel(file: string) {

184
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/remote-queries-manager.test.ts
View File

@@ -1,6 +1,4 @@
import { assert, expect } from "chai";
import * as path from "path";
import * as sinon from "sinon";
import {
CancellationTokenSource,
commands,
@@ -14,7 +12,7 @@ import * as os from "os";
import * as yaml from "js-yaml";
import { QlPack } from "../../../remote-queries/run-remote-query";
import { CliVersionConstraint, CodeQLCliServer } from "../../../cli";
import { CodeQLCliServer } from "../../../cli";
import { CodeQLExtensionInterface } from "../../../extension";
import {
setRemoteControllerRepo,
@@ -35,7 +33,10 @@ import {
restoreWorkspaceReferences,
} from "../global.helper";
describe("Remote queries", function () {
// up to 3 minutes per test
jest.setTimeout(3 * 60 * 1000);
describe("Remote queries", () => {
const baseDir = path.join(
__dirname,
"../../../../src/vscode-tests/cli-integration",
@@ -45,26 +46,21 @@ describe("Remote queries", function () {
"data-remote-qlpack/qlpack.yml",
).fsPath;
let sandbox: sinon.SinonSandbox;
// up to 3 minutes per test
this.timeout(3 * 60 * 1000);
let cli: CodeQLCliServer;
let cancellationTokenSource: CancellationTokenSource;
let progress: sinon.SinonSpy;
let showQuickPickSpy: sinon.SinonStub;
let getRepositoryFromNwoStub: sinon.SinonStub;
const progress = jest.fn();
const showQuickPickSpy = jest.spyOn(window, "showQuickPick");
const getRepositoryFromNwoStub = jest.spyOn(
ghApiClient,
"getRepositoryFromNwo",
);
let ctx: ExtensionContext;
let logger: any;
let remoteQueriesManager: RemoteQueriesManager;
let originalDeps: Record<string, string> | undefined;
// use `function` so we have access to `this`
beforeEach(async function () {
sandbox = sinon.createSandbox();
beforeEach(async () => {
const extension = await extensions
.getExtension<CodeQLExtensionInterface | Record<string, never>>(
"GitHub.vscode-codeql",
@@ -80,13 +76,6 @@ describe("Remote queries", function () {
ctx = createMockExtensionContext();
if (!(await cli.cliConstraints.supportsRemoteQueries())) {
console.log(
`Remote queries are not supported on CodeQL CLI v${CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES}. Skipping this test.`,
);
this.skip();
}
logger = new OutputChannelLogger("test-logger");
remoteQueriesManager = new RemoteQueriesManager(
ctx,
@@ -97,16 +86,15 @@ describe("Remote queries", function () {
cancellationTokenSource = new CancellationTokenSource();
progress = sandbox.spy();
progress.mockReset();
// Should not have asked for a language
showQuickPickSpy = sandbox
.stub(window, "showQuickPick")
.onFirstCall()
.resolves({
showQuickPickSpy
.mockReset()
.mockResolvedValueOnce({
repositories: ["github/vscode-codeql"],
} as unknown as QuickPickItem)
.onSecondCall()
.resolves("javascript" as unknown as QuickPickItem);
.mockResolvedValue("javascript" as unknown as QuickPickItem);
const dummyRepository: Repository = {
id: 123,
@@ -114,9 +102,7 @@ describe("Remote queries", function () {
full_name: "github/vscode-codeql",
private: false,
};
getRepositoryFromNwoStub = sandbox
.stub(ghApiClient, "getRepositoryFromNwo")
.resolves(dummyRepository);
getRepositoryFromNwoStub.mockReset().mockResolvedValue(dummyRepository);
// always run in the vscode-codeql repo
await setRemoteControllerRepo("github/vscode-codeql");
@@ -130,7 +116,7 @@ describe("Remote queries", function () {
request: undefined,
}),
} as unknown as Credentials;
sandbox.stub(Credentials, "initialize").resolves(mockCredentials);
jest.spyOn(Credentials, "initialize").mockResolvedValue(mockCredentials);
// Only new version support `${workspace}` in qlpack.yml
originalDeps = await fixWorkspaceReferences(
@@ -140,25 +126,23 @@ describe("Remote queries", function () {
});
afterEach(async () => {
sandbox.restore();
await restoreWorkspaceReferences(qlpackFileWithWorkspaceRefs, originalDeps);
});
describe("runRemoteQuery", () => {
let mockSubmitRemoteQueries: sinon.SinonStub;
let executeCommandSpy: sinon.SinonStub;
const mockSubmitRemoteQueries = jest.spyOn(
ghApiClient,
"submitRemoteQueries",
);
const executeCommandSpy = jest.spyOn(commands, "executeCommand");
beforeEach(() => {
executeCommandSpy = sandbox
.stub(commands, "executeCommand")
.callThrough();
mockSubmitRemoteQueries.mockReset().mockResolvedValue({
workflow_run_id: 20,
repositories_queried: ["octodemo/hello-world-1"],
});
mockSubmitRemoteQueries = sandbox
.stub(ghApiClient, "submitRemoteQueries")
.resolves({
workflow_run_id: 20,
repositories_queried: ["octodemo/hello-world-1"],
});
executeCommandSpy.mockRestore();
});
it("should run a remote query that is part of a qlpack", async () => {
@@ -170,40 +154,40 @@ describe("Remote queries", function () {
cancellationTokenSource.token,
);
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
expect(executeCommandSpy).to.have.been.calledWith(
expect(mockSubmitRemoteQueries).toBeCalledTimes(1);
expect(executeCommandSpy).toBeCalledWith(
"codeQL.monitorRemoteQuery",
sinon.match.string,
sinon.match.has("queryFilePath", fileUri.fsPath),
expect.any(String),
expect.objectContaining({ queryFilePath: fileUri.fsPath }),
);
const request: RemoteQueriesSubmission =
mockSubmitRemoteQueries.getCall(0).lastArg;
mockSubmitRemoteQueries.mock.calls[0][1];
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
expect(showQuickPickSpy).toBeCalledTimes(1);
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
expect(getRepositoryFromNwoStub).toBeCalledTimes(1);
// check a few files that we know should exist and others that we know should not
expect(packFS.fileExists("in-pack.ql")).to.be.true;
expect(packFS.fileExists("lib.qll")).to.be.true;
expect(packFS.fileExists("qlpack.yml")).to.be.true;
expect(packFS.fileExists("in-pack.ql")).toBe(true);
expect(packFS.fileExists("lib.qll")).toBe(true);
expect(packFS.fileExists("qlpack.yml")).toBe(true);
// depending on the cli version, we should have one of these files
expect(
packFS.fileExists("qlpack.lock.yml") ||
packFS.fileExists("codeql-pack.lock.yml"),
).to.be.true;
expect(packFS.fileExists("not-in-pack.ql")).to.be.false;
).toBe(true);
expect(packFS.fileExists("not-in-pack.ql")).toBe(false);
// should have generated a correct qlpack file
const qlpackContents: any = yaml.load(
packFS.fileContents("qlpack.yml").toString("utf-8"),
);
expect(qlpackContents.name).to.equal("codeql-remote/query");
expect(qlpackContents.name).toBe("codeql-remote/query");
verifyQlPack(
"in-pack.ql",
@@ -218,8 +202,8 @@ describe("Remote queries", function () {
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal("javascript-all");
expect(packNames.length).to.be.lessThan(3).toBeGreaterThan(0);
expect(packNames[0]).toEqual("javascript-all");
});
it("should run a remote query that is not part of a qlpack", async () => {
@@ -231,34 +215,34 @@ describe("Remote queries", function () {
cancellationTokenSource.token,
);
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
expect(executeCommandSpy).to.have.been.calledWith(
expect(mockSubmitRemoteQueries).toBeCalledTimes(1);
expect(executeCommandSpy).toBeCalledWith(
"codeQL.monitorRemoteQuery",
sinon.match.string,
sinon.match.has("queryFilePath", fileUri.fsPath),
expect.any(String),
expect.objectContaining({ queryFilePath: fileUri.fsPath }),
);
const request: RemoteQueriesSubmission =
mockSubmitRemoteQueries.getCall(0).lastArg;
mockSubmitRemoteQueries.mock.calls[0][1];
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
// and a second time to ask for the language
expect(showQuickPickSpy).to.have.been.calledTwice;
expect(showQuickPickSpy).toBeCalledTimes(2);
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
expect(getRepositoryFromNwoStub).toBeCalledTimes(1);
// check a few files that we know should exist and others that we know should not
expect(packFS.fileExists("in-pack.ql")).to.be.true;
expect(packFS.fileExists("qlpack.yml")).to.be.true;
expect(packFS.fileExists("in-pack.ql")).toBe(true);
expect(packFS.fileExists("qlpack.yml")).toBe(true);
// depending on the cli version, we should have one of these files
expect(
packFS.fileExists("qlpack.lock.yml") ||
packFS.fileExists("codeql-pack.lock.yml"),
).to.be.true;
expect(packFS.fileExists("lib.qll")).to.be.false;
expect(packFS.fileExists("not-in-pack.ql")).to.be.false;
).toBe(true);
expect(packFS.fileExists("lib.qll")).toBe(false);
expect(packFS.fileExists("not-in-pack.ql")).toBe(false);
// the compiled pack
verifyQlPack(
@@ -272,11 +256,9 @@ describe("Remote queries", function () {
const qlpackContents: any = yaml.load(
packFS.fileContents("qlpack.yml").toString("utf-8"),
);
expect(qlpackContents.name).to.equal("codeql-remote/query");
expect(qlpackContents.version).to.equal("0.0.0");
expect(qlpackContents.dependencies?.["codeql/javascript-all"]).to.equal(
"*",
);
expect(qlpackContents.name).toBe("codeql-remote/query");
expect(qlpackContents.version).toBe("0.0.0");
expect(qlpackContents.dependencies?.["codeql/javascript-all"]).toBe("*");
const libraryDir = ".codeql/libraries/codeql";
const packNames = packFS.directoryContents(libraryDir).sort();
@@ -284,8 +266,8 @@ describe("Remote queries", function () {
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal("javascript-all");
expect(packNames.length).to.be.lessThan(3).toBeGreaterThan(0);
expect(packNames[0]).toEqual("javascript-all");
});
it("should run a remote query that is nested inside a qlpack", async () => {
@@ -297,33 +279,33 @@ describe("Remote queries", function () {
cancellationTokenSource.token,
);
expect(mockSubmitRemoteQueries).to.have.been.calledOnce;
expect(executeCommandSpy).to.have.been.calledWith(
expect(mockSubmitRemoteQueries).toBeCalledTimes(1);
expect(executeCommandSpy).toBeCalledWith(
"codeQL.monitorRemoteQuery",
sinon.match.string,
sinon.match.has("queryFilePath", fileUri.fsPath),
expect.any(String),
expect.objectContaining({ queryFilePath: fileUri.fsPath }),
);
const request: RemoteQueriesSubmission =
mockSubmitRemoteQueries.getCall(0).lastArg;
mockSubmitRemoteQueries.mock.calls[0][1];
const packFS = await readBundledPack(request.queryPack);
// to retrieve the list of repositories
expect(showQuickPickSpy).to.have.been.calledOnce;
expect(showQuickPickSpy).toBeCalledTimes(1);
expect(getRepositoryFromNwoStub).to.have.been.calledOnce;
expect(getRepositoryFromNwoStub).toBeCalledTimes(1);
// check a few files that we know should exist and others that we know should not
expect(packFS.fileExists("subfolder/in-pack.ql")).to.be.true;
expect(packFS.fileExists("qlpack.yml")).to.be.true;
expect(packFS.fileExists("subfolder/in-pack.ql")).toBe(true);
expect(packFS.fileExists("qlpack.yml")).toBe(true);
// depending on the cli version, we should have one of these files
expect(
packFS.fileExists("qlpack.lock.yml") ||
packFS.fileExists("codeql-pack.lock.yml"),
).to.be.true;
expect(packFS.fileExists("otherfolder/lib.qll")).to.be.true;
expect(packFS.fileExists("not-in-pack.ql")).to.be.false;
).toBe(true);
expect(packFS.fileExists("otherfolder/lib.qll")).toBe(true);
expect(packFS.fileExists("not-in-pack.ql")).toBe(false);
// the compiled pack
verifyQlPack(
@@ -337,11 +319,9 @@ describe("Remote queries", function () {
const qlpackContents: any = yaml.load(
packFS.fileContents("qlpack.yml").toString("utf-8"),
);
expect(qlpackContents.name).to.equal("codeql-remote/query");
expect(qlpackContents.version).to.equal("0.0.0");
expect(qlpackContents.dependencies?.["codeql/javascript-all"]).to.equal(
"*",
);
expect(qlpackContents.name).toBe("codeql-remote/query");
expect(qlpackContents.version).toBe("0.0.0");
expect(qlpackContents.dependencies?.["codeql/javascript-all"]).toBe("*");
const libraryDir = ".codeql/libraries/codeql";
const packNames = packFS.directoryContents(libraryDir).sort();
@@ -349,8 +329,8 @@ describe("Remote queries", function () {
// check dependencies.
// 2.7.4 and earlier have ['javascript-all', 'javascript-upgrades']
// later only have ['javascript-all']. ensure this test can handle either
expect(packNames.length).to.be.lessThan(3).and.greaterThan(0);
expect(packNames[0]).to.deep.equal("javascript-all");
expect(packNames.length).to.be.lessThan(3).toBeGreaterThan(0);
expect(packNames[0]).toEqual("javascript-all");
});
it("should cancel a run before uploading", async () => {
@@ -366,9 +346,9 @@ describe("Remote queries", function () {
try {
await promise;
assert.fail("should have thrown");
fail("should have thrown");
} catch (e) {
expect(e).to.be.instanceof(UserCancellationException);
expect(e).toBeInstanceOf(UserCancellationException);
}
});
});
@@ -389,7 +369,7 @@ describe("Remote queries", function () {
// don't check the build metadata since it is variable
delete (qlPack as any).buildMetadata;
expect(qlPack).to.deep.equal({
expect(qlPack).toEqual({
name: "codeql-remote/query",
version: packVersion,
dependencies: {

522
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts
View File

File diff suppressed because it is too large Load Diff

132
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-monitor.test.ts
View File

@@ -1,6 +1,9 @@
import * as sinon from "sinon";
import { expect } from "chai";
import { CancellationTokenSource, commands, extensions } from "vscode";
import {
CancellationToken,
CancellationTokenSource,
commands,
extensions,
} from "vscode";
import { CodeQLExtensionInterface } from "../../../extension";
import * as config from "../../../config";
@@ -17,6 +20,7 @@ import {
} from "../../factories/remote-queries/gh-api/variant-analysis-api-response";
import {
VariantAnalysis,
VariantAnalysisScannedRepository,
VariantAnalysisStatus,
} from "../../../remote-queries/shared/variant-analysis";
import { createMockScannedRepos } from "../../factories/remote-queries/gh-api/scanned-repositories";
@@ -29,21 +33,28 @@ import { Credentials } from "../../../authentication";
import { createMockVariantAnalysis } from "../../factories/remote-queries/shared/variant-analysis";
import { VariantAnalysisManager } from "../../../remote-queries/variant-analysis-manager";
describe("Variant Analysis Monitor", async function () {
this.timeout(60000);
jest.setTimeout(60_000);
let sandbox: sinon.SinonSandbox;
describe("Variant Analysis Monitor", async () => {
let extension: CodeQLExtensionInterface | Record<string, never>;
let mockGetVariantAnalysis: sinon.SinonStub;
const mockGetVariantAnalysis = jest.spyOn(ghApiClient, "getVariantAnalysis");
let cancellationTokenSource: CancellationTokenSource;
let variantAnalysisMonitor: VariantAnalysisMonitor;
let variantAnalysis: VariantAnalysis;
let variantAnalysisManager: VariantAnalysisManager;
let mockGetDownloadResult: sinon.SinonStub;
let mockGetDownloadResult: jest.SpyInstance<
Promise<void>,
[
scannedRepo: VariantAnalysisScannedRepository,
variantAnalysis: VariantAnalysis,
cancellationToken: CancellationToken,
]
>;
beforeEach(async () => {
sandbox = sinon.createSandbox();
sandbox.stub(config, "isVariantAnalysisLiveResultsEnabled").returns(false);
jest
.spyOn(config, "isVariantAnalysisLiveResultsEnabled")
.mockReturnValue(false);
cancellationTokenSource = new CancellationTokenSource();
@@ -61,21 +72,22 @@ describe("Variant Analysis Monitor", async function () {
}
variantAnalysisManager = extension.variantAnalysisManager;
mockGetDownloadResult = sandbox.stub(
variantAnalysisManager,
"autoDownloadVariantAnalysisResult",
);
mockGetDownloadResult = jest
.spyOn(variantAnalysisManager, "autoDownloadVariantAnalysisResult")
.mockResolvedValue(undefined);
mockGetVariantAnalysis
.mockReset()
.mockRejectedValue(new Error("Not mocked"));
limitNumberOfAttemptsToMonitor();
});
afterEach(async () => {
sandbox.restore();
});
describe("when credentials are invalid", async () => {
beforeEach(async () => {
sandbox.stub(Credentials, "initialize").resolves(undefined);
jest
.spyOn(Credentials, "initialize")
.mockResolvedValue(undefined as unknown as Credentials);
});
it("should return early if credentials are wrong", async () => {
@@ -85,7 +97,7 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
} catch (error: any) {
expect(error.message).to.equal("Error authenticating with GitHub");
expect(error.message).toBe("Error authenticating with GitHub");
}
});
});
@@ -95,10 +107,10 @@ describe("Variant Analysis Monitor", async function () {
const mockCredentials = {
getOctokit: () =>
Promise.resolve({
request: mockGetVariantAnalysis,
request: jest.fn(),
}),
} as unknown as Credentials;
sandbox.stub(Credentials, "initialize").resolves(mockCredentials);
jest.spyOn(Credentials, "initialize").mockResolvedValue(mockCredentials);
});
it("should return early if variant analysis is cancelled", async () => {
@@ -109,17 +121,15 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(result).to.eql({ status: "Canceled" });
expect(result).toEqual({ status: "Canceled" });
});
describe("when the variant analysis fails", async () => {
let mockFailedApiResponse: VariantAnalysisApiResponse;
beforeEach(async function () {
beforeEach(async () => {
mockFailedApiResponse = createFailedMockApiResponse();
mockGetVariantAnalysis = sandbox
.stub(ghApiClient, "getVariantAnalysis")
.resolves(mockFailedApiResponse);
mockGetVariantAnalysis.mockResolvedValue(mockFailedApiResponse);
});
it("should mark as failed locally and stop monitoring", async () => {
@@ -128,12 +138,12 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(mockGetVariantAnalysis.calledOnce).to.be.true;
expect(result.status).to.eql("Completed");
expect(result.variantAnalysis?.status).to.equal(
expect(mockGetVariantAnalysis).toHaveBeenCalledTimes(1);
expect(result.status).toEqual("Completed");
expect(result.variantAnalysis?.status).toBe(
VariantAnalysisStatus.Failed,
);
expect(result.variantAnalysis?.failureReason).to.equal(
expect(result.variantAnalysis?.failureReason).toBe(
processFailureReason(
mockFailedApiResponse.failure_reason as VariantAnalysisFailureReason,
),
@@ -141,7 +151,7 @@ describe("Variant Analysis Monitor", async function () {
});
it("should emit `onVariantAnalysisChange`", async () => {
const spy = sandbox.spy();
const spy = jest.fn();
variantAnalysisMonitor.onVariantAnalysisChange(spy);
const result = await variantAnalysisMonitor.monitorVariantAnalysis(
@@ -149,7 +159,7 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(spy).to.have.been.calledWith(result.variantAnalysis);
expect(spy).toBeCalledWith(result.variantAnalysis);
});
});
@@ -159,7 +169,7 @@ describe("Variant Analysis Monitor", async function () {
let succeededRepos: ApiVariantAnalysisScannedRepository[];
describe("when there are successfully scanned repos", async () => {
beforeEach(async function () {
beforeEach(async () => {
scannedRepos = createMockScannedRepos([
"pending",
"pending",
@@ -170,9 +180,7 @@ describe("Variant Analysis Monitor", async function () {
"succeeded",
]);
mockApiResponse = createMockApiResponse("succeeded", scannedRepos);
mockGetVariantAnalysis = sandbox
.stub(ghApiClient, "getVariantAnalysis")
.resolves(mockApiResponse);
mockGetVariantAnalysis.mockResolvedValue(mockApiResponse);
succeededRepos = scannedRepos.filter(
(r) => r.analysis_status === "succeeded",
);
@@ -184,8 +192,8 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(result.status).to.equal("Completed");
expect(result.scannedReposDownloaded).to.eql(
expect(result.status).toBe("Completed");
expect(result.scannedReposDownloaded).toEqual(
succeededRepos.map((r) => r.repository.id),
);
});
@@ -194,23 +202,22 @@ describe("Variant Analysis Monitor", async function () {
const succeededRepos = scannedRepos.filter(
(r) => r.analysis_status === "succeeded",
);
const commandSpy = sandbox.spy(commands, "executeCommand");
const commandSpy = jest
.spyOn(commands, "executeCommand")
.mockResolvedValue(undefined);
await variantAnalysisMonitor.monitorVariantAnalysis(
variantAnalysis,
cancellationTokenSource.token,
);
expect(commandSpy).to.have.callCount(succeededRepos.length);
expect(commandSpy).toBeCalledTimes(succeededRepos.length);
succeededRepos.forEach((succeededRepo, index) => {
expect(commandSpy.getCall(index).args[0]).to.eq(
expect(commandSpy).toHaveBeenNthCalledWith(
index + 1,
"codeQL.autoDownloadVariantAnalysisResult",
);
expect(commandSpy.getCall(index).args[1]).to.deep.eq(
processScannedRepository(succeededRepo),
);
expect(commandSpy.getCall(index).args[2]).to.deep.eq(
processUpdatedVariantAnalysis(variantAnalysis, mockApiResponse),
);
});
@@ -222,15 +229,12 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(mockGetDownloadResult).to.have.callCount(
succeededRepos.length,
);
expect(mockGetDownloadResult).toBeCalledTimes(succeededRepos.length);
succeededRepos.forEach((succeededRepo, index) => {
expect(mockGetDownloadResult.getCall(index).args[0]).to.deep.eq(
expect(mockGetDownloadResult).toHaveBeenNthCalledWith(
index + 1,
processScannedRepository(succeededRepo),
);
expect(mockGetDownloadResult.getCall(index).args[1]).to.deep.eq(
processUpdatedVariantAnalysis(variantAnalysis, mockApiResponse),
);
});
@@ -240,12 +244,10 @@ describe("Variant Analysis Monitor", async function () {
describe("when there are only in progress repos", async () => {
let scannedRepos: ApiVariantAnalysisScannedRepository[];
beforeEach(async function () {
beforeEach(async () => {
scannedRepos = createMockScannedRepos(["pending", "in_progress"]);
mockApiResponse = createMockApiResponse("in_progress", scannedRepos);
mockGetVariantAnalysis = sandbox
.stub(ghApiClient, "getVariantAnalysis")
.resolves(mockApiResponse);
mockGetVariantAnalysis.mockResolvedValue(mockApiResponse);
});
it("should succeed and return an empty list of scanned repo ids", async () => {
@@ -254,8 +256,8 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(result.status).to.equal("Completed");
expect(result.scannedReposDownloaded).to.eql([]);
expect(result.status).toBe("Completed");
expect(result.scannedReposDownloaded).toEqual([]);
});
it("should not try to download any repos", async () => {
@@ -264,17 +266,15 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(mockGetDownloadResult).to.not.have.been.called;
expect(mockGetDownloadResult).not.toBeCalled();
});
});
describe("when there are no repos to scan", async () => {
beforeEach(async function () {
beforeEach(async () => {
scannedRepos = [];
mockApiResponse = createMockApiResponse("succeeded", scannedRepos);
mockGetVariantAnalysis = sandbox
.stub(ghApiClient, "getVariantAnalysis")
.resolves(mockApiResponse);
mockGetVariantAnalysis.mockResolvedValue(mockApiResponse);
});
it("should succeed and return an empty list of scanned repo ids", async () => {
@@ -283,8 +283,8 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(result.status).to.equal("Completed");
expect(result.scannedReposDownloaded).to.eql([]);
expect(result.status).toBe("Completed");
expect(result.scannedReposDownloaded).toEqual([]);
});
it("should not try to download any repos", async () => {
@@ -293,7 +293,7 @@ describe("Variant Analysis Monitor", async function () {
cancellationTokenSource.token,
);
expect(mockGetDownloadResult).to.not.have.been.called;
expect(mockGetDownloadResult).not.toBeCalled();
});
});
});

58
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-results-manager.test.ts
View File

@@ -1,5 +1,3 @@
import * as sinon from "sinon";
import { expect } from "chai";
import { extensions } from "vscode";
import { CodeQLExtensionInterface } from "../../../extension";
import { logger } from "../../../logging";
@@ -15,20 +13,17 @@ import * as ghApiClient from "../../../remote-queries/gh-api/gh-api-client";
import { createMockVariantAnalysisRepositoryTask } from "../../factories/remote-queries/shared/variant-analysis-repo-tasks";
import { VariantAnalysisRepositoryTask } from "../../../remote-queries/shared/variant-analysis";
describe(VariantAnalysisResultsManager.name, function () {
this.timeout(10000);
jest.setTimeout(10_000);
let sandbox: sinon.SinonSandbox;
describe(VariantAnalysisResultsManager.name, () => {
let cli: CodeQLCliServer;
let variantAnalysisId: number;
let variantAnalysisResultsManager: VariantAnalysisResultsManager;
let getVariantAnalysisRepoResultStub: sinon.SinonStub;
beforeEach(async () => {
sandbox = sinon.createSandbox();
sandbox.stub(logger, "log");
sandbox.stub(fs, "mkdirSync");
sandbox.stub(fs, "writeFile");
jest.spyOn(logger, "log").mockResolvedValue(undefined);
jest.spyOn(fs, "mkdirSync").mockReturnValue(undefined);
jest.spyOn(fs, "writeFile").mockReturnValue(undefined);
variantAnalysisId = faker.datatype.number();
@@ -48,16 +43,11 @@ describe(VariantAnalysisResultsManager.name, function () {
}
});
afterEach(async () => {
sandbox.restore();
});
describe("download", () => {
let getOctokitStub: sinon.SinonStub;
const mockCredentials = {
getOctokit: () =>
Promise.resolve({
request: getOctokitStub,
request: jest.fn(),
}),
} as unknown as Credentials;
let dummyRepoTask: VariantAnalysisRepositoryTask;
@@ -91,7 +81,7 @@ describe(VariantAnalysisResultsManager.name, function () {
variantAnalysisStoragePath,
dummyRepoTask.repository.fullName,
),
).to.equal(false);
).toBe(false);
});
});
@@ -108,9 +98,9 @@ describe(VariantAnalysisResultsManager.name, function () {
variantAnalysisStoragePath,
);
expect.fail("Expected an error to be thrown");
fail("Expected an error to be thrown");
} catch (e: any) {
expect(e.message).to.equal("Missing artifact URL");
expect(e.message).toBe("Missing artifact URL");
}
});
});
@@ -118,6 +108,11 @@ describe(VariantAnalysisResultsManager.name, function () {
describe("when the artifact_url is present", async () => {
let arrayBuffer: ArrayBuffer;
const getVariantAnalysisRepoResultStub = jest.spyOn(
ghApiClient,
"getVariantAnalysisRepoResult",
);
beforeEach(async () => {
const sourceFilePath = path.join(
__dirname,
@@ -125,10 +120,16 @@ describe(VariantAnalysisResultsManager.name, function () {
);
arrayBuffer = fs.readFileSync(sourceFilePath).buffer;
getVariantAnalysisRepoResultStub = sandbox
.stub(ghApiClient, "getVariantAnalysisRepoResult")
.withArgs(mockCredentials, dummyRepoTask.artifactUrl as string)
.resolves(arrayBuffer);
getVariantAnalysisRepoResultStub
.mockReset()
.mockImplementation(
(_credentials: Credentials, downloadUrl: string) => {
if (downloadUrl === dummyRepoTask.artifactUrl) {
return Promise.resolve(arrayBuffer);
}
return Promise.reject(new Error("Unexpected artifact URL"));
},
);
});
it("should call the API to download the results", async () => {
@@ -139,7 +140,7 @@ describe(VariantAnalysisResultsManager.name, function () {
variantAnalysisStoragePath,
);
expect(getVariantAnalysisRepoResultStub.calledOnce).to.be.true;
expect(getVariantAnalysisRepoResultStub).toHaveBeenCalledTimes(1);
});
it("should save the results zip file to disk", async () => {
@@ -150,8 +151,9 @@ describe(VariantAnalysisResultsManager.name, function () {
variantAnalysisStoragePath,
);
expect(fs.existsSync(`${repoTaskStorageDirectory}/results.zip`)).to.be
.true;
expect(fs.existsSync(`${repoTaskStorageDirectory}/results.zip`)).toBe(
true,
);
});
it("should unzip the results in a `results/` folder", async () => {
@@ -164,7 +166,7 @@ describe(VariantAnalysisResultsManager.name, function () {
expect(
fs.existsSync(`${repoTaskStorageDirectory}/results/results.sarif`),
).to.be.true;
).toBe(true);
});
describe("isVariantAnalysisRepoDownloaded", () => {
@@ -181,7 +183,7 @@ describe(VariantAnalysisResultsManager.name, function () {
variantAnalysisStoragePath,
dummyRepoTask.repository.fullName,
),
).to.equal(true);
).toBe(true);
});
});
});

99
extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-submission-integration.test.ts
View File

@@ -1,8 +1,13 @@
import * as path from "path";
import * as sinon from "sinon";
import { commands, extensions, TextDocument, window, workspace } from "vscode";
import {
commands,
extensions,
QuickPickItem,
TextDocument,
window,
workspace,
} from "vscode";
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
@@ -11,10 +16,12 @@ import * as config from "../../../config";
import { Credentials } from "../../../authentication";
import { MockGitHubApiServer } from "../../../mocks/mock-gh-api-server";
jest.setTimeout(10_000);
const mockServer = new MockGitHubApiServer();
before(() => mockServer.startServer());
beforeAll(() => mockServer.startServer());
afterEach(() => mockServer.unloadScenario());
after(() => mockServer.stopServer());
afterAll(() => mockServer.stopServer());
async function showQlDocument(name: string): Promise<TextDocument> {
const folderPath = workspace.workspaceFolders![0].uri.fsPath;
@@ -24,35 +31,30 @@ async function showQlDocument(name: string): Promise<TextDocument> {
return document;
}
describe("Variant Analysis Submission Integration", function () {
this.timeout(10_000);
let sandbox: sinon.SinonSandbox;
let quickPickSpy: sinon.SinonStub;
let inputBoxSpy: sinon.SinonStub;
let executeCommandSpy: sinon.SinonStub;
let showErrorMessageSpy: sinon.SinonStub;
describe("Variant Analysis Submission Integration", () => {
const quickPickSpy = jest.spyOn(window, "showQuickPick");
const inputBoxSpy = jest.spyOn(window, "showInputBox");
const executeCommandSpy = jest.spyOn(commands, "executeCommand");
const showErrorMessageSpy = jest.spyOn(window, "showErrorMessage");
beforeEach(async () => {
sandbox = sinon.createSandbox();
sandbox.stub(config, "isCanary").returns(true);
sandbox.stub(config, "isVariantAnalysisLiveResultsEnabled").returns(true);
jest.spyOn(config, "isCanary").mockReturnValue(true);
jest
.spyOn(config, "isVariantAnalysisLiveResultsEnabled")
.mockReturnValue(true);
const mockCredentials = {
getOctokit: () => Promise.resolve(new Octokit.Octokit({ retry })),
} as unknown as Credentials;
sandbox.stub(Credentials, "initialize").resolves(mockCredentials);
jest.spyOn(Credentials, "initialize").mockResolvedValue(mockCredentials);
await config.setRemoteControllerRepo("github/vscode-codeql");
quickPickSpy = sandbox.stub(window, "showQuickPick").resolves(undefined);
inputBoxSpy = sandbox.stub(window, "showInputBox").resolves(undefined);
quickPickSpy.mockReset().mockResolvedValue(undefined);
inputBoxSpy.mockReset().mockResolvedValue(undefined);
executeCommandSpy = sandbox.stub(commands, "executeCommand").callThrough();
showErrorMessageSpy = sandbox
.stub(window, "showErrorMessage")
.resolves(undefined);
executeCommandSpy.mockRestore();
showErrorMessageSpy.mockReset().mockResolvedValue(undefined);
try {
await extensions
@@ -65,10 +67,6 @@ describe("Variant Analysis Submission Integration", function () {
}
});
afterEach(() => {
sandbox.restore();
});
describe("Successful scenario", () => {
beforeEach(async () => {
await mockServer.loadScenario("problem-query-success");
@@ -78,18 +76,19 @@ describe("Variant Analysis Submission Integration", function () {
await showQlDocument("query.ql");
// Select a repository list
quickPickSpy.onFirstCall().resolves({
quickPickSpy.mockResolvedValueOnce({
useCustomRepo: true,
});
} as unknown as QuickPickItem);
// Enter a GitHub repository
inputBoxSpy.onFirstCall().resolves("github/codeql");
inputBoxSpy.mockResolvedValueOnce("github/codeql");
// Select target language for your query
quickPickSpy.onSecondCall().resolves("javascript");
quickPickSpy.mockResolvedValueOnce(
"javascript" as unknown as QuickPickItem,
);
await commands.executeCommand("codeQL.runVariantAnalysis");
sinon.assert.calledWith(
executeCommandSpy,
expect(executeCommandSpy).toHaveBeenCalledWith(
"codeQL.openVariantAnalysisView",
146,
);
@@ -105,18 +104,19 @@ describe("Variant Analysis Submission Integration", function () {
await showQlDocument("query.ql");
// Select a repository list
quickPickSpy.onFirstCall().resolves({
quickPickSpy.mockResolvedValueOnce({
useCustomRepo: true,
});
} as unknown as QuickPickItem);
// Enter a GitHub repository
inputBoxSpy.onFirstCall().resolves("github/codeql");
inputBoxSpy.mockResolvedValueOnce("github/codeql");
await commands.executeCommand("codeQL.runVariantAnalysis");
sinon.assert.calledWith(
showErrorMessageSpy,
sinon.match('Controller repository "github/vscode-codeql" not found'),
sinon.match.string,
expect(showErrorMessageSpy).toHaveBeenCalledWith(
expect.stringContaining(
'Controller repository "github/vscode-codeql" not found',
),
expect.any(String),
);
});
});
@@ -130,20 +130,21 @@ describe("Variant Analysis Submission Integration", function () {
await showQlDocument("query.ql");
// Select a repository list
quickPickSpy.onFirstCall().resolves({
quickPickSpy.mockResolvedValueOnce({
useCustomRepo: true,
});
} as unknown as QuickPickItem);
// Enter a GitHub repository
inputBoxSpy.onFirstCall().resolves("github/codeql");
inputBoxSpy.mockResolvedValueOnce("github/codeql");
// Select target language for your query
quickPickSpy.onSecondCall().resolves("javascript");
quickPickSpy.mockResolvedValueOnce(
"javascript" as unknown as QuickPickItem,
);
await commands.executeCommand("codeQL.runVariantAnalysis");
sinon.assert.calledWith(
showErrorMessageSpy,
sinon.match("No repositories could be queried."),
sinon.match.string,
expect(showErrorMessageSpy).toHaveBeenCalledWith(
expect.stringContaining("No repositories could be queried."),
expect.any(String),
);
});
});

89
extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts
View File

@@ -1,11 +1,10 @@
import { expect } from "chai";
import { extensions, Uri } from "vscode";
import * as path from "path";
import { SemVer } from "semver";
import { CodeQLCliServer, QueryInfoByLanguage } from "../../cli";
import { CodeQLExtensionInterface } from "../../extension";
import { skipIfNoCodeQL } from "../ensureCli";
import { itWithCodeQL } from "../cli";
import {
getOnDiskWorkspaceFolders,
getQlPackForDbscheme,
@@ -15,12 +14,12 @@ import { resolveQueries } from "../../contextual/queryResolver";
import { KeyType } from "../../contextual/keyType";
import { fail } from "assert";
jest.setTimeout(60_000);
/**
* Perform proper integration tests by running the CLI
*/
describe("Use cli", function () {
this.timeout(60000);
describe("Use cli", () => {
let cli: CodeQLCliServer;
let supportedLanguages: string[];
@@ -42,7 +41,7 @@ describe("Use cli", function () {
if (process.env.CLI_VERSION && process.env.CLI_VERSION !== "nightly") {
it("should have the correct version of the cli", async () => {
expect((await cli.getVersion()).toString()).to.eq(
expect((await cli.getVersion()).toString()).toBe(
new SemVer(process.env.CLI_VERSION || "").toString(),
);
});
@@ -50,11 +49,10 @@ describe("Use cli", function () {
it("should resolve ram", async () => {
const result = await (cli as any).resolveRam(8192);
expect(result).to.deep.eq(["-J-Xmx4096M", "--off-heap-ram=4096"]);
expect(result).toEqual(["-J-Xmx4096M", "--off-heap-ram=4096"]);
});
it("should resolve query packs", async function () {
skipIfNoCodeQL(this);
itWithCodeQL()("should resolve query packs", async () => {
const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
// Depending on the version of the CLI, the qlpacks may have different names
// (e.g. "codeql/javascript-all" vs "codeql-javascript"),
@@ -64,19 +62,17 @@ describe("Use cli", function () {
}
});
it("should support the expected languages", async function () {
skipIfNoCodeQL(this);
itWithCodeQL()("should support the expected languages", async () => {
// Just check a few examples that definitely are/aren't supported.
expect(supportedLanguages).to.include.members([
"go",
"javascript",
"python",
]);
expect(supportedLanguages).to.not.include.members(["xml", "properties"]);
expect(supportedLanguages).toEqual(
expect.arrayContaining(["go", "javascript", "python"]),
);
expect(supportedLanguages).not.toEqual(
expect.arrayContaining(["xml", "properties"]),
);
});
it("should resolve query by language", async function () {
skipIfNoCodeQL(this);
itWithCodeQL()("should resolve query by language", async () => {
const queryPath = path.join(
__dirname,
"data",
@@ -86,33 +82,38 @@ describe("Use cli", function () {
getOnDiskWorkspaceFolders(),
Uri.file(queryPath),
);
expect(Object.keys(queryInfo.byLanguage)[0]).to.eql("javascript");
expect(Object.keys(queryInfo.byLanguage)[0]).toEqual("javascript");
});
it("should resolve printAST queries for supported languages", async function () {
skipIfNoCodeQL(this);
try {
for (const lang of supportedLanguages) {
if (lang === "go") {
// The codeql-go submodule is not available in the integration tests.
return;
itWithCodeQL()(
"should resolve printAST queries for supported languages",
async () => {
try {
for (const lang of supportedLanguages) {
if (lang === "go") {
// The codeql-go submodule is not available in the integration tests.
return;
}
console.log(`resolving printAST queries for ${lang}`);
const pack = await getQlPackForDbscheme(
cli,
languageToDbScheme[lang],
);
expect(pack.dbschemePack).toEqual(expect.arrayContaining([lang]));
if (pack.dbschemePackIsLibraryPack) {
expect(pack.queryPack).toEqual(expect.arrayContaining([lang]));
}
const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
// It doesn't matter what the name or path of the query is, only
// that we have found exactly one query.
expect(result.length).toBe(1);
}
console.log(`resolving printAST queries for ${lang}`);
const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]);
expect(pack.dbschemePack).to.contain(lang);
if (pack.dbschemePackIsLibraryPack) {
expect(pack.queryPack).to.contain(lang);
}
const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
// It doesn't matter what the name or path of the query is, only
// that we have found exactly one query.
expect(result.length).to.eq(1);
} catch (e) {
fail(e as Error);
}
} catch (e) {
fail(e as Error);
}
});
},
);
});

15
extensions/ql-vscode/src/vscode-tests/cli-integration/sourcemap.test.ts
View File

@@ -2,16 +2,15 @@ import { fail } from "assert";
import { commands, Selection, window, workspace } from "vscode";
import * as path from "path";
import * as assert from "assert";
import { expect } from "chai";
import { tmpDir } from "../../helpers";
import * as fs from "fs-extra";
jest.setTimeout(20_000);
/**
* Integration tests for queries
*/
describe("SourceMap", function () {
this.timeout(20000);
describe("SourceMap", () => {
it("should jump to QL code", async () => {
try {
const root = workspace.workspaceFolders![0].uri.fsPath;
@@ -41,12 +40,12 @@ describe("SourceMap", function () {
await commands.executeCommand("codeQL.gotoQL");
const newEditor = window.activeTextEditor;
expect(newEditor).to.be.not.undefined;
expect(newEditor).toBeDefined();
const newDocument = newEditor!.document;
expect(path.basename(newDocument.fileName)).to.equal("Namespace.qll");
expect(path.basename(newDocument.fileName)).toBe("Namespace.qll");
const newSelection = newEditor!.selection;
expect(newSelection.start.line).to.equal(60);
expect(newSelection.start.character).to.equal(2);
expect(newSelection.start.line).toBe(60);
expect(newSelection.start.character).toBe(2);
} catch (e) {
console.error("Test Failed");
fail(e as Error);

44
extensions/ql-vscode/src/vscode-tests/cli.ts Normal file
View File

@@ -0,0 +1,44 @@
import { workspace } from "vscode";
/**
* Heuristically determines if the codeql libraries are installed in this
* workspace. Looks for the existance of a folder whose path ends in `/codeql`
*/
function hasCodeQL() {
const folders = workspace.workspaceFolders;
return !!folders?.some((folder) => folder.uri.path.endsWith("/codeql"));
}
// describeWithCodeQL will be equal to describe if the CodeQL libraries are
// available in this workspace. Otherwise, it will skip the tests.
export function describeWithCodeQL() {
if (!hasCodeQL()) {
console.log(
[
"The CodeQL libraries are not available as a folder in this workspace.",
"To fix in CI: checkout the github/codeql repository and set the 'TEST_CODEQL_PATH' environment variable to the checked out directory.",
"To fix when running from vs code, see the comment in the launch.json file in the 'Launch Integration Tests - With CLI' section.",
].join("\n\n"),
);
return describe.skip;
}
return describe;
}
// itWithCodeQL will be equal to it if the CodeQL libraries are
// available in this workspace. Otherwise, it will skip the tests.
export function itWithCodeQL() {
if (!hasCodeQL()) {
console.log(
[
"The CodeQL libraries are not available as a folder in this workspace.",
"To fix in CI: checkout the github/codeql repository and set the 'TEST_CODEQL_PATH' environment variable to the checked out directory.",
"To fix when running from vs code, see the comment in the launch.json file in the 'Launch Integration Tests - With CLI' section.",
].join("\n\n"),
);
return it.skip;
}
return it;
}

29
extensions/ql-vscode/src/vscode-tests/ensureCli.ts
View File

@@ -1,12 +1,11 @@
import * as fs from "fs-extra";
import * as path from "path";
import {
DistributionManager,
getRequiredAssetName,
extractZipArchive,
codeQlLauncherName,
} from "../distribution";
} from "../pure/distribution";
import fetch from "node-fetch";
import { workspace } from "vscode";
/**
* This module ensures that the proper CLI is available for tests of the extension.
@@ -64,7 +63,7 @@ export async function ensureCli(useCli: boolean) {
return;
}
const assetName = DistributionManager.getRequiredAssetName();
const assetName = getRequiredAssetName();
const url = getCliDownloadUrl(assetName);
const unzipDir = getCliUnzipDir();
const downloadedFilePath = getDownloadFilePath(assetName);
@@ -136,28 +135,6 @@ export async function ensureCli(useCli: boolean) {
}
}
/**
* Heuristically determines if the codeql libraries are installed in this
* workspace. Looks for the existance of a folder whose path ends in `/codeql`
*/
function hasCodeQL() {
const folders = workspace.workspaceFolders;
return !!folders?.some((folder) => folder.uri.path.endsWith("/codeql"));
}
export function skipIfNoCodeQL(context: Mocha.Context) {
if (!hasCodeQL()) {
console.log(
[
"The CodeQL libraries are not available as a folder in this workspace.",
"To fix in CI: checkout the github/codeql repository and set the 'TEST_CODEQL_PATH' environment variable to the checked out directory.",
"To fix when running from vs code, see the comment in the launch.json file in the 'Launch Integration Tests - With CLI' section.",
].join("\n\n"),
);
context.skip();
}
}
/**
* Url to download from
*/

2
extensions/ql-vscode/src/vscode-tests/jest-runner-vscode.config.base.ts
View File

@@ -9,8 +9,8 @@ export const rootDir = path.resolve(__dirname, "../..");
const config: RunnerOptions = {
version: "stable",
launchArgs: [
"--disable-extensions",
"--disable-gpu",
"--extensions-dir=" + path.join(rootDir, ".vscode-test", "extensions"),
"--user-data-dir=" + path.join(tmpDir.name, "user-data"),
],
extensionDevelopmentPath: rootDir,

12
extensions/ql-vscode/src/vscode-tests/jest.setup.ts
View File

@@ -1,5 +1,17 @@
import { env } from "vscode";
import { jestTestConfigHelper } from "./test-config";
(env as any).openExternal = () => {
/**/
};
function fail(reason = "fail was called in a test.") {
throw new Error(reason);
}
// Jest doesn't seem to define this function anymore, but it's in the types, so should be valid.
(global as any).fail = fail;
export default async function setupEnv() {
await jestTestConfigHelper();
}

1
extensions/ql-vscode/src/vscode-tests/no-workspace/jest-runner-vscode.config.ts
View File

@@ -4,6 +4,7 @@ import baseConfig from "../jest-runner-vscode.config.base";
const config: RunnerOptions = {
...baseConfig,
launchArgs: [...(baseConfig.launchArgs ?? []), "--disable-extensions"],
};
// We are purposefully not using export default here since that would result in an ESModule, which doesn't seem to be

20
extensions/ql-vscode/src/vscode-tests/test-config.ts
View File

@@ -127,3 +127,23 @@ export const testConfigHelper = async (mocha: Mocha) => {
},
});
};
export const jestTestConfigHelper = async () => {
// Read in all current settings
await Promise.all(TEST_SETTINGS.map((setting) => setting.initialSetup()));
beforeEach(async () => {
// Reset the settings to their initial values before each test
await Promise.all(TEST_SETTINGS.map((setting) => setting.setup()));
});
afterAll(async () => {
// Restore all settings to their default values after each test suite
// Only do this outside of CI since the sometimes hangs on CI.
if (process.env.CI !== "true") {
await Promise.all(
TEST_SETTINGS.map((setting) => setting.restoreToInitialValues()),
);
}
});
};