Merge remote-tracking branch 'origin/main' into koesie10/upload-sourcemaps-release

This commit is contained in:
Koen Vlaswinkel
2023-02-08 13:31:13 +00:00
20 changed files with 806 additions and 322 deletions

View File

@@ -228,6 +228,13 @@ Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's poss
1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
* Go through all recent PRs and make sure they are properly accounted for.
* Make sure all changelog entries have links back to their PR(s) if appropriate.
* For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
* Making substantial new features available to all users. This can include lifting a feature flag.
* Breakage in compatibility with recent versions of the CLI.
* Minimum required version of VS Code is increased.
* New telemetry events are added.
* Deprecation or removal of commands.
* Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
* `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
* `.github/workflows/main.yml` - all the "node-version: <version>" settings

View File

@@ -244,7 +244,7 @@ This requires running a MRVA query and seeing the results view.
1. By name
2. By results
3. By stars
4. By last commit
4. By last updated
9. Can filter repos
10. Shows correct statistics
1. Total number of results

View File

@@ -0,0 +1,193 @@
/**
* This scripts helps finding the original source file and line number for a
* given file and line number in the compiled extension. It currently only
* works with released extensions.
*
* Usage: npx ts-node scripts/source-map.ts <version-number> <filename>:<line>:<column>
* For example: npx ts-node scripts/source-map.ts v1.7.8 "/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:131164:13"
*
* Alternative usage: npx ts-node scripts/source-map.ts <version-number> <multi-line-stacktrace>
* For example: npx ts-node scripts/source-map.ts v1.7.8 'Error: Failed to find CodeQL distribution.
* at CodeQLCliServer.getCodeQlPath (/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:131164:13)
* at CodeQLCliServer.launchProcess (/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:131169:24)
* at CodeQLCliServer.runCodeQlCliInternal (/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:131194:24)
* at CodeQLCliServer.runJsonCodeQlCliCommand (/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:131330:20)
* at CodeQLCliServer.resolveRam (/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:131455:12)
* at QueryServerClient2.startQueryServerImpl (/Users/user/.vscode/extensions/github.vscode-codeql-1.7.8/out/extension.js:138618:21)'
*/
import { spawnSync } from "child_process";
import { basename, resolve } from "path";
import { pathExists, readJSON } from "fs-extra";
import { RawSourceMap, SourceMapConsumer } from "source-map";
if (process.argv.length !== 4) {
console.error(
"Expected 2 arguments - the version number and the filename:line number",
);
}
const stackLineRegex =
/at (?<name>.*)? \((?<file>.*):(?<line>\d+):(?<column>\d+)\)/gm;
const versionNumber = process.argv[2].startsWith("v")
? process.argv[2]
: `v${process.argv[2]}`;
const stacktrace = process.argv[3];
async function extractSourceMap() {
const sourceMapsDirectory = resolve(
__dirname,
"..",
"artifacts",
"source-maps",
versionNumber,
);
if (!(await pathExists(sourceMapsDirectory))) {
console.log("Downloading source maps...");
const workflowRuns = runGhJSON<WorkflowRunListItem[]>([
"run",
"list",
"--workflow",
"release.yml",
"--branch",
versionNumber,
"--json",
"databaseId,number",
]);
if (workflowRuns.length !== 1) {
throw new Error(
`Expected exactly one workflow run for ${versionNumber}, got ${workflowRuns.length}`,
);
}
const workflowRun = workflowRuns[0];
runGh([
"run",
"download",
workflowRun.databaseId.toString(),
"--name",
"vscode-codeql-sourcemaps",
"--dir",
sourceMapsDirectory,
]);
}
if (stacktrace.includes("at")) {
const rawSourceMaps = new Map<string, RawSourceMap>();
const mappedStacktrace = await replaceAsync(
stacktrace,
stackLineRegex,
async (match, name, file, line, column) => {
if (!rawSourceMaps.has(file)) {
const rawSourceMap: RawSourceMap = await readJSON(
resolve(sourceMapsDirectory, `${basename(file)}.map`),
);
rawSourceMaps.set(file, rawSourceMap);
}
const originalPosition = await SourceMapConsumer.with(
rawSourceMaps.get(file) as RawSourceMap,
null,
async function (consumer) {
return consumer.originalPositionFor({
line: parseInt(line, 10),
column: parseInt(column, 10),
});
},
);
if (!originalPosition.source) {
return match;
}
const originalFilename = resolve(file, "..", originalPosition.source);
return `at ${originalPosition.name ?? name} (${originalFilename}:${
originalPosition.line
}:${originalPosition.column})`;
},
);
console.log(mappedStacktrace);
} else {
// This means it's just a filename:line:column
const [filename, line, column] = stacktrace.split(":", 3);
const fileBasename = basename(filename);
const sourcemapName = `${fileBasename}.map`;
const sourcemapPath = resolve(sourceMapsDirectory, sourcemapName);
if (!(await pathExists(sourcemapPath))) {
throw new Error(`No source map found for ${fileBasename}`);
}
const rawSourceMap: RawSourceMap = await readJSON(sourcemapPath);
const originalPosition = await SourceMapConsumer.with(
rawSourceMap,
null,
async function (consumer) {
return consumer.originalPositionFor({
line: parseInt(line, 10),
column: parseInt(column, 10),
});
},
);
if (!originalPosition.source) {
throw new Error(`No source found for ${stacktrace}`);
}
const originalFilename = resolve(filename, "..", originalPosition.source);
console.log(
`${originalFilename}:${originalPosition.line}:${originalPosition.column}`,
);
}
}
extractSourceMap().catch((e: unknown) => {
console.error(e);
process.exit(2);
});
function runGh(args: readonly string[]): string {
const gh = spawnSync("gh", args);
if (gh.status !== 0) {
throw new Error(
`Failed to get the source map for ${versionNumber}: ${gh.stderr}`,
);
}
return gh.stdout.toString("utf-8");
}
function runGhJSON<T>(args: readonly string[]): T {
return JSON.parse(runGh(args));
}
type WorkflowRunListItem = {
databaseId: number;
number: number;
};
async function replaceAsync(
str: string,
regex: RegExp,
replacer: (substring: string, ...args: any[]) => Promise<string>,
) {
const promises: Array<Promise<string>> = [];
str.replace(regex, (match, ...args) => {
const promise = replacer(match, ...args);
promises.push(promise);
return match;
});
const data = await Promise.all(promises);
return str.replace(regex, () => data.shift() as string);
}

View File

@@ -56,15 +56,6 @@ export class Setting {
.getConfiguration(this.parent.qualifiedName)
.update(this.name, value, target);
}
inspect<T>(): InspectionResult<T> | undefined {
if (this.parent === undefined) {
throw new Error("Cannot update the value of a root setting.");
}
return workspace
.getConfiguration(this.parent.qualifiedName)
.inspect(this.name);
}
}
export interface InspectionResult<T> {

View File

@@ -9,6 +9,8 @@ import {
showAndLogInformationMessage,
isLikelyDatabaseRoot,
showAndLogExceptionWithTelemetry,
isFolderAlreadyInWorkspace,
showBinaryChoiceDialog,
} from "./helpers";
import { ProgressCallback, withProgress } from "./commandRunner";
import {
@@ -23,6 +25,7 @@ import { asError, getErrorMessage } from "./pure/helpers-pure";
import { QueryRunner } from "./queryRunner";
import { pathsEqual } from "./pure/files";
import { redactableError } from "./pure/errors";
import { isCodespacesTemplate } from "./config";
/**
* databases.ts
@@ -151,67 +154,69 @@ export async function findSourceArchive(
return undefined;
}
async function resolveDatabase(
databasePath: string,
): Promise<DatabaseContents> {
const name = basename(databasePath);
// Look for dataset and source archive.
const datasetUri = await findDataset(databasePath);
const sourceArchiveUri = await findSourceArchive(databasePath);
return {
kind: DatabaseKind.Database,
name,
datasetUri,
sourceArchiveUri,
};
}
/** Gets the relative paths of all `.dbscheme` files in the given directory. */
async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
return await glob("*.dbscheme", { cwd: dbDirectory });
}
async function resolveDatabaseContents(
uri: vscode.Uri,
): Promise<DatabaseContents> {
if (uri.scheme !== "file") {
throw new Error(
`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`,
);
}
const databasePath = uri.fsPath;
if (!(await pathExists(databasePath))) {
throw new InvalidDatabaseError(
`Database '${databasePath}' does not exist.`,
);
export class DatabaseResolver {
public static async resolveDatabaseContents(
uri: vscode.Uri,
): Promise<DatabaseContents> {
if (uri.scheme !== "file") {
throw new Error(
`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`,
);
}
const databasePath = uri.fsPath;
if (!(await pathExists(databasePath))) {
throw new InvalidDatabaseError(
`Database '${databasePath}' does not exist.`,
);
}
const contents = await this.resolveDatabase(databasePath);
if (contents === undefined) {
throw new InvalidDatabaseError(
`'${databasePath}' is not a valid database.`,
);
}
// Look for a single dbscheme file within the database.
// This should be found in the dataset directory, regardless of the form of database.
const dbPath = contents.datasetUri.fsPath;
const dbSchemeFiles = await getDbSchemeFiles(dbPath);
if (dbSchemeFiles.length === 0) {
throw new InvalidDatabaseError(
`Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`,
);
} else if (dbSchemeFiles.length > 1) {
throw new InvalidDatabaseError(
`Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`,
);
} else {
contents.dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
}
return contents;
}
const contents = await resolveDatabase(databasePath);
public static async resolveDatabase(
databasePath: string,
): Promise<DatabaseContents> {
const name = basename(databasePath);
if (contents === undefined) {
throw new InvalidDatabaseError(
`'${databasePath}' is not a valid database.`,
);
}
// Look for dataset and source archive.
const datasetUri = await findDataset(databasePath);
const sourceArchiveUri = await findSourceArchive(databasePath);
// Look for a single dbscheme file within the database.
// This should be found in the dataset directory, regardless of the form of database.
const dbPath = contents.datasetUri.fsPath;
const dbSchemeFiles = await getDbSchemeFiles(dbPath);
if (dbSchemeFiles.length === 0) {
throw new InvalidDatabaseError(
`Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`,
);
} else if (dbSchemeFiles.length > 1) {
throw new InvalidDatabaseError(
`Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`,
);
} else {
contents.dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
return {
kind: DatabaseKind.Database,
name,
datasetUri,
sourceArchiveUri,
};
}
return contents;
}
/** An item in the list of available databases */
@@ -367,7 +372,9 @@ export class DatabaseItemImpl implements DatabaseItem {
public async refresh(): Promise<void> {
try {
try {
this._contents = await resolveDatabaseContents(this.databaseUri);
this._contents = await DatabaseResolver.resolveDatabaseContents(
this.databaseUri,
);
this._error = undefined;
} catch (e) {
this._contents = undefined;
@@ -599,7 +606,7 @@ export class DatabaseManager extends DisposableObject {
uri: vscode.Uri,
displayName?: string,
): Promise<DatabaseItem> {
const contents = await resolveDatabaseContents(uri);
const contents = await DatabaseResolver.resolveDatabaseContents(uri);
// Ignore the source archive for QLTest databases by default.
const isQLTestDatabase = extname(uri.fsPath) === ".testproj";
const fullOptions: FullDatabaseOptions = {
@@ -621,9 +628,38 @@ export class DatabaseManager extends DisposableObject {
await this.addDatabaseItem(progress, token, databaseItem);
await this.addDatabaseSourceArchiveFolder(databaseItem);
if (isCodespacesTemplate()) {
await this.createSkeletonPacks(databaseItem);
}
return databaseItem;
}
public async createSkeletonPacks(databaseItem: DatabaseItem) {
if (databaseItem === undefined) {
void this.logger.log(
"Could not create QL pack because no database is selected. Please add a database.",
);
return;
}
if (databaseItem.language === "") {
void this.logger.log(
"Could not create skeleton QL pack because the selected database's language is not set.",
);
return;
}
const folderName = `codeql-custom-queries-${databaseItem.language}`;
if (isFolderAlreadyInWorkspace(folderName)) {
return;
}
await showBinaryChoiceDialog(
`We've noticed you don't have a QL pack downloaded to analyze this database. Can we set up a ${databaseItem.language} query pack for you`,
);
}
private async reregisterDatabases(
progress: ProgressCallback,
token: vscode.CancellationToken,

View File

@@ -255,6 +255,15 @@ export function getOnDiskWorkspaceFolders() {
return diskWorkspaceFolders;
}
/** Check if folder is already present in workspace */
export function isFolderAlreadyInWorkspace(folderName: string) {
const workspaceFolders = workspace.workspaceFolders || [];
return !!workspaceFolders.find(
(workspaceFolder) => workspaceFolder.name === folderName,
);
}
/**
* Provides a utility method to invoke a function only if a minimum time interval has elapsed since
* the last invocation of that function.

View File

@@ -32,7 +32,7 @@ export const RepositoriesSort = ({ value, onChange, className }: Props) => {
<VSCodeOption value={SortKey.Name}>Name</VSCodeOption>
<VSCodeOption value={SortKey.ResultsCount}>Results</VSCodeOption>
<VSCodeOption value={SortKey.Stars}>Stars</VSCodeOption>
<VSCodeOption value={SortKey.LastUpdated}>Last commit</VSCodeOption>
<VSCodeOption value={SortKey.LastUpdated}>Last updated</VSCodeOption>
</Dropdown>
);
};

View File

@@ -1,5 +1,5 @@
[
"v2.12.1",
"v2.12.2",
"v2.11.6",
"v2.7.6",
"v2.8.5",

View File

@@ -8,23 +8,20 @@ import { dirname } from "path";
import fetch from "node-fetch";
import { DB_URL, dbLoc, setStoragePath, storagePath } from "./global.helper";
import * as tmp from "tmp";
import { getTestSetting } from "../test-config";
import { CUSTOM_CODEQL_PATH_SETTING } from "../../../src/config";
import { extensions, workspace } from "vscode";
import baseJestSetup from "../jest.setup";
export default baseJestSetup;
import { ConfigurationTarget, env, extensions, workspace } from "vscode";
import { beforeEachAction } from "../test-config";
// create an extension storage location
let removeStorage: tmp.DirResult["removeCallback"] | undefined;
beforeAll(async () => {
// Set the CLI version here before activation to ensure we don't accidentally try to download a cli
await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setInitialTestValue(
await beforeEachAction();
await CUSTOM_CODEQL_PATH_SETTING.updateValue(
process.env.CLI_PATH,
ConfigurationTarget.Workspace,
);
await getTestSetting(CUSTOM_CODEQL_PATH_SETTING)?.setup();
// ensure the test database is downloaded
mkdirpSync(dirname(dbLoc));
@@ -78,6 +75,17 @@ beforeAll(async () => {
await extensions.getExtension("GitHub.vscode-codeql")?.activate();
});
beforeEach(async () => {
jest.spyOn(env, "openExternal").mockResolvedValue(false);
await beforeEachAction();
await CUSTOM_CODEQL_PATH_SETTING.updateValue(
process.env.CLI_PATH,
ConfigurationTarget.Workspace,
);
});
// ensure extension is cleaned up.
afterAll(async () => {
// ensure temp directory is cleaned up.

View File

@@ -3,6 +3,7 @@ import { resolve } from "path";
import {
authentication,
commands,
ConfigurationTarget,
extensions,
QuickPickItem,
TextDocument,
@@ -12,7 +13,10 @@ import {
import { CodeQLExtensionInterface } from "../../../../src/extension";
import { MockGitHubApiServer } from "../../../../src/mocks/mock-gh-api-server";
import { mockConfiguration } from "../../utils/configuration-helpers";
import {
CANARY_FEATURES,
setRemoteControllerRepo,
} from "../../../../src/config";
jest.setTimeout(30_000);
@@ -36,17 +40,8 @@ describe("Variant Analysis Submission Integration", () => {
let showErrorMessageSpy: jest.SpiedFunction<typeof window.showErrorMessage>;
beforeEach(async () => {
mockConfiguration({
values: {
codeQL: {
canary: true,
},
"codeQL.variantAnalysis": {
liveResults: true,
controllerRepo: "github/vscode-codeql",
},
},
});
await CANARY_FEATURES.updateValue(true, ConfigurationTarget.Global);
await setRemoteControllerRepo("github/vscode-codeql");
jest.spyOn(authentication, "getSession").mockResolvedValue({
id: "test",

View File

@@ -1,10 +1,8 @@
import { env } from "vscode";
import { jestTestConfigHelper } from "./test-config";
import { beforeEachAction } from "./test-config";
(env as any).openExternal = () => {
/**/
};
beforeEach(async () => {
jest.spyOn(env, "openExternal").mockResolvedValue(false);
export default async function setupEnv() {
await jestTestConfigHelper();
}
await beforeEachAction();
});

View File

@@ -6,8 +6,13 @@ import {
QueryHistoryConfigListener,
QueryServerConfigListener,
} from "../../../src/config";
import { vscodeGetConfigurationMock } from "../test-config";
describe("config listeners", () => {
beforeEach(() => {
vscodeGetConfigurationMock.mockRestore();
});
interface TestConfig<T> {
clazz: new () => ConfigListener;
settings: Array<{
@@ -108,7 +113,7 @@ describe("config listeners", () => {
await wait();
const newValue = listener[setting.property as keyof typeof listener];
expect(newValue).toEqual(setting.values[1]);
expect(onDidChangeConfiguration).toHaveBeenCalledTimes(1);
expect(onDidChangeConfiguration).toHaveBeenCalled();
});
});
});

View File

@@ -10,6 +10,7 @@ import {
DatabaseContents,
FullDatabaseOptions,
findSourceArchive,
DatabaseResolver,
} from "../../../src/databases";
import { Logger } from "../../../src/common";
import { ProgressCallback } from "../../../src/commandRunner";
@@ -20,6 +21,8 @@ import {
} from "../../../src/archive-filesystem-provider";
import { testDisposeHandler } from "../test-dispose-handler";
import { QueryRunner } from "../../../src/queryRunner";
import * as helpers from "../../../src/helpers";
import { Setting } from "../../../src/config";
describe("databases", () => {
const MOCK_DB_OPTIONS: FullDatabaseOptions = {
@@ -34,6 +37,11 @@ describe("databases", () => {
let registerSpy: jest.Mock<Promise<void>, []>;
let deregisterSpy: jest.Mock<Promise<void>, []>;
let resolveDatabaseSpy: jest.Mock<Promise<DbInfo>, []>;
let logSpy: jest.Mock<any, []>;
let showBinaryChoiceDialogSpy: jest.SpiedFunction<
typeof helpers.showBinaryChoiceDialog
>;
let dir: tmp.DirResult;
@@ -44,6 +52,13 @@ describe("databases", () => {
registerSpy = jest.fn(() => Promise.resolve(undefined));
deregisterSpy = jest.fn(() => Promise.resolve(undefined));
resolveDatabaseSpy = jest.fn(() => Promise.resolve({} as DbInfo));
logSpy = jest.fn(() => {
/* */
});
showBinaryChoiceDialogSpy = jest
.spyOn(helpers, "showBinaryChoiceDialog")
.mockResolvedValue(true);
databaseManager = new DatabaseManager(
{
@@ -66,9 +81,7 @@ describe("databases", () => {
resolveDatabase: resolveDatabaseSpy,
} as unknown as CodeQLCliServer,
{
log: () => {
/**/
},
log: logSpy,
} as unknown as Logger,
);
@@ -122,29 +135,31 @@ describe("databases", () => {
});
});
it("should rename a db item and emit an event", async () => {
const mockDbItem = createMockDB();
const onDidChangeDatabaseItem = jest.fn();
databaseManager.onDidChangeDatabaseItem(onDidChangeDatabaseItem);
await (databaseManager as any).addDatabaseItem(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem,
);
describe("renameDatabaseItem", () => {
it("should rename a db item and emit an event", async () => {
const mockDbItem = createMockDB();
const onDidChangeDatabaseItem = jest.fn();
databaseManager.onDidChangeDatabaseItem(onDidChangeDatabaseItem);
await (databaseManager as any).addDatabaseItem(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem,
);
await databaseManager.renameDatabaseItem(mockDbItem, "new name");
await databaseManager.renameDatabaseItem(mockDbItem, "new name");
expect(mockDbItem.name).toBe("new name");
expect(updateSpy).toBeCalledWith("databaseList", [
{
options: { ...MOCK_DB_OPTIONS, displayName: "new name" },
uri: dbLocationUri().toString(true),
},
]);
expect(mockDbItem.name).toBe("new name");
expect(updateSpy).toBeCalledWith("databaseList", [
{
options: { ...MOCK_DB_OPTIONS, displayName: "new name" },
uri: dbLocationUri().toString(true),
},
]);
expect(onDidChangeDatabaseItem).toBeCalledWith({
item: undefined,
kind: DatabaseEventKind.Rename,
expect(onDidChangeDatabaseItem).toBeCalledWith({
item: undefined,
kind: DatabaseEventKind.Rename,
});
});
});
@@ -287,7 +302,10 @@ describe("databases", () => {
describe("resolveSourceFile", () => {
it("should fail to resolve when not a uri", () => {
const db = createMockDB(Uri.parse("file:/sourceArchive-uri/"));
const db = createMockDB(
MOCK_DB_OPTIONS,
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
expect(() => db.resolveSourceFile("abc")).toThrowError(
"Scheme is missing",
@@ -295,7 +313,10 @@ describe("databases", () => {
});
it("should fail to resolve when not a file uri", () => {
const db = createMockDB(Uri.parse("file:/sourceArchive-uri/"));
const db = createMockDB(
MOCK_DB_OPTIONS,
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
expect(() => db.resolveSourceFile("http://abc")).toThrowError(
"Invalid uri scheme",
@@ -304,14 +325,20 @@ describe("databases", () => {
describe("no source archive", () => {
it("should resolve undefined", () => {
const db = createMockDB(Uri.parse("file:/sourceArchive-uri/"));
const db = createMockDB(
MOCK_DB_OPTIONS,
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
const resolved = db.resolveSourceFile(undefined);
expect(resolved.toString(true)).toBe(dbLocationUri().toString(true));
});
it("should resolve an empty file", () => {
const db = createMockDB(Uri.parse("file:/sourceArchive-uri/"));
const db = createMockDB(
MOCK_DB_OPTIONS,
Uri.parse("file:/sourceArchive-uri/"),
);
(db as any)._contents.sourceArchiveUri = undefined;
const resolved = db.resolveSourceFile("file:");
expect(resolved.toString()).toBe("file:///");
@@ -321,6 +348,7 @@ describe("databases", () => {
describe("zipped source archive", () => {
it("should encode a source archive url", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
encodeSourceArchiveUri({
sourceArchiveZipPath: "sourceArchive-uri",
pathWithinSourceArchive: "def",
@@ -340,6 +368,7 @@ describe("databases", () => {
it("should encode a source archive url with trailing slash", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
encodeSourceArchiveUri({
sourceArchiveZipPath: "sourceArchive-uri",
pathWithinSourceArchive: "def/",
@@ -359,6 +388,7 @@ describe("databases", () => {
it("should encode an empty source archive url", () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
encodeSourceArchiveUri({
sourceArchiveZipPath: "sourceArchive-uri",
pathWithinSourceArchive: "def",
@@ -372,26 +402,35 @@ describe("databases", () => {
});
it("should handle an empty file", () => {
const db = createMockDB(Uri.parse("file:/sourceArchive-uri/"));
const db = createMockDB(
MOCK_DB_OPTIONS,
Uri.parse("file:/sourceArchive-uri/"),
);
const resolved = db.resolveSourceFile("");
expect(resolved.toString()).toBe("file:///sourceArchive-uri/");
});
});
it("should get the primary language", async () => {
resolveDatabaseSpy.mockResolvedValue({
languages: ["python"],
} as unknown as DbInfo);
const result = await (databaseManager as any).getPrimaryLanguage("hucairz");
expect(result).toBe("python");
});
describe("getPrimaryLanguage", () => {
it("should get the primary language", async () => {
resolveDatabaseSpy.mockResolvedValue({
languages: ["python"],
} as unknown as DbInfo);
const result = await (databaseManager as any).getPrimaryLanguage(
"hucairz",
);
expect(result).toBe("python");
});
it("should handle missing the primary language", async () => {
resolveDatabaseSpy.mockResolvedValue({
languages: [],
} as unknown as DbInfo);
const result = await (databaseManager as any).getPrimaryLanguage("hucairz");
expect(result).toBe("");
it("should handle missing the primary language", async () => {
resolveDatabaseSpy.mockResolvedValue({
languages: [],
} as unknown as DbInfo);
const result = await (databaseManager as any).getPrimaryLanguage(
"hucairz",
);
expect(result).toBe("");
});
});
describe("isAffectedByTest", () => {
@@ -409,12 +448,17 @@ describe("databases", () => {
});
it("should return true for testproj database in test directory", async () => {
const db = createMockDB(sourceLocationUri(), Uri.file(projectPath));
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(directoryPath)).toBe(true);
});
it("should return false for non-existent test directory", async () => {
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(join(dir.name, "non-existent/non-existent.testproj")),
);
@@ -428,6 +472,7 @@ describe("databases", () => {
await fs.writeFile(anotherProjectPath, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(anotherProjectPath),
);
@@ -441,6 +486,7 @@ describe("databases", () => {
await fs.writeFile(anotherProjectPath, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(anotherProjectPath),
);
@@ -448,20 +494,32 @@ describe("databases", () => {
});
it("should return false for testproj database for prefix directory", async () => {
const db = createMockDB(sourceLocationUri(), Uri.file(projectPath));
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(projectPath),
);
// /d is a prefix of /dir/dir.testproj, but
// /dir/dir.testproj is not under /d
expect(await db.isAffectedByTest(join(directoryPath, "d"))).toBe(false);
});
it("should return true for testproj database for test file", async () => {
const db = createMockDB(sourceLocationUri(), Uri.file(projectPath));
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(qlFilePath)).toBe(true);
});
it("should return false for non-existent test file", async () => {
const otherTestFile = join(directoryPath, "other-test.ql");
const db = createMockDB(sourceLocationUri(), Uri.file(projectPath));
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(otherTestFile)).toBe(false);
});
@@ -470,6 +528,7 @@ describe("databases", () => {
await fs.writeFile(anotherProjectPath, "");
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(anotherProjectPath),
);
@@ -480,7 +539,11 @@ describe("databases", () => {
const otherTestFile = join(dir.name, "test.ql");
await fs.writeFile(otherTestFile, "");
const db = createMockDB(sourceLocationUri(), Uri.file(projectPath));
const db = createMockDB(
MOCK_DB_OPTIONS,
sourceLocationUri(),
Uri.file(projectPath),
);
expect(await db.isAffectedByTest(otherTestFile)).toBe(false);
});
});
@@ -524,7 +587,138 @@ describe("databases", () => {
});
});
describe("createSkeletonPacks", () => {
let mockDbItem: DatabaseItemImpl;
describe("when the language is set", () => {
it("should offer the user to set up a skeleton QL pack", async () => {
const options: FullDatabaseOptions = {
dateAdded: 123,
ignoreSourceArchive: false,
language: "ruby",
};
mockDbItem = createMockDB(options);
await (databaseManager as any).createSkeletonPacks(mockDbItem);
expect(showBinaryChoiceDialogSpy).toBeCalledTimes(1);
});
});
describe("when the language is not set", () => {
it("should fail gracefully", async () => {
mockDbItem = createMockDB();
await (databaseManager as any).createSkeletonPacks(mockDbItem);
expect(logSpy).toHaveBeenCalledWith(
"Could not create skeleton QL pack because the selected database's language is not set.",
);
});
});
describe("when the databaseItem is not set", () => {
it("should fail gracefully", async () => {
await (databaseManager as any).createSkeletonPacks(undefined);
expect(logSpy).toHaveBeenCalledWith(
"Could not create QL pack because no database is selected. Please add a database.",
);
});
});
});
describe("openDatabase", () => {
let createSkeletonPacksSpy: jest.SpyInstance;
let resolveDatabaseContentsSpy: jest.SpyInstance;
let addDatabaseSourceArchiveFolderSpy: jest.SpyInstance;
let mockDbItem: DatabaseItemImpl;
beforeEach(() => {
createSkeletonPacksSpy = jest
.spyOn(databaseManager, "createSkeletonPacks")
.mockImplementation(async () => {
/* no-op */
});
resolveDatabaseContentsSpy = jest
.spyOn(DatabaseResolver, "resolveDatabaseContents")
.mockResolvedValue({} as DatabaseContents);
addDatabaseSourceArchiveFolderSpy = jest.spyOn(
databaseManager,
"addDatabaseSourceArchiveFolder",
);
jest.mock("fs", () => ({
promises: {
pathExists: jest.fn().mockResolvedValue(true),
},
}));
mockDbItem = createMockDB();
});
it("should resolve the database contents", async () => {
await databaseManager.openDatabase(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem.databaseUri,
);
expect(resolveDatabaseContentsSpy).toBeCalledTimes(1);
});
it("should add database source archive folder", async () => {
await databaseManager.openDatabase(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem.databaseUri,
);
expect(addDatabaseSourceArchiveFolderSpy).toBeCalledTimes(1);
});
describe("when codeQL.codespacesTemplate is set to true", () => {
it("should create a skeleton QL pack", async () => {
jest.spyOn(Setting.prototype, "getValue").mockReturnValue(true);
await databaseManager.openDatabase(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem.databaseUri,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(1);
});
});
describe("when codeQL.codespacesTemplate is set to false", () => {
it("should not create a skeleton QL pack", async () => {
jest.spyOn(Setting.prototype, "getValue").mockReturnValue(false);
await databaseManager.openDatabase(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem.databaseUri,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(0);
});
});
describe("when codeQL.codespacesTemplate is not set", () => {
it("should not create a skeleton QL pack", async () => {
jest.spyOn(Setting.prototype, "getValue").mockReturnValue(undefined);
await databaseManager.openDatabase(
{} as ProgressCallback,
{} as CancellationToken,
mockDbItem.databaseUri,
);
expect(createSkeletonPacksSpy).toBeCalledTimes(0);
});
});
});
function createMockDB(
mockDbOptions = MOCK_DB_OPTIONS,
// source archive location must be a real(-ish) location since
// tests will add this to the workspace location
sourceArchiveUri = sourceLocationUri(),
@@ -536,7 +730,7 @@ describe("databases", () => {
sourceArchiveUri,
datasetUri: databaseUri,
} as DatabaseContents,
MOCK_DB_OPTIONS,
mockDbOptions,
() => void 0,
);
}

View File

@@ -13,7 +13,7 @@ import { DbTreeViewItem } from "../../../../src/databases/ui/db-tree-view-item";
import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
import { createMockExtensionContext } from "../../../factories/extension-context";
import { createDbConfig } from "../../../factories/db-config-factories";
import { mockConfiguration } from "../../utils/configuration-helpers";
import { setRemoteControllerRepo } from "../../../../src/config";
describe("db panel rendering nodes", () => {
const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -50,12 +50,8 @@ describe("db panel rendering nodes", () => {
});
describe("when controller repo is not set", () => {
mockConfiguration({
values: {
"codeQL.variantAnalysis": {
controllerRepo: undefined,
},
},
beforeEach(async () => {
await setRemoteControllerRepo(undefined);
});
it("should not have any items", async () => {
@@ -81,14 +77,8 @@ describe("db panel rendering nodes", () => {
});
describe("when controller repo is set", () => {
beforeEach(() => {
mockConfiguration({
values: {
"codeQL.variantAnalysis": {
controllerRepo: "github/codeql",
},
},
});
beforeEach(async () => {
await setRemoteControllerRepo("github/codeql");
});
it("should render default remote nodes when the config is empty", async () => {

View File

@@ -9,7 +9,7 @@ import { DbTreeViewItem } from "../../../../src/databases/ui/db-tree-view-item";
import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
import { createMockExtensionContext } from "../../../factories/extension-context";
import { createDbConfig } from "../../../factories/db-config-factories";
import { mockConfiguration } from "../../utils/configuration-helpers";
import { setRemoteControllerRepo } from "../../../../src/config";
describe("db panel", () => {
const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -40,13 +40,7 @@ describe("db panel", () => {
beforeEach(async () => {
await ensureDir(workspaceStoragePath);
mockConfiguration({
values: {
"codeQL.variantAnalysis": {
controllerRepo: "github/codeql",
},
},
});
await setRemoteControllerRepo("github/codeql");
});
afterEach(async () => {

View File

@@ -15,7 +15,7 @@ import {
import { ExtensionApp } from "../../../../src/common/vscode/vscode-app";
import { createMockExtensionContext } from "../../../factories/extension-context";
import { createDbConfig } from "../../../factories/db-config-factories";
import { mockConfiguration } from "../../utils/configuration-helpers";
import { setRemoteControllerRepo } from "../../../../src/config";
describe("db panel selection", () => {
const workspaceStoragePath = join(__dirname, "test-workspace-storage");
@@ -46,13 +46,7 @@ describe("db panel selection", () => {
beforeEach(async () => {
await ensureDir(workspaceStoragePath);
mockConfiguration({
values: {
"codeQL.variantAnalysis": {
controllerRepo: "github/codeql",
},
},
});
await setRemoteControllerRepo("github/codeql");
});
afterEach(async () => {

View File

@@ -9,6 +9,8 @@ import {
SecretStorageChangeEvent,
Uri,
window,
workspace,
WorkspaceFolder,
} from "vscode";
import { dump } from "js-yaml";
import * as tmp from "tmp";
@@ -19,6 +21,7 @@ import { DirResult } from "tmp";
import {
getInitialQueryContents,
InvocationRateLimiter,
isFolderAlreadyInWorkspace,
isLikelyDatabaseRoot,
isLikelyDbLanguageFolder,
showBinaryChoiceDialog,
@@ -533,3 +536,21 @@ describe("walkDirectory", () => {
expect(files.sort()).toEqual([file1, file2, file3, file4, file5, file6]);
});
});
describe("isFolderAlreadyInWorkspace", () => {
beforeEach(() => {
const folders = [
{ name: "/first/path" },
{ name: "/second/path" },
] as WorkspaceFolder[];
jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValue(folders);
});
it("should return true if the folder is already in the workspace", () => {
expect(isFolderAlreadyInWorkspace("/first/path")).toBe(true);
});
it("should return false if the folder is not in the workspace", () => {
expect(isFolderAlreadyInWorkspace("/third/path")).toBe(false);
});
});

View File

@@ -13,6 +13,7 @@ import { UserCancellationException } from "../../../src/commandRunner";
import { ENABLE_TELEMETRY } from "../../../src/config";
import * as Config from "../../../src/config";
import { createMockExtensionContext } from "./index";
import { vscodeGetConfigurationMock } from "../test-config";
import { redactableError } from "../../../src/pure/errors";
// setting preferences can trigger lots of background activity
@@ -41,6 +42,8 @@ describe("telemetry reporting", () => {
>;
beforeEach(async () => {
vscodeGetConfigurationMock.mockRestore();
try {
// in case a previous test has accidentally activated this extension,
// need to disable it first.

View File

@@ -1,126 +1,225 @@
import {
ConfigurationScope,
ConfigurationTarget,
workspace,
WorkspaceConfiguration as VSCodeWorkspaceConfiguration,
} from "vscode";
import { readFileSync } from "fs-extra";
import { join } from "path";
import { ConfigurationTarget } from "vscode";
import { ALL_SETTINGS, InspectionResult, Setting } from "../../src/config";
class TestSetting<T> {
private initialSettingState: InspectionResult<T> | undefined;
function getIn(object: any, path: string): any {
const parts = path.split(".");
let current = object;
for (const part of parts) {
current = current[part];
if (current === undefined) {
return undefined;
}
}
return current;
}
constructor(
public readonly setting: Setting,
private initialTestValue: T | undefined = undefined,
) {}
function setIn(object: any, path: string, value: any): void {
const parts = path.split(".");
let current = object;
for (const part of parts.slice(0, -1)) {
if (current[part] === undefined) {
current[part] = {};
}
current = current[part];
}
current[parts[parts.length - 1]] = value;
}
public async get(): Promise<T | undefined> {
return this.setting.getValue();
interface WorkspaceConfiguration {
scope?: ConfigurationTarget;
get<T>(section: string | undefined, key: string): T | undefined;
has(section: string | undefined, key: string): boolean;
update(section: string | undefined, key: string, value: unknown): void;
}
class InMemoryConfiguration implements WorkspaceConfiguration {
private readonly values: Record<string, unknown> = {};
public constructor(public readonly scope: ConfigurationTarget) {}
public get<T>(section: string | undefined, key: string): T | undefined {
return getIn(this.values, this.getKey(section, key)) as T | undefined;
}
public async set(
value: T | undefined,
target: ConfigurationTarget = ConfigurationTarget.Global,
): Promise<void> {
await this.setting.updateValue(value, target);
public has(section: string | undefined, key: string): boolean {
return getIn(this.values, this.getKey(section, key)) !== undefined;
}
public async setInitialTestValue(value: T | undefined) {
this.initialTestValue = value;
public update(
section: string | undefined,
key: string,
value: unknown,
): void {
setIn(this.values, this.getKey(section, key), value);
}
public async initialSetup() {
this.initialSettingState = this.setting.inspect();
private getKey(section: string | undefined, key: string): string {
return section ? `${section}.${key}` : key;
}
}
// Unfortunately it's not well-documented how to check whether we can write to a workspace
// configuration. This is the best I could come up with. It only fails for initial test values
// which are not undefined.
if (this.initialSettingState?.workspaceValue !== undefined) {
await this.set(this.initialTestValue, ConfigurationTarget.Workspace);
}
if (this.initialSettingState?.workspaceFolderValue !== undefined) {
await this.set(
this.initialTestValue,
ConfigurationTarget.WorkspaceFolder,
);
}
class DefaultConfiguration implements WorkspaceConfiguration {
private readonly values: Record<string, unknown> = {};
await this.setup();
public constructor(configurations: Record<string, { default: unknown }>) {
for (const [section, config] of Object.entries(configurations)) {
setIn(this.values, section, config.default);
}
}
public async setup() {
await this.set(this.initialTestValue, ConfigurationTarget.Global);
public get<T>(section: string | undefined, key: string): T | undefined {
return getIn(this.values, this.getKey(section, key)) as T | undefined;
}
public async restoreToInitialValues() {
const state = this.setting.inspect();
public has(section: string | undefined, key: string): boolean {
return getIn(this.values, this.getKey(section, key)) !== undefined;
}
// We need to check the state of the setting before we restore it. This is less important for the global
// configuration target, but the workspace/workspace folder configuration might not even exist. If they
// don't exist, VSCode will error when trying to write the new value (even if that value is undefined).
if (state?.globalValue !== this.initialSettingState?.globalValue) {
await this.set(
this.initialSettingState?.globalValue,
ConfigurationTarget.Global,
);
public update(
_section: string | undefined,
_key: string,
_value: unknown,
): void {
throw new Error("Cannot update default configuration");
}
private getKey(section: string | undefined, key: string): string {
return section ? `${section}.${key}` : key;
}
}
class ChainedInMemoryConfiguration {
constructor(private readonly configurations: WorkspaceConfiguration[]) {}
public getConfiguration(target: ConfigurationTarget) {
const configuration = this.configurations.find(
(configuration) => configuration.scope === target,
);
if (configuration === undefined) {
throw new Error(`Unknown configuration target ${target}`);
}
if (state?.workspaceValue !== this.initialSettingState?.workspaceValue) {
await this.set(
this.initialSettingState?.workspaceValue,
ConfigurationTarget.Workspace,
);
}
if (
state?.workspaceFolderValue !==
this.initialSettingState?.workspaceFolderValue
) {
await this.set(
this.initialSettingState?.workspaceFolderValue,
ConfigurationTarget.WorkspaceFolder,
);
return configuration;
}
public get<T>(section: string | undefined, key: string): T | undefined {
for (const configuration of this.configurations) {
if (configuration.has(section, key)) {
return configuration.get(section, key);
}
}
return undefined;
}
public has(section: string | undefined, key: string): boolean {
return this.configurations.some((configuration) =>
configuration.has(section, key),
);
}
public update<T>(
section: string | undefined,
key: string,
value: T,
target: ConfigurationTarget,
): void {
const configuration = this.getConfiguration(target);
configuration.update(section, key, value);
}
}
// Public configuration keys are the ones defined in the package.json.
// These keys are documented in the settings page. Other keys are
// internal and not documented.
const PKG_CONFIGURATION: Record<string, any> =
(function initConfigurationKeys() {
// Note we are using synchronous file reads here. This is fine because
// we are in tests.
const pkg = JSON.parse(
readFileSync(join(__dirname, "../../package.json"), "utf-8"),
const packageConfiguration: Record<
string,
{
default: any | undefined;
}
> = (function initConfigurationKeys() {
// Note we are using synchronous file reads here. This is fine because
// we are in tests.
const pkg = JSON.parse(
readFileSync(join(__dirname, "../../package.json"), "utf-8"),
);
return pkg.contributes.configuration.properties;
})();
export const vsCodeGetConfiguration = workspace.getConfiguration;
export let vscodeGetConfigurationMock: jest.SpiedFunction<
typeof workspace.getConfiguration
>;
export const beforeEachAction = async () => {
const defaultConfiguration = new DefaultConfiguration(packageConfiguration);
const configuration = new ChainedInMemoryConfiguration([
new InMemoryConfiguration(ConfigurationTarget.WorkspaceFolder),
new InMemoryConfiguration(ConfigurationTarget.Workspace),
new InMemoryConfiguration(ConfigurationTarget.Global),
defaultConfiguration,
]);
vscodeGetConfigurationMock = jest
.spyOn(workspace, "getConfiguration")
.mockImplementation(
(
section?: string,
scope?: ConfigurationScope | null,
): VSCodeWorkspaceConfiguration => {
if (scope) {
throw new Error("Scope is not supported in tests");
}
return {
get(key: string, defaultValue?: unknown) {
return configuration.get(section, key) ?? defaultValue;
},
has(key: string) {
return configuration.has(section, key);
},
inspect(_key: string) {
throw new Error("inspect is not supported in tests");
},
async update(
key: string,
value: unknown,
configurationTarget?: ConfigurationTarget | boolean | null,
overrideInLanguage?: boolean,
) {
if (overrideInLanguage) {
throw new Error("overrideInLanguage is not supported in tests");
}
function getActualConfigurationTarget(): ConfigurationTarget {
if (
configurationTarget === undefined ||
configurationTarget === null
) {
return ConfigurationTarget.Global;
}
if (typeof configurationTarget === "boolean") {
return configurationTarget
? ConfigurationTarget.Workspace
: ConfigurationTarget.Global;
}
return configurationTarget;
}
const target = getActualConfigurationTarget();
configuration.update(section, key, value, target);
},
};
},
);
return pkg.contributes.configuration.properties;
})();
// The test settings are all settings in ALL_SETTINGS which don't have any children
// and are also not hidden settings like the codeQL.canary.
const TEST_SETTINGS = ALL_SETTINGS.filter(
(setting) =>
setting.qualifiedName in PKG_CONFIGURATION && !setting.hasChildren,
).map((setting) => new TestSetting(setting));
export const getTestSetting = (
setting: Setting,
): TestSetting<unknown> | undefined => {
return TEST_SETTINGS.find((testSetting) => testSetting.setting === setting);
};
export const jestTestConfigHelper = async () => {
// Read in all current settings
await Promise.all(TEST_SETTINGS.map((setting) => setting.initialSetup()));
beforeEach(async () => {
// Reset the settings to their initial values before each test
await Promise.all(TEST_SETTINGS.map((setting) => setting.setup()));
});
afterAll(async () => {
// Restore all settings to their default values after each test suite
// Only do this outside of CI since the sometimes hangs on CI.
if (process.env.CI !== "true") {
await Promise.all(
TEST_SETTINGS.map((setting) => setting.restoreToInitialValues()),
);
}
});
};

View File

@@ -1,53 +0,0 @@
import { workspace } from "vscode";
type MockConfigurationConfig = {
values: {
[section: string]: {
[scope: string]: any | (() => any);
};
};
};
export function mockConfiguration(config: MockConfigurationConfig) {
const originalGetConfiguration = workspace.getConfiguration;
jest
.spyOn(workspace, "getConfiguration")
.mockImplementation((section, scope) => {
const configuration = originalGetConfiguration(section, scope);
return {
get(key: string, defaultValue?: unknown) {
if (
section &&
config.values[section] &&
config.values[section][key]
) {
const value = config.values[section][key];
return typeof value === "function" ? value() : value;
}
return configuration.get(key, defaultValue);
},
has(key: string) {
return configuration.has(key);
},
inspect(key: string) {
return configuration.inspect(key);
},
update(
key: string,
value: unknown,
configurationTarget?: boolean,
overrideInLanguage?: boolean,
) {
return configuration.update(
key,
value,
configurationTarget,
overrideInLanguage,
);
},
};
});
}