mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Test tests

Browse Source
This commit is contained in:
Dave Bartolomeo
2023-04-13 17:00:15 -04:00
parent 397b5852c1
commit cb93c84611
4 changed files with 407 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
extensions/ql-vscode/src/test-manager.ts
View File

@@ -109,11 +109,6 @@ class WorkspaceFolderHandler extends DisposableObject {
* debugging of tests.
*/
export class TestManager extends TestManagerBase {
private readonly testController: TestController = tests.createTestController(
"codeql",
"Fancy CodeQL Tests",
);
/**
* Maps from each workspace folder being tracked to the `WorkspaceFolderHandler` responsible for
* tracking it.
@@ -127,6 +122,11 @@ export class TestManager extends TestManagerBase {
app: App,
private readonly testRunner: TestRunner,
private readonly cliServer: CodeQLCliServer,
// Having this as a parameter with a default value makes passing in a mock easier.
private readonly testController: TestController = tests.createTestController(
"codeql",
"Fancy CodeQL Tests",
),
) {
super(app);
@@ -245,8 +245,10 @@ export class TestManager extends TestManagerBase {
/**
* Run the tests specified by the `TestRunRequest` parameter.
*
* Public because this is used in unit tests.
*/
private async run(
public async run(
request: TestRunRequest,
token: CancellationToken,
): Promise<void> {

263
extensions/ql-vscode/test/vscode-tests/no-workspace/test-adapter.test.ts
View File

@@ -1,90 +1,46 @@
import { Uri, WorkspaceFolder } from "vscode";
import {
CancellationTokenSource,
Range,
TestItem,
TestItemCollection,
TestRun,
TestRunRequest,
Uri,
WorkspaceFolder,
tests,
} from "vscode";
import { QLTestAdapter } from "../../../src/test-adapter";
import { CodeQLCliServer } from "../../../src/cli";
import {
DatabaseItem,
DatabaseItemImpl,
DatabaseManager,
FullDatabaseOptions,
} from "../../../src/local-databases";
import { DatabaseManager } from "../../../src/local-databases";
import { mockedObject } from "../utils/mocking.helpers";
import { TestRunner } from "../../../src/test-runner";
import {
createMockCliServerForTestRun,
mockEmptyDatabaseManager,
mockTestsInfo,
} from "./test-runner-helpers";
import { TestManager } from "../../../src/test-manager";
import { createMockApp } from "../../__mocks__/appMock";
jest.mock("fs-extra", () => {
const original = jest.requireActual("fs-extra");
return {
...original,
access: jest.fn(),
};
});
type IdTestItemPair = [id: string, testItem: TestItem];
describe("test-adapter", () => {
let testRunner: TestRunner;
let adapter: QLTestAdapter;
let fakeDatabaseManager: DatabaseManager;
let fakeCliServer: CodeQLCliServer;
let currentDatabaseItem: DatabaseItem | undefined;
let databaseItems: DatabaseItem[] = [];
const openDatabaseSpy = jest.fn();
const removeDatabaseItemSpy = jest.fn();
const renameDatabaseItemSpy = jest.fn();
const setCurrentDatabaseItemSpy = jest.fn();
const runTestsSpy = jest.fn();
const resolveTestsSpy = jest.fn();
const resolveQlpacksSpy = jest.fn();
const preTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
(_) => {
/* no change event listener */
},
);
const postTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
(_) => {
/* no change event listener */
},
);
beforeEach(() => {
mockRunTests();
openDatabaseSpy.mockResolvedValue(postTestDatabaseItem);
removeDatabaseItemSpy.mockResolvedValue(undefined);
renameDatabaseItemSpy.mockResolvedValue(undefined);
setCurrentDatabaseItemSpy.mockResolvedValue(undefined);
resolveQlpacksSpy.mockResolvedValue({});
resolveTestsSpy.mockResolvedValue([]);
fakeDatabaseManager = mockedObject<DatabaseManager>(
{
openDatabase: openDatabaseSpy,
removeDatabaseItem: removeDatabaseItemSpy,
renameDatabaseItem: renameDatabaseItemSpy,
setCurrentDatabaseItem: setCurrentDatabaseItemSpy,
},
{
dynamicProperties: {
currentDatabaseItem: () => currentDatabaseItem,
databaseItems: () => databaseItems,
},
},
);
fakeDatabaseManager = mockEmptyDatabaseManager();
jest.spyOn(preTestDatabaseItem, "isAffectedByTest").mockResolvedValue(true);
fakeCliServer = mockedObject<CodeQLCliServer>({
runTests: runTestsSpy,
resolveQlpacks: resolveQlpacksSpy,
resolveTests: resolveTestsSpy,
});
const mockCli = createMockCliServerForTestRun();
fakeCliServer = mockCli.cliServer;
testRunner = new TestRunner(fakeDatabaseManager, fakeCliServer);
});
adapter = new QLTestAdapter(
it("legacy test adapter should run some tests", async () => {
const adapter = new QLTestAdapter(
mockedObject<WorkspaceFolder>({
name: "ABC",
uri: Uri.parse("file:/ab/c"),
@@ -92,121 +48,128 @@ describe("test-adapter", () => {
testRunner,
fakeCliServer,
);
});
it("should run some tests", async () => {
const listenerSpy = jest.fn();
adapter.testStates(listenerSpy);
const testsPath = Uri.parse("file:/ab/c").fsPath;
const dPath = Uri.parse("file:/ab/c/d.ql").fsPath;
const gPath = Uri.parse("file:/ab/c/e/f/g.ql").fsPath;
const hPath = Uri.parse("file:/ab/c/e/f/h.ql").fsPath;
await adapter.run([testsPath]);
await adapter.run([mockTestsInfo.testsPath]);
expect(listenerSpy).toBeCalledTimes(5);
expect(listenerSpy).toHaveBeenNthCalledWith(1, {
type: "started",
tests: [testsPath],
tests: [mockTestsInfo.testsPath],
});
expect(listenerSpy).toHaveBeenNthCalledWith(2, {
type: "test",
state: "passed",
test: dPath,
test: mockTestsInfo.dPath,
message: undefined,
decorations: [],
});
expect(listenerSpy).toHaveBeenNthCalledWith(3, {
type: "test",
state: "errored",
test: gPath,
message: `\ncompilation error: ${gPath}\nERROR: abc\n`,
test: mockTestsInfo.gPath,
message: `\ncompilation error: ${mockTestsInfo.gPath}\nERROR: abc\n`,
decorations: [{ line: 1, message: "abc" }],
});
expect(listenerSpy).toHaveBeenNthCalledWith(4, {
type: "test",
state: "failed",
test: hPath,
message: `\nfailed: ${hPath}\njkh\ntuv\n`,
test: mockTestsInfo.hPath,
message: `\nfailed: ${mockTestsInfo.hPath}\njkh\ntuv\n`,
decorations: [],
});
expect(listenerSpy).toHaveBeenNthCalledWith(5, { type: "finished" });
});
it("should reregister testproj databases around test run", async () => {
currentDatabaseItem = preTestDatabaseItem;
databaseItems = [preTestDatabaseItem];
await adapter.run(["/path/to/test/dir"]);
it("native test manager should run some tests", async () => {
const enqueuedSpy = jest.fn();
const passedSpy = jest.fn();
const erroredSpy = jest.fn();
const failedSpy = jest.fn();
const endSpy = jest.fn();
expect(removeDatabaseItemSpy.mock.invocationCallOrder[0]).toBeLessThan(
runTestsSpy.mock.invocationCallOrder[0],
const testController = tests.createTestController("codeql", "CodeQL Tests");
testController.createTestRun = jest.fn().mockImplementation(() =>
mockedObject<TestRun>({
enqueued: enqueuedSpy,
passed: passedSpy,
errored: erroredSpy,
failed: failedSpy,
end: endSpy,
}),
);
expect(openDatabaseSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
runTestsSpy.mock.invocationCallOrder[0],
);
expect(renameDatabaseItemSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
openDatabaseSpy.mock.invocationCallOrder[0],
);
expect(
setCurrentDatabaseItemSpy.mock.invocationCallOrder[0],
).toBeGreaterThan(openDatabaseSpy.mock.invocationCallOrder[0]);
expect(removeDatabaseItemSpy).toBeCalledTimes(1);
expect(removeDatabaseItemSpy).toBeCalledWith(
expect.anything(),
expect.anything(),
preTestDatabaseItem,
const testManager = new TestManager(
createMockApp({}),
testRunner,
fakeCliServer,
testController,
);
expect(openDatabaseSpy).toBeCalledTimes(1);
expect(openDatabaseSpy).toBeCalledWith(
expect.anything(),
expect.anything(),
preTestDatabaseItem.databaseUri,
);
const childItems: TestItem[] = [
{
children: { size: 0 } as TestItemCollection,
id: `test ${mockTestsInfo.dPath}`,
uri: Uri.file(mockTestsInfo.dPath),
} as TestItem,
{
children: { size: 0 } as TestItemCollection,
id: `test ${mockTestsInfo.gPath}`,
uri: Uri.file(mockTestsInfo.gPath),
} as TestItem,
{
children: { size: 0 } as TestItemCollection,
id: `test ${mockTestsInfo.hPath}`,
uri: Uri.file(mockTestsInfo.hPath),
} as TestItem,
];
const childElements: IdTestItemPair[] = childItems.map((childItem) => [
childItem.id,
childItem,
]);
const childIteratorFunc: () => Iterator<IdTestItemPair> = () =>
childElements[Symbol.iterator]();
expect(renameDatabaseItemSpy).toBeCalledTimes(1);
expect(renameDatabaseItemSpy).toBeCalledWith(
postTestDatabaseItem,
preTestDatabaseItem.name,
);
const rootItem = {
id: `dir ${mockTestsInfo.testsPath}`,
uri: Uri.file(mockTestsInfo.testsPath),
children: {
size: 3,
[Symbol.iterator]: childIteratorFunc,
} as TestItemCollection,
} as TestItem;
expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1);
expect(setCurrentDatabaseItemSpy).toBeCalledWith(
postTestDatabaseItem,
true,
);
});
const request = new TestRunRequest([rootItem]);
await testManager.run(request, new CancellationTokenSource().token);
function mockRunTests() {
// runTests is an async generator function. This is not directly supported in sinon
// However, we can pretend the same thing by just returning an async array.
runTestsSpy.mockReturnValue(
(async function* () {
yield Promise.resolve({
test: Uri.parse("file:/ab/c/d.ql").fsPath,
pass: true,
messages: [],
});
yield Promise.resolve({
test: Uri.parse("file:/ab/c/e/f/g.ql").fsPath,
pass: false,
diff: ["pqr", "xyz"],
// a compile error
failureStage: "COMPILATION",
messages: [
{ position: { line: 1 }, message: "abc", severity: "ERROR" },
expect(enqueuedSpy).toBeCalledTimes(3);
expect(passedSpy).toBeCalledTimes(1);
expect(passedSpy).toHaveBeenCalledWith(childItems[0], 3000);
expect(erroredSpy).toHaveBeenCalledTimes(1);
expect(erroredSpy).toHaveBeenCalledWith(
childItems[1],
[
{
location: {
range: new Range(0, 0, 1, 1),
uri: Uri.file(mockTestsInfo.gPath),
},
message: "abc",
},
],
});
yield Promise.resolve({
test: Uri.parse("file:/ab/c/e/f/h.ql").fsPath,
pass: false,
diff: ["jkh", "tuv"],
failureStage: "RESULT",
messages: [],
});
})(),
4000,
);
}
expect(failedSpy).toHaveBeenCalledWith(
childItems[2],
[
{
message: "Test failed",
},
],
11000,
);
expect(failedSpy).toBeCalledTimes(1);
expect(endSpy).toBeCalledTimes(1);
});
});

96
extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner-helpers.ts Normal file
View File

@@ -0,0 +1,96 @@
import { Uri } from "vscode";
import { mockedObject } from "../utils/mocking.helpers";
import { CodeQLCliServer } from "../../../src/cli";
import { DatabaseManager } from "../../../src/local-databases";
/**
* Fake QL tests used by various tests.
*/
export const mockTestsInfo = {
testsPath: Uri.parse("file:/ab/c").fsPath,
dPath: Uri.parse("file:/ab/c/d.ql").fsPath,
gPath: Uri.parse("file:/ab/c/e/f/g.ql").fsPath,
hPath: Uri.parse("file:/ab/c/e/f/h.ql").fsPath,
};
/**
* Create a mock of a `DatabaseManager` with no databases loaded.
*/
export function mockEmptyDatabaseManager(): DatabaseManager {
return mockedObject<DatabaseManager>({
currentDatabaseItem: undefined,
databaseItems: [],
});
}
/**
* Creates a `CodeQLCliServer` that "runs" the mock tests. Also returns the spy
* hook for the `runTests` function on the CLI server.
*/
export function createMockCliServerForTestRun() {
const resolveQlpacksSpy = jest.fn();
resolveQlpacksSpy.mockResolvedValue({});
const resolveTestsSpy = jest.fn();
resolveTestsSpy.mockResolvedValue([]);
const runTestsSpy = mockRunTests();
return {
cliServer: mockedObject<CodeQLCliServer>({
runTests: runTestsSpy,
resolveQlpacks: resolveQlpacksSpy,
resolveTests: resolveTestsSpy,
}),
runTestsSpy,
};
}
function mockRunTests(): jest.Mock<any, any> {
const runTestsSpy = jest.fn();
// runTests is an async generator function. This is not directly supported in sinon
// However, we can pretend the same thing by just returning an async array.
runTestsSpy.mockReturnValue(
(async function* () {
yield Promise.resolve({
test: mockTestsInfo.dPath,
pass: true,
messages: [],
compilationMs: 1000,
evaluationMs: 2000,
});
yield Promise.resolve({
test: mockTestsInfo.gPath,
pass: false,
diff: ["pqr", "xyz"],
// a compile error
failureStage: "COMPILATION",
compilationMs: 4000,
evaluationMs: 0,
messages: [
{
position: {
fileName: mockTestsInfo.gPath,
line: 1,
column: 1,
endLine: 2,
endColumn: 2,
},
message: "abc",
severity: "ERROR",
},
],
});
yield Promise.resolve({
test: mockTestsInfo.hPath,
pass: false,
diff: ["jkh", "tuv"],
failureStage: "RESULT",
compilationMs: 5000,
evaluationMs: 6000,
messages: [],
});
})(),
);
return runTestsSpy;
}

189
extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner.test.ts Normal file
View File

@@ -0,0 +1,189 @@
import { CancellationTokenSource, Uri } from "vscode";
import { CodeQLCliServer } from "../../../src/cli";
import {
DatabaseItem,
DatabaseItemImpl,
DatabaseManager,
FullDatabaseOptions,
} from "../../../src/local-databases";
import { mockedObject } from "../utils/mocking.helpers";
import { TestRunner } from "../../../src/test-runner";
import { createMockLogger } from "../../__mocks__/loggerMock";
import {
createMockCliServerForTestRun,
mockTestsInfo,
} from "./test-runner-helpers";
jest.mock("fs-extra", () => {
const original = jest.requireActual("fs-extra");
return {
...original,
access: jest.fn(),
};
});
describe("test-runner", () => {
let testRunner: TestRunner;
let fakeDatabaseManager: DatabaseManager;
let fakeCliServer: CodeQLCliServer;
let currentDatabaseItem: DatabaseItem | undefined;
let databaseItems: DatabaseItem[] = [];
const openDatabaseSpy = jest.fn();
const removeDatabaseItemSpy = jest.fn();
const renameDatabaseItemSpy = jest.fn();
const setCurrentDatabaseItemSpy = jest.fn();
let runTestsSpy: jest.Mock<any, any>;
const resolveTestsSpy = jest.fn();
const resolveQlpacksSpy = jest.fn();
const preTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
(_) => {
/* no change event listener */
},
);
const postTestDatabaseItem = new DatabaseItemImpl(
Uri.file("/path/to/test/dir/dir.testproj"),
undefined,
mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
(_) => {
/* no change event listener */
},
);
beforeEach(() => {
openDatabaseSpy.mockResolvedValue(postTestDatabaseItem);
removeDatabaseItemSpy.mockResolvedValue(undefined);
renameDatabaseItemSpy.mockResolvedValue(undefined);
setCurrentDatabaseItemSpy.mockResolvedValue(undefined);
resolveQlpacksSpy.mockResolvedValue({});
resolveTestsSpy.mockResolvedValue([]);
fakeDatabaseManager = mockedObject<DatabaseManager>(
{
openDatabase: openDatabaseSpy,
removeDatabaseItem: removeDatabaseItemSpy,
renameDatabaseItem: renameDatabaseItemSpy,
setCurrentDatabaseItem: setCurrentDatabaseItemSpy,
},
{
dynamicProperties: {
currentDatabaseItem: () => currentDatabaseItem,
databaseItems: () => databaseItems,
},
},
);
jest.spyOn(preTestDatabaseItem, "isAffectedByTest").mockResolvedValue(true);
const mockCli = createMockCliServerForTestRun();
fakeCliServer = mockCli.cliServer;
runTestsSpy = mockCli.runTestsSpy;
testRunner = new TestRunner(fakeDatabaseManager, fakeCliServer);
});
it("should run some tests", async () => {
const eventHandlerSpy = jest.fn();
await testRunner.run(
[mockTestsInfo.dPath, mockTestsInfo.gPath, mockTestsInfo.hPath],
createMockLogger(),
new CancellationTokenSource().token,
eventHandlerSpy,
);
expect(eventHandlerSpy).toBeCalledTimes(3);
expect(eventHandlerSpy).toHaveBeenNthCalledWith(1, {
test: mockTestsInfo.dPath,
pass: true,
compilationMs: 1000,
evaluationMs: 2000,
messages: [],
});
expect(eventHandlerSpy).toHaveBeenNthCalledWith(2, {
test: mockTestsInfo.gPath,
pass: false,
compilationMs: 4000,
evaluationMs: 0,
diff: ["pqr", "xyz"],
failureStage: "COMPILATION",
messages: [
{
message: "abc",
position: {
line: 1,
column: 1,
endLine: 2,
endColumn: 2,
fileName: mockTestsInfo.gPath,
},
severity: "ERROR",
},
],
});
expect(eventHandlerSpy).toHaveBeenNthCalledWith(3, {
test: mockTestsInfo.hPath,
pass: false,
compilationMs: 5000,
evaluationMs: 6000,
diff: ["jkh", "tuv"],
failureStage: "RESULT",
messages: [],
});
});
it("should reregister testproj databases around test run", async () => {
currentDatabaseItem = preTestDatabaseItem;
databaseItems = [preTestDatabaseItem];
await testRunner.run(
["/path/to/test/dir"],
createMockLogger(),
new CancellationTokenSource().token,
async () => {
/***/
},
);
expect(removeDatabaseItemSpy.mock.invocationCallOrder[0]).toBeLessThan(
runTestsSpy.mock.invocationCallOrder[0],
);
expect(openDatabaseSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
runTestsSpy.mock.invocationCallOrder[0],
);
expect(renameDatabaseItemSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
openDatabaseSpy.mock.invocationCallOrder[0],
);
expect(
setCurrentDatabaseItemSpy.mock.invocationCallOrder[0],
).toBeGreaterThan(openDatabaseSpy.mock.invocationCallOrder[0]);
expect(removeDatabaseItemSpy).toBeCalledTimes(1);
expect(removeDatabaseItemSpy).toBeCalledWith(
expect.anything(),
expect.anything(),
preTestDatabaseItem,
);
expect(openDatabaseSpy).toBeCalledTimes(1);
expect(openDatabaseSpy).toBeCalledWith(
expect.anything(),
expect.anything(),
preTestDatabaseItem.databaseUri,
);
expect(renameDatabaseItemSpy).toBeCalledTimes(1);
expect(renameDatabaseItemSpy).toBeCalledWith(
postTestDatabaseItem,
preTestDatabaseItem.name,
);
expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1);
expect(setCurrentDatabaseItemSpy).toBeCalledWith(
postTestDatabaseItem,
true,
);
});
});