From cb93c84611d7d746b3dec6d8f22c53822b7f0c1a Mon Sep 17 00:00:00 2001 From: Dave Bartolomeo Date: Thu, 13 Apr 2023 17:00:15 -0400 Subject: [PATCH] Test tests --- extensions/ql-vscode/src/test-manager.ts | 14 +- .../no-workspace/test-adapter.test.ts | 265 ++++++++---------- .../no-workspace/test-runner-helpers.ts | 96 +++++++ .../no-workspace/test-runner.test.ts | 189 +++++++++++++ 4 files changed, 407 insertions(+), 157 deletions(-) create mode 100644 extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner-helpers.ts create mode 100644 extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner.test.ts diff --git a/extensions/ql-vscode/src/test-manager.ts b/extensions/ql-vscode/src/test-manager.ts index 70f217503..e83577468 100644 --- a/extensions/ql-vscode/src/test-manager.ts +++ b/extensions/ql-vscode/src/test-manager.ts @@ -109,11 +109,6 @@ class WorkspaceFolderHandler extends DisposableObject { * debugging of tests. */ export class TestManager extends TestManagerBase { - private readonly testController: TestController = tests.createTestController( - "codeql", - "Fancy CodeQL Tests", - ); - /** * Maps from each workspace folder being tracked to the `WorkspaceFolderHandler` responsible for * tracking it. @@ -127,6 +122,11 @@ export class TestManager extends TestManagerBase { app: App, private readonly testRunner: TestRunner, private readonly cliServer: CodeQLCliServer, + // Having this as a parameter with a default value makes passing in a mock easier. + private readonly testController: TestController = tests.createTestController( + "codeql", + "Fancy CodeQL Tests", + ), ) { super(app); @@ -245,8 +245,10 @@ export class TestManager extends TestManagerBase { /** * Run the tests specified by the `TestRunRequest` parameter. + * + * Public because this is used in unit tests. */ - private async run( + public async run( request: TestRunRequest, token: CancellationToken, ): Promise { diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/test-adapter.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/test-adapter.test.ts index 42c26815e..3bc9f6120 100644 --- a/extensions/ql-vscode/test/vscode-tests/no-workspace/test-adapter.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/test-adapter.test.ts @@ -1,90 +1,46 @@ -import { Uri, WorkspaceFolder } from "vscode"; +import { + CancellationTokenSource, + Range, + TestItem, + TestItemCollection, + TestRun, + TestRunRequest, + Uri, + WorkspaceFolder, + tests, +} from "vscode"; import { QLTestAdapter } from "../../../src/test-adapter"; import { CodeQLCliServer } from "../../../src/cli"; -import { - DatabaseItem, - DatabaseItemImpl, - DatabaseManager, - FullDatabaseOptions, -} from "../../../src/local-databases"; +import { DatabaseManager } from "../../../src/local-databases"; import { mockedObject } from "../utils/mocking.helpers"; import { TestRunner } from "../../../src/test-runner"; +import { + createMockCliServerForTestRun, + mockEmptyDatabaseManager, + mockTestsInfo, +} from "./test-runner-helpers"; +import { TestManager } from "../../../src/test-manager"; +import { createMockApp } from "../../__mocks__/appMock"; -jest.mock("fs-extra", () => { - const original = jest.requireActual("fs-extra"); - return { - ...original, - access: jest.fn(), - }; -}); +type IdTestItemPair = [id: string, testItem: TestItem]; describe("test-adapter", () => { let testRunner: TestRunner; - let adapter: QLTestAdapter; let fakeDatabaseManager: DatabaseManager; let fakeCliServer: CodeQLCliServer; - let currentDatabaseItem: DatabaseItem | undefined; - let databaseItems: DatabaseItem[] = []; - const openDatabaseSpy = jest.fn(); - const removeDatabaseItemSpy = jest.fn(); - const renameDatabaseItemSpy = jest.fn(); - const setCurrentDatabaseItemSpy = jest.fn(); - const runTestsSpy = jest.fn(); - const resolveTestsSpy = jest.fn(); - const resolveQlpacksSpy = jest.fn(); - - const preTestDatabaseItem = new DatabaseItemImpl( - Uri.file("/path/to/test/dir/dir.testproj"), - undefined, - mockedObject({ displayName: "custom display name" }), - (_) => { - /* no change event listener */ - }, - ); - const postTestDatabaseItem = new DatabaseItemImpl( - Uri.file("/path/to/test/dir/dir.testproj"), - undefined, - mockedObject({ displayName: "default name" }), - (_) => { - /* no change event listener */ - }, - ); beforeEach(() => { - mockRunTests(); - openDatabaseSpy.mockResolvedValue(postTestDatabaseItem); - removeDatabaseItemSpy.mockResolvedValue(undefined); - renameDatabaseItemSpy.mockResolvedValue(undefined); - setCurrentDatabaseItemSpy.mockResolvedValue(undefined); - resolveQlpacksSpy.mockResolvedValue({}); - resolveTestsSpy.mockResolvedValue([]); - fakeDatabaseManager = mockedObject( - { - openDatabase: openDatabaseSpy, - removeDatabaseItem: removeDatabaseItemSpy, - renameDatabaseItem: renameDatabaseItemSpy, - setCurrentDatabaseItem: setCurrentDatabaseItemSpy, - }, - { - dynamicProperties: { - currentDatabaseItem: () => currentDatabaseItem, - databaseItems: () => databaseItems, - }, - }, - ); + fakeDatabaseManager = mockEmptyDatabaseManager(); - jest.spyOn(preTestDatabaseItem, "isAffectedByTest").mockResolvedValue(true); - - fakeCliServer = mockedObject({ - runTests: runTestsSpy, - resolveQlpacks: resolveQlpacksSpy, - resolveTests: resolveTestsSpy, - }); + const mockCli = createMockCliServerForTestRun(); + fakeCliServer = mockCli.cliServer; testRunner = new TestRunner(fakeDatabaseManager, fakeCliServer); + }); - adapter = new QLTestAdapter( + it("legacy test adapter should run some tests", async () => { + const adapter = new QLTestAdapter( mockedObject({ name: "ABC", uri: Uri.parse("file:/ab/c"), @@ -92,121 +48,128 @@ describe("test-adapter", () => { testRunner, fakeCliServer, ); - }); - it("should run some tests", async () => { const listenerSpy = jest.fn(); adapter.testStates(listenerSpy); - const testsPath = Uri.parse("file:/ab/c").fsPath; - const dPath = Uri.parse("file:/ab/c/d.ql").fsPath; - const gPath = Uri.parse("file:/ab/c/e/f/g.ql").fsPath; - const hPath = Uri.parse("file:/ab/c/e/f/h.ql").fsPath; - - await adapter.run([testsPath]); + await adapter.run([mockTestsInfo.testsPath]); expect(listenerSpy).toBeCalledTimes(5); expect(listenerSpy).toHaveBeenNthCalledWith(1, { type: "started", - tests: [testsPath], + tests: [mockTestsInfo.testsPath], }); expect(listenerSpy).toHaveBeenNthCalledWith(2, { type: "test", state: "passed", - test: dPath, + test: mockTestsInfo.dPath, message: undefined, decorations: [], }); expect(listenerSpy).toHaveBeenNthCalledWith(3, { type: "test", state: "errored", - test: gPath, - message: `\ncompilation error: ${gPath}\nERROR: abc\n`, + test: mockTestsInfo.gPath, + message: `\ncompilation error: ${mockTestsInfo.gPath}\nERROR: abc\n`, decorations: [{ line: 1, message: "abc" }], }); expect(listenerSpy).toHaveBeenNthCalledWith(4, { type: "test", state: "failed", - test: hPath, - message: `\nfailed: ${hPath}\njkh\ntuv\n`, + test: mockTestsInfo.hPath, + message: `\nfailed: ${mockTestsInfo.hPath}\njkh\ntuv\n`, decorations: [], }); expect(listenerSpy).toHaveBeenNthCalledWith(5, { type: "finished" }); }); - it("should reregister testproj databases around test run", async () => { - currentDatabaseItem = preTestDatabaseItem; - databaseItems = [preTestDatabaseItem]; - await adapter.run(["/path/to/test/dir"]); + it("native test manager should run some tests", async () => { + const enqueuedSpy = jest.fn(); + const passedSpy = jest.fn(); + const erroredSpy = jest.fn(); + const failedSpy = jest.fn(); + const endSpy = jest.fn(); - expect(removeDatabaseItemSpy.mock.invocationCallOrder[0]).toBeLessThan( - runTestsSpy.mock.invocationCallOrder[0], + const testController = tests.createTestController("codeql", "CodeQL Tests"); + testController.createTestRun = jest.fn().mockImplementation(() => + mockedObject({ + enqueued: enqueuedSpy, + passed: passedSpy, + errored: erroredSpy, + failed: failedSpy, + end: endSpy, + }), ); - expect(openDatabaseSpy.mock.invocationCallOrder[0]).toBeGreaterThan( - runTestsSpy.mock.invocationCallOrder[0], - ); - expect(renameDatabaseItemSpy.mock.invocationCallOrder[0]).toBeGreaterThan( - openDatabaseSpy.mock.invocationCallOrder[0], - ); - expect( - setCurrentDatabaseItemSpy.mock.invocationCallOrder[0], - ).toBeGreaterThan(openDatabaseSpy.mock.invocationCallOrder[0]); - - expect(removeDatabaseItemSpy).toBeCalledTimes(1); - expect(removeDatabaseItemSpy).toBeCalledWith( - expect.anything(), - expect.anything(), - preTestDatabaseItem, + const testManager = new TestManager( + createMockApp({}), + testRunner, + fakeCliServer, + testController, ); - expect(openDatabaseSpy).toBeCalledTimes(1); - expect(openDatabaseSpy).toBeCalledWith( - expect.anything(), - expect.anything(), - preTestDatabaseItem.databaseUri, - ); + const childItems: TestItem[] = [ + { + children: { size: 0 } as TestItemCollection, + id: `test ${mockTestsInfo.dPath}`, + uri: Uri.file(mockTestsInfo.dPath), + } as TestItem, + { + children: { size: 0 } as TestItemCollection, + id: `test ${mockTestsInfo.gPath}`, + uri: Uri.file(mockTestsInfo.gPath), + } as TestItem, + { + children: { size: 0 } as TestItemCollection, + id: `test ${mockTestsInfo.hPath}`, + uri: Uri.file(mockTestsInfo.hPath), + } as TestItem, + ]; + const childElements: IdTestItemPair[] = childItems.map((childItem) => [ + childItem.id, + childItem, + ]); + const childIteratorFunc: () => Iterator = () => + childElements[Symbol.iterator](); - expect(renameDatabaseItemSpy).toBeCalledTimes(1); - expect(renameDatabaseItemSpy).toBeCalledWith( - postTestDatabaseItem, - preTestDatabaseItem.name, - ); + const rootItem = { + id: `dir ${mockTestsInfo.testsPath}`, + uri: Uri.file(mockTestsInfo.testsPath), + children: { + size: 3, + [Symbol.iterator]: childIteratorFunc, + } as TestItemCollection, + } as TestItem; - expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1); - expect(setCurrentDatabaseItemSpy).toBeCalledWith( - postTestDatabaseItem, - true, + const request = new TestRunRequest([rootItem]); + await testManager.run(request, new CancellationTokenSource().token); + + expect(enqueuedSpy).toBeCalledTimes(3); + expect(passedSpy).toBeCalledTimes(1); + expect(passedSpy).toHaveBeenCalledWith(childItems[0], 3000); + expect(erroredSpy).toHaveBeenCalledTimes(1); + expect(erroredSpy).toHaveBeenCalledWith( + childItems[1], + [ + { + location: { + range: new Range(0, 0, 1, 1), + uri: Uri.file(mockTestsInfo.gPath), + }, + message: "abc", + }, + ], + 4000, ); + expect(failedSpy).toHaveBeenCalledWith( + childItems[2], + [ + { + message: "Test failed", + }, + ], + 11000, + ); + expect(failedSpy).toBeCalledTimes(1); + expect(endSpy).toBeCalledTimes(1); }); - - function mockRunTests() { - // runTests is an async generator function. This is not directly supported in sinon - // However, we can pretend the same thing by just returning an async array. - runTestsSpy.mockReturnValue( - (async function* () { - yield Promise.resolve({ - test: Uri.parse("file:/ab/c/d.ql").fsPath, - pass: true, - messages: [], - }); - yield Promise.resolve({ - test: Uri.parse("file:/ab/c/e/f/g.ql").fsPath, - pass: false, - diff: ["pqr", "xyz"], - // a compile error - failureStage: "COMPILATION", - messages: [ - { position: { line: 1 }, message: "abc", severity: "ERROR" }, - ], - }); - yield Promise.resolve({ - test: Uri.parse("file:/ab/c/e/f/h.ql").fsPath, - pass: false, - diff: ["jkh", "tuv"], - failureStage: "RESULT", - messages: [], - }); - })(), - ); - } }); diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner-helpers.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner-helpers.ts new file mode 100644 index 000000000..b41d45841 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner-helpers.ts @@ -0,0 +1,96 @@ +import { Uri } from "vscode"; +import { mockedObject } from "../utils/mocking.helpers"; +import { CodeQLCliServer } from "../../../src/cli"; +import { DatabaseManager } from "../../../src/local-databases"; + +/** + * Fake QL tests used by various tests. + */ +export const mockTestsInfo = { + testsPath: Uri.parse("file:/ab/c").fsPath, + dPath: Uri.parse("file:/ab/c/d.ql").fsPath, + gPath: Uri.parse("file:/ab/c/e/f/g.ql").fsPath, + hPath: Uri.parse("file:/ab/c/e/f/h.ql").fsPath, +}; + +/** + * Create a mock of a `DatabaseManager` with no databases loaded. + */ +export function mockEmptyDatabaseManager(): DatabaseManager { + return mockedObject({ + currentDatabaseItem: undefined, + databaseItems: [], + }); +} + +/** + * Creates a `CodeQLCliServer` that "runs" the mock tests. Also returns the spy + * hook for the `runTests` function on the CLI server. + */ +export function createMockCliServerForTestRun() { + const resolveQlpacksSpy = jest.fn(); + resolveQlpacksSpy.mockResolvedValue({}); + + const resolveTestsSpy = jest.fn(); + resolveTestsSpy.mockResolvedValue([]); + + const runTestsSpy = mockRunTests(); + return { + cliServer: mockedObject({ + runTests: runTestsSpy, + resolveQlpacks: resolveQlpacksSpy, + resolveTests: resolveTestsSpy, + }), + runTestsSpy, + }; +} + +function mockRunTests(): jest.Mock { + const runTestsSpy = jest.fn(); + // runTests is an async generator function. This is not directly supported in sinon + // However, we can pretend the same thing by just returning an async array. + runTestsSpy.mockReturnValue( + (async function* () { + yield Promise.resolve({ + test: mockTestsInfo.dPath, + pass: true, + messages: [], + compilationMs: 1000, + evaluationMs: 2000, + }); + yield Promise.resolve({ + test: mockTestsInfo.gPath, + pass: false, + diff: ["pqr", "xyz"], + // a compile error + failureStage: "COMPILATION", + compilationMs: 4000, + evaluationMs: 0, + messages: [ + { + position: { + fileName: mockTestsInfo.gPath, + line: 1, + column: 1, + endLine: 2, + endColumn: 2, + }, + message: "abc", + severity: "ERROR", + }, + ], + }); + yield Promise.resolve({ + test: mockTestsInfo.hPath, + pass: false, + diff: ["jkh", "tuv"], + failureStage: "RESULT", + compilationMs: 5000, + evaluationMs: 6000, + messages: [], + }); + })(), + ); + + return runTestsSpy; +} diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner.test.ts new file mode 100644 index 000000000..d1cd60a7d --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner.test.ts @@ -0,0 +1,189 @@ +import { CancellationTokenSource, Uri } from "vscode"; +import { CodeQLCliServer } from "../../../src/cli"; +import { + DatabaseItem, + DatabaseItemImpl, + DatabaseManager, + FullDatabaseOptions, +} from "../../../src/local-databases"; +import { mockedObject } from "../utils/mocking.helpers"; +import { TestRunner } from "../../../src/test-runner"; +import { createMockLogger } from "../../__mocks__/loggerMock"; +import { + createMockCliServerForTestRun, + mockTestsInfo, +} from "./test-runner-helpers"; + +jest.mock("fs-extra", () => { + const original = jest.requireActual("fs-extra"); + return { + ...original, + access: jest.fn(), + }; +}); + +describe("test-runner", () => { + let testRunner: TestRunner; + let fakeDatabaseManager: DatabaseManager; + let fakeCliServer: CodeQLCliServer; + let currentDatabaseItem: DatabaseItem | undefined; + let databaseItems: DatabaseItem[] = []; + const openDatabaseSpy = jest.fn(); + const removeDatabaseItemSpy = jest.fn(); + const renameDatabaseItemSpy = jest.fn(); + const setCurrentDatabaseItemSpy = jest.fn(); + let runTestsSpy: jest.Mock; + const resolveTestsSpy = jest.fn(); + const resolveQlpacksSpy = jest.fn(); + + const preTestDatabaseItem = new DatabaseItemImpl( + Uri.file("/path/to/test/dir/dir.testproj"), + undefined, + mockedObject({ displayName: "custom display name" }), + (_) => { + /* no change event listener */ + }, + ); + const postTestDatabaseItem = new DatabaseItemImpl( + Uri.file("/path/to/test/dir/dir.testproj"), + undefined, + mockedObject({ displayName: "default name" }), + (_) => { + /* no change event listener */ + }, + ); + + beforeEach(() => { + openDatabaseSpy.mockResolvedValue(postTestDatabaseItem); + removeDatabaseItemSpy.mockResolvedValue(undefined); + renameDatabaseItemSpy.mockResolvedValue(undefined); + setCurrentDatabaseItemSpy.mockResolvedValue(undefined); + resolveQlpacksSpy.mockResolvedValue({}); + resolveTestsSpy.mockResolvedValue([]); + fakeDatabaseManager = mockedObject( + { + openDatabase: openDatabaseSpy, + removeDatabaseItem: removeDatabaseItemSpy, + renameDatabaseItem: renameDatabaseItemSpy, + setCurrentDatabaseItem: setCurrentDatabaseItemSpy, + }, + { + dynamicProperties: { + currentDatabaseItem: () => currentDatabaseItem, + databaseItems: () => databaseItems, + }, + }, + ); + + jest.spyOn(preTestDatabaseItem, "isAffectedByTest").mockResolvedValue(true); + + const mockCli = createMockCliServerForTestRun(); + fakeCliServer = mockCli.cliServer; + runTestsSpy = mockCli.runTestsSpy; + + testRunner = new TestRunner(fakeDatabaseManager, fakeCliServer); + }); + + it("should run some tests", async () => { + const eventHandlerSpy = jest.fn(); + + await testRunner.run( + [mockTestsInfo.dPath, mockTestsInfo.gPath, mockTestsInfo.hPath], + createMockLogger(), + new CancellationTokenSource().token, + eventHandlerSpy, + ); + + expect(eventHandlerSpy).toBeCalledTimes(3); + + expect(eventHandlerSpy).toHaveBeenNthCalledWith(1, { + test: mockTestsInfo.dPath, + pass: true, + compilationMs: 1000, + evaluationMs: 2000, + messages: [], + }); + expect(eventHandlerSpy).toHaveBeenNthCalledWith(2, { + test: mockTestsInfo.gPath, + pass: false, + compilationMs: 4000, + evaluationMs: 0, + diff: ["pqr", "xyz"], + failureStage: "COMPILATION", + messages: [ + { + message: "abc", + position: { + line: 1, + column: 1, + endLine: 2, + endColumn: 2, + fileName: mockTestsInfo.gPath, + }, + severity: "ERROR", + }, + ], + }); + expect(eventHandlerSpy).toHaveBeenNthCalledWith(3, { + test: mockTestsInfo.hPath, + pass: false, + compilationMs: 5000, + evaluationMs: 6000, + diff: ["jkh", "tuv"], + failureStage: "RESULT", + messages: [], + }); + }); + + it("should reregister testproj databases around test run", async () => { + currentDatabaseItem = preTestDatabaseItem; + databaseItems = [preTestDatabaseItem]; + await testRunner.run( + ["/path/to/test/dir"], + createMockLogger(), + new CancellationTokenSource().token, + async () => { + /***/ + }, + ); + + expect(removeDatabaseItemSpy.mock.invocationCallOrder[0]).toBeLessThan( + runTestsSpy.mock.invocationCallOrder[0], + ); + expect(openDatabaseSpy.mock.invocationCallOrder[0]).toBeGreaterThan( + runTestsSpy.mock.invocationCallOrder[0], + ); + expect(renameDatabaseItemSpy.mock.invocationCallOrder[0]).toBeGreaterThan( + openDatabaseSpy.mock.invocationCallOrder[0], + ); + expect( + setCurrentDatabaseItemSpy.mock.invocationCallOrder[0], + ).toBeGreaterThan(openDatabaseSpy.mock.invocationCallOrder[0]); + + expect(removeDatabaseItemSpy).toBeCalledTimes(1); + expect(removeDatabaseItemSpy).toBeCalledWith( + expect.anything(), + expect.anything(), + preTestDatabaseItem, + ); + + expect(openDatabaseSpy).toBeCalledTimes(1); + expect(openDatabaseSpy).toBeCalledWith( + expect.anything(), + expect.anything(), + preTestDatabaseItem.databaseUri, + ); + + expect(renameDatabaseItemSpy).toBeCalledTimes(1); + expect(renameDatabaseItemSpy).toBeCalledWith( + postTestDatabaseItem, + preTestDatabaseItem.name, + ); + + expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1); + expect(setCurrentDatabaseItemSpy).toBeCalledWith( + postTestDatabaseItem, + true, + ); + }); +});