Test tests

extensions/ql-vscode/src/test-manager.ts

@@ -109,11 +109,6 @@ class WorkspaceFolderHandler extends DisposableObject {
  * debugging of tests.
  */
 export class TestManager extends TestManagerBase {
-  private readonly testController: TestController = tests.createTestController(
-    "codeql",
-    "Fancy CodeQL Tests",
-  );
-
   /**
    * Maps from each workspace folder being tracked to the `WorkspaceFolderHandler` responsible for
    * tracking it.
@@ -127,6 +122,11 @@ export class TestManager extends TestManagerBase {
     app: App,
     private readonly testRunner: TestRunner,
     private readonly cliServer: CodeQLCliServer,
+    // Having this as a parameter with a default value makes passing in a mock easier.
+    private readonly testController: TestController = tests.createTestController(
+      "codeql",
+      "Fancy CodeQL Tests",
+    ),
   ) {
     super(app);
 
@@ -245,8 +245,10 @@ export class TestManager extends TestManagerBase {
 
   /**
    * Run the tests specified by the `TestRunRequest` parameter.
+   *
+   * Public because this is used in unit tests.
    */
-  private async run(
+  public async run(
     request: TestRunRequest,
     token: CancellationToken,
   ): Promise<void> {

extensions/ql-vscode/test/vscode-tests/no-workspace/test-adapter.test.ts

@@ -1,90 +1,46 @@
-import { Uri, WorkspaceFolder } from "vscode";
+import {
+  CancellationTokenSource,
+  Range,
+  TestItem,
+  TestItemCollection,
+  TestRun,
+  TestRunRequest,
+  Uri,
+  WorkspaceFolder,
+  tests,
+} from "vscode";
 
 import { QLTestAdapter } from "../../../src/test-adapter";
 import { CodeQLCliServer } from "../../../src/cli";
-import {
-  DatabaseItem,
-  DatabaseItemImpl,
-  DatabaseManager,
-  FullDatabaseOptions,
-} from "../../../src/local-databases";
+import { DatabaseManager } from "../../../src/local-databases";
 import { mockedObject } from "../utils/mocking.helpers";
 import { TestRunner } from "../../../src/test-runner";
+import {
+  createMockCliServerForTestRun,
+  mockEmptyDatabaseManager,
+  mockTestsInfo,
+} from "./test-runner-helpers";
+import { TestManager } from "../../../src/test-manager";
+import { createMockApp } from "../../__mocks__/appMock";
 
-jest.mock("fs-extra", () => {
-  const original = jest.requireActual("fs-extra");
-  return {
-    ...original,
-    access: jest.fn(),
-  };
-});
+type IdTestItemPair = [id: string, testItem: TestItem];
 
 describe("test-adapter", () => {
   let testRunner: TestRunner;
-  let adapter: QLTestAdapter;
   let fakeDatabaseManager: DatabaseManager;
   let fakeCliServer: CodeQLCliServer;
-  let currentDatabaseItem: DatabaseItem | undefined;
-  let databaseItems: DatabaseItem[] = [];
-  const openDatabaseSpy = jest.fn();
-  const removeDatabaseItemSpy = jest.fn();
-  const renameDatabaseItemSpy = jest.fn();
-  const setCurrentDatabaseItemSpy = jest.fn();
-  const runTestsSpy = jest.fn();
-  const resolveTestsSpy = jest.fn();
-  const resolveQlpacksSpy = jest.fn();
-
-  const preTestDatabaseItem = new DatabaseItemImpl(
-    Uri.file("/path/to/test/dir/dir.testproj"),
-    undefined,
-    mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
-    (_) => {
-      /* no change event listener */
-    },
-  );
-  const postTestDatabaseItem = new DatabaseItemImpl(
-    Uri.file("/path/to/test/dir/dir.testproj"),
-    undefined,
-    mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
-    (_) => {
-      /* no change event listener */
-    },
-  );
 
   beforeEach(() => {
-    mockRunTests();
-    openDatabaseSpy.mockResolvedValue(postTestDatabaseItem);
-    removeDatabaseItemSpy.mockResolvedValue(undefined);
-    renameDatabaseItemSpy.mockResolvedValue(undefined);
-    setCurrentDatabaseItemSpy.mockResolvedValue(undefined);
-    resolveQlpacksSpy.mockResolvedValue({});
-    resolveTestsSpy.mockResolvedValue([]);
-    fakeDatabaseManager = mockedObject<DatabaseManager>(
-      {
-        openDatabase: openDatabaseSpy,
-        removeDatabaseItem: removeDatabaseItemSpy,
-        renameDatabaseItem: renameDatabaseItemSpy,
-        setCurrentDatabaseItem: setCurrentDatabaseItemSpy,
-      },
-      {
-        dynamicProperties: {
-          currentDatabaseItem: () => currentDatabaseItem,
-          databaseItems: () => databaseItems,
-        },
-      },
-    );
+    fakeDatabaseManager = mockEmptyDatabaseManager();
 
-    jest.spyOn(preTestDatabaseItem, "isAffectedByTest").mockResolvedValue(true);
-
-    fakeCliServer = mockedObject<CodeQLCliServer>({
-      runTests: runTestsSpy,
-      resolveQlpacks: resolveQlpacksSpy,
-      resolveTests: resolveTestsSpy,
-    });
+    const mockCli = createMockCliServerForTestRun();
+    fakeCliServer = mockCli.cliServer;
 
     testRunner = new TestRunner(fakeDatabaseManager, fakeCliServer);
+  });
 
-    adapter = new QLTestAdapter(
+  it("legacy test adapter should run some tests", async () => {
+    const adapter = new QLTestAdapter(
       mockedObject<WorkspaceFolder>({
         name: "ABC",
         uri: Uri.parse("file:/ab/c"),
@@ -92,121 +48,128 @@ describe("test-adapter", () => {
       testRunner,
       fakeCliServer,
     );
-  });
 
-  it("should run some tests", async () => {
     const listenerSpy = jest.fn();
     adapter.testStates(listenerSpy);
-    const testsPath = Uri.parse("file:/ab/c").fsPath;
-    const dPath = Uri.parse("file:/ab/c/d.ql").fsPath;
-    const gPath = Uri.parse("file:/ab/c/e/f/g.ql").fsPath;
-    const hPath = Uri.parse("file:/ab/c/e/f/h.ql").fsPath;
-
-    await adapter.run([testsPath]);
+    await adapter.run([mockTestsInfo.testsPath]);
 
     expect(listenerSpy).toBeCalledTimes(5);
 
     expect(listenerSpy).toHaveBeenNthCalledWith(1, {
       type: "started",
-      tests: [testsPath],
+      tests: [mockTestsInfo.testsPath],
     });
     expect(listenerSpy).toHaveBeenNthCalledWith(2, {
       type: "test",
       state: "passed",
-      test: dPath,
+      test: mockTestsInfo.dPath,
       message: undefined,
       decorations: [],
     });
     expect(listenerSpy).toHaveBeenNthCalledWith(3, {
       type: "test",
       state: "errored",
-      test: gPath,
-      message: `\ncompilation error: ${gPath}\nERROR: abc\n`,
+      test: mockTestsInfo.gPath,
+      message: `\ncompilation error: ${mockTestsInfo.gPath}\nERROR: abc\n`,
       decorations: [{ line: 1, message: "abc" }],
     });
     expect(listenerSpy).toHaveBeenNthCalledWith(4, {
       type: "test",
       state: "failed",
-      test: hPath,
-      message: `\nfailed: ${hPath}\njkh\ntuv\n`,
+      test: mockTestsInfo.hPath,
+      message: `\nfailed: ${mockTestsInfo.hPath}\njkh\ntuv\n`,
       decorations: [],
     });
     expect(listenerSpy).toHaveBeenNthCalledWith(5, { type: "finished" });
   });
 
-  it("should reregister testproj databases around test run", async () => {
-    currentDatabaseItem = preTestDatabaseItem;
-    databaseItems = [preTestDatabaseItem];
-    await adapter.run(["/path/to/test/dir"]);
+  it("native test manager should run some tests", async () => {
+    const enqueuedSpy = jest.fn();
+    const passedSpy = jest.fn();
+    const erroredSpy = jest.fn();
+    const failedSpy = jest.fn();
+    const endSpy = jest.fn();
 
-    expect(removeDatabaseItemSpy.mock.invocationCallOrder[0]).toBeLessThan(
-      runTestsSpy.mock.invocationCallOrder[0],
+    const testController = tests.createTestController("codeql", "CodeQL Tests");
+    testController.createTestRun = jest.fn().mockImplementation(() =>
+      mockedObject<TestRun>({
+        enqueued: enqueuedSpy,
+        passed: passedSpy,
+        errored: erroredSpy,
+        failed: failedSpy,
+        end: endSpy,
+      }),
     );
-    expect(openDatabaseSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
-      runTestsSpy.mock.invocationCallOrder[0],
-    );
-    expect(renameDatabaseItemSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
-      openDatabaseSpy.mock.invocationCallOrder[0],
-    );
-    expect(
-      setCurrentDatabaseItemSpy.mock.invocationCallOrder[0],
-    ).toBeGreaterThan(openDatabaseSpy.mock.invocationCallOrder[0]);
-
-    expect(removeDatabaseItemSpy).toBeCalledTimes(1);
-    expect(removeDatabaseItemSpy).toBeCalledWith(
-      expect.anything(),
-      expect.anything(),
-      preTestDatabaseItem,
+    const testManager = new TestManager(
+      createMockApp({}),
+      testRunner,
+      fakeCliServer,
+      testController,
     );
 
-    expect(openDatabaseSpy).toBeCalledTimes(1);
-    expect(openDatabaseSpy).toBeCalledWith(
-      expect.anything(),
-      expect.anything(),
-      preTestDatabaseItem.databaseUri,
-    );
+    const childItems: TestItem[] = [
+      {
+        children: { size: 0 } as TestItemCollection,
+        id: `test ${mockTestsInfo.dPath}`,
+        uri: Uri.file(mockTestsInfo.dPath),
+      } as TestItem,
+      {
+        children: { size: 0 } as TestItemCollection,
+        id: `test ${mockTestsInfo.gPath}`,
+        uri: Uri.file(mockTestsInfo.gPath),
+      } as TestItem,
+      {
+        children: { size: 0 } as TestItemCollection,
+        id: `test ${mockTestsInfo.hPath}`,
+        uri: Uri.file(mockTestsInfo.hPath),
+      } as TestItem,
+    ];
+    const childElements: IdTestItemPair[] = childItems.map((childItem) => [
+      childItem.id,
+      childItem,
+    ]);
+    const childIteratorFunc: () => Iterator<IdTestItemPair> = () =>
+      childElements[Symbol.iterator]();
 
-    expect(renameDatabaseItemSpy).toBeCalledTimes(1);
-    expect(renameDatabaseItemSpy).toBeCalledWith(
-      postTestDatabaseItem,
-      preTestDatabaseItem.name,
-    );
+    const rootItem = {
+      id: `dir ${mockTestsInfo.testsPath}`,
+      uri: Uri.file(mockTestsInfo.testsPath),
+      children: {
+        size: 3,
+        [Symbol.iterator]: childIteratorFunc,
+      } as TestItemCollection,
+    } as TestItem;
 
-    expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1);
-    expect(setCurrentDatabaseItemSpy).toBeCalledWith(
-      postTestDatabaseItem,
-      true,
+    const request = new TestRunRequest([rootItem]);
+    await testManager.run(request, new CancellationTokenSource().token);
+
+    expect(enqueuedSpy).toBeCalledTimes(3);
+    expect(passedSpy).toBeCalledTimes(1);
+    expect(passedSpy).toHaveBeenCalledWith(childItems[0], 3000);
+    expect(erroredSpy).toHaveBeenCalledTimes(1);
+    expect(erroredSpy).toHaveBeenCalledWith(
+      childItems[1],
+      [
+        {
+          location: {
+            range: new Range(0, 0, 1, 1),
+            uri: Uri.file(mockTestsInfo.gPath),
+          },
+          message: "abc",
+        },
+      ],
+      4000,
     );
+    expect(failedSpy).toHaveBeenCalledWith(
+      childItems[2],
+      [
+        {
+          message: "Test failed",
+        },
+      ],
+      11000,
+    );
+    expect(failedSpy).toBeCalledTimes(1);
+    expect(endSpy).toBeCalledTimes(1);
   });
-
-  function mockRunTests() {
-    // runTests is an async generator function. This is not directly supported in sinon
-    // However, we can pretend the same thing by just returning an async array.
-    runTestsSpy.mockReturnValue(
-      (async function* () {
-        yield Promise.resolve({
-          test: Uri.parse("file:/ab/c/d.ql").fsPath,
-          pass: true,
-          messages: [],
-        });
-        yield Promise.resolve({
-          test: Uri.parse("file:/ab/c/e/f/g.ql").fsPath,
-          pass: false,
-          diff: ["pqr", "xyz"],
-          // a compile error
-          failureStage: "COMPILATION",
-          messages: [
-            { position: { line: 1 }, message: "abc", severity: "ERROR" },
-          ],
-        });
-        yield Promise.resolve({
-          test: Uri.parse("file:/ab/c/e/f/h.ql").fsPath,
-          pass: false,
-          diff: ["jkh", "tuv"],
-          failureStage: "RESULT",
-          messages: [],
-        });
-      })(),
-    );
-  }
 });

extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner-helpers.ts

@@ -0,0 +1,96 @@
+import { Uri } from "vscode";
+import { mockedObject } from "../utils/mocking.helpers";
+import { CodeQLCliServer } from "../../../src/cli";
+import { DatabaseManager } from "../../../src/local-databases";
+
+/**
+ * Fake QL tests used by various tests.
+ */
+export const mockTestsInfo = {
+  testsPath: Uri.parse("file:/ab/c").fsPath,
+  dPath: Uri.parse("file:/ab/c/d.ql").fsPath,
+  gPath: Uri.parse("file:/ab/c/e/f/g.ql").fsPath,
+  hPath: Uri.parse("file:/ab/c/e/f/h.ql").fsPath,
+};
+
+/**
+ * Create a mock of a `DatabaseManager` with no databases loaded.
+ */
+export function mockEmptyDatabaseManager(): DatabaseManager {
+  return mockedObject<DatabaseManager>({
+    currentDatabaseItem: undefined,
+    databaseItems: [],
+  });
+}
+
+/**
+ * Creates a `CodeQLCliServer` that "runs" the mock tests. Also returns the spy
+ * hook for the `runTests` function on the CLI server.
+ */
+export function createMockCliServerForTestRun() {
+  const resolveQlpacksSpy = jest.fn();
+  resolveQlpacksSpy.mockResolvedValue({});
+
+  const resolveTestsSpy = jest.fn();
+  resolveTestsSpy.mockResolvedValue([]);
+
+  const runTestsSpy = mockRunTests();
+  return {
+    cliServer: mockedObject<CodeQLCliServer>({
+      runTests: runTestsSpy,
+      resolveQlpacks: resolveQlpacksSpy,
+      resolveTests: resolveTestsSpy,
+    }),
+    runTestsSpy,
+  };
+}
+
+function mockRunTests(): jest.Mock<any, any> {
+  const runTestsSpy = jest.fn();
+  // runTests is an async generator function. This is not directly supported in sinon
+  // However, we can pretend the same thing by just returning an async array.
+  runTestsSpy.mockReturnValue(
+    (async function* () {
+      yield Promise.resolve({
+        test: mockTestsInfo.dPath,
+        pass: true,
+        messages: [],
+        compilationMs: 1000,
+        evaluationMs: 2000,
+      });
+      yield Promise.resolve({
+        test: mockTestsInfo.gPath,
+        pass: false,
+        diff: ["pqr", "xyz"],
+        // a compile error
+        failureStage: "COMPILATION",
+        compilationMs: 4000,
+        evaluationMs: 0,
+        messages: [
+          {
+            position: {
+              fileName: mockTestsInfo.gPath,
+              line: 1,
+              column: 1,
+              endLine: 2,
+              endColumn: 2,
+            },
+            message: "abc",
+            severity: "ERROR",
+          },
+        ],
+      });
+      yield Promise.resolve({
+        test: mockTestsInfo.hPath,
+        pass: false,
+        diff: ["jkh", "tuv"],
+        failureStage: "RESULT",
+        compilationMs: 5000,
+        evaluationMs: 6000,
+        messages: [],
+      });
+    })(),
+  );
+
+  return runTestsSpy;
+}

extensions/ql-vscode/test/vscode-tests/no-workspace/test-runner.test.ts

@@ -0,0 +1,189 @@
+import { CancellationTokenSource, Uri } from "vscode";
+import { CodeQLCliServer } from "../../../src/cli";
+import {
+  DatabaseItem,
+  DatabaseItemImpl,
+  DatabaseManager,
+  FullDatabaseOptions,
+} from "../../../src/local-databases";
+import { mockedObject } from "../utils/mocking.helpers";
+import { TestRunner } from "../../../src/test-runner";
+import { createMockLogger } from "../../__mocks__/loggerMock";
+import {
+  createMockCliServerForTestRun,
+  mockTestsInfo,
+} from "./test-runner-helpers";
+
+jest.mock("fs-extra", () => {
+  const original = jest.requireActual("fs-extra");
+  return {
+    ...original,
+    access: jest.fn(),
+  };
+});
+
+describe("test-runner", () => {
+  let testRunner: TestRunner;
+  let fakeDatabaseManager: DatabaseManager;
+  let fakeCliServer: CodeQLCliServer;
+  let currentDatabaseItem: DatabaseItem | undefined;
+  let databaseItems: DatabaseItem[] = [];
+  const openDatabaseSpy = jest.fn();
+  const removeDatabaseItemSpy = jest.fn();
+  const renameDatabaseItemSpy = jest.fn();
+  const setCurrentDatabaseItemSpy = jest.fn();
+  let runTestsSpy: jest.Mock<any, any>;
+  const resolveTestsSpy = jest.fn();
+  const resolveQlpacksSpy = jest.fn();
+
+  const preTestDatabaseItem = new DatabaseItemImpl(
+    Uri.file("/path/to/test/dir/dir.testproj"),
+    undefined,
+    mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
+    (_) => {
+      /* no change event listener */
+    },
+  );
+  const postTestDatabaseItem = new DatabaseItemImpl(
+    Uri.file("/path/to/test/dir/dir.testproj"),
+    undefined,
+    mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
+    (_) => {
+      /* no change event listener */
+    },
+  );
+
+  beforeEach(() => {
+    openDatabaseSpy.mockResolvedValue(postTestDatabaseItem);
+    removeDatabaseItemSpy.mockResolvedValue(undefined);
+    renameDatabaseItemSpy.mockResolvedValue(undefined);
+    setCurrentDatabaseItemSpy.mockResolvedValue(undefined);
+    resolveQlpacksSpy.mockResolvedValue({});
+    resolveTestsSpy.mockResolvedValue([]);
+    fakeDatabaseManager = mockedObject<DatabaseManager>(
+      {
+        openDatabase: openDatabaseSpy,
+        removeDatabaseItem: removeDatabaseItemSpy,
+        renameDatabaseItem: renameDatabaseItemSpy,
+        setCurrentDatabaseItem: setCurrentDatabaseItemSpy,
+      },
+      {
+        dynamicProperties: {
+          currentDatabaseItem: () => currentDatabaseItem,
+          databaseItems: () => databaseItems,
+        },
+      },
+    );
+
+    jest.spyOn(preTestDatabaseItem, "isAffectedByTest").mockResolvedValue(true);
+
+    const mockCli = createMockCliServerForTestRun();
+    fakeCliServer = mockCli.cliServer;
+    runTestsSpy = mockCli.runTestsSpy;
+
+    testRunner = new TestRunner(fakeDatabaseManager, fakeCliServer);
+  });
+
+  it("should run some tests", async () => {
+    const eventHandlerSpy = jest.fn();
+
+    await testRunner.run(
+      [mockTestsInfo.dPath, mockTestsInfo.gPath, mockTestsInfo.hPath],
+      createMockLogger(),
+      new CancellationTokenSource().token,
+      eventHandlerSpy,
+    );
+
+    expect(eventHandlerSpy).toBeCalledTimes(3);
+
+    expect(eventHandlerSpy).toHaveBeenNthCalledWith(1, {
+      test: mockTestsInfo.dPath,
+      pass: true,
+      compilationMs: 1000,
+      evaluationMs: 2000,
+      messages: [],
+    });
+    expect(eventHandlerSpy).toHaveBeenNthCalledWith(2, {
+      test: mockTestsInfo.gPath,
+      pass: false,
+      compilationMs: 4000,
+      evaluationMs: 0,
+      diff: ["pqr", "xyz"],
+      failureStage: "COMPILATION",
+      messages: [
+        {
+          message: "abc",
+          position: {
+            line: 1,
+            column: 1,
+            endLine: 2,
+            endColumn: 2,
+            fileName: mockTestsInfo.gPath,
+          },
+          severity: "ERROR",
+        },
+      ],
+    });
+    expect(eventHandlerSpy).toHaveBeenNthCalledWith(3, {
+      test: mockTestsInfo.hPath,
+      pass: false,
+      compilationMs: 5000,
+      evaluationMs: 6000,
+      diff: ["jkh", "tuv"],
+      failureStage: "RESULT",
+      messages: [],
+    });
+  });
+
+  it("should reregister testproj databases around test run", async () => {
+    currentDatabaseItem = preTestDatabaseItem;
+    databaseItems = [preTestDatabaseItem];
+    await testRunner.run(
+      ["/path/to/test/dir"],
+      createMockLogger(),
+      new CancellationTokenSource().token,
+      async () => {
+        /***/
+      },
+    );
+
+    expect(removeDatabaseItemSpy.mock.invocationCallOrder[0]).toBeLessThan(
+      runTestsSpy.mock.invocationCallOrder[0],
+    );
+    expect(openDatabaseSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
+      runTestsSpy.mock.invocationCallOrder[0],
+    );
+    expect(renameDatabaseItemSpy.mock.invocationCallOrder[0]).toBeGreaterThan(
+      openDatabaseSpy.mock.invocationCallOrder[0],
+    );
+    expect(
+      setCurrentDatabaseItemSpy.mock.invocationCallOrder[0],
+    ).toBeGreaterThan(openDatabaseSpy.mock.invocationCallOrder[0]);
+
+    expect(removeDatabaseItemSpy).toBeCalledTimes(1);
+    expect(removeDatabaseItemSpy).toBeCalledWith(
+      expect.anything(),
+      expect.anything(),
+      preTestDatabaseItem,
+    );
+
+    expect(openDatabaseSpy).toBeCalledTimes(1);
+    expect(openDatabaseSpy).toBeCalledWith(
+      expect.anything(),
+      expect.anything(),
+      preTestDatabaseItem.databaseUri,
+    );
+
+    expect(renameDatabaseItemSpy).toBeCalledTimes(1);
+    expect(renameDatabaseItemSpy).toBeCalledWith(
+      postTestDatabaseItem,
+      preTestDatabaseItem.name,
+    );
+
+    expect(setCurrentDatabaseItemSpy).toBeCalledTimes(1);
+    expect(setCurrentDatabaseItemSpy).toBeCalledWith(
+      postTestDatabaseItem,
+      true,
+    );
+  });
+});