Add FilePathDiscovery

extensions/ql-vscode/src/common/vscode/file-path-discovery.ts

@@ -0,0 +1,190 @@
+import { Discovery } from "../discovery";
+import {
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFoldersChangeEvent,
+  workspace,
+} from "vscode";
+import { MultiFileSystemWatcher } from "./multi-file-system-watcher";
+import { AppEventEmitter } from "../events";
+import { extLogger } from "..";
+import { FilePathSet } from "../file-path-set";
+import { exists, lstat } from "fs-extra";
+import { containsPath } from "../../pure/files";
+import { getOnDiskWorkspaceFoldersObjects } from "./workspace-folders";
+
+interface PathData {
+  path: string;
+}
+
+/**
+ * Discovers all files matching a given filter contained in the workspace.
+ *
+ * Scans the whole workspace on startup, and then watches for changes to files
+ * to do the minimum work to keep up with changes.
+ *
+ * Can configure which changes it watches for, which files are considered
+ * relevant, and what extra data to compute for each file.
+ */
+export abstract class FilePathDiscovery<T extends PathData> extends Discovery {
+  /** The set of known paths we are tracking */
+  protected paths: T[] = [];
+  protected readonly onDidChangePathsEmitter: AppEventEmitter<void>;
+
+  private readonly changedFilePaths = new FilePathSet();
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  /**
+   * @param name Name of the discovery operation, for logging purposes.
+   * @param fileWatchPattern Passed to `vscode.RelativePattern` to determine the files to watch for changes to.
+   */
+  constructor(name: string, private readonly fileWatchPattern: string) {
+    super(name, extLogger);
+
+    this.onDidChangePathsEmitter = this.push(new EventEmitter<void>());
+    this.push(
+      workspace.onDidChangeWorkspaceFolders(
+        this.workspaceFoldersChanged.bind(this),
+      ),
+    );
+    this.push(this.watcher.onDidChange(this.fileChanged.bind(this)));
+  }
+
+  /**
+   * Compute any extra data to be stored regarding the given path.
+   */
+  protected abstract getDataForPath(path: string): Promise<T>;
+
+  /**
+   * Is the given path relevant to this discovery operation?
+   */
+  protected abstract pathIsRelevant(path: string): boolean;
+
+  /**
+   * Should the given new data overwrite the existing data we have stored?
+   */
+  protected abstract shouldOverwriteExistingData(
+    newData: T,
+    existingData: T,
+  ): boolean;
+
+  /**
+   * Do the initial scan of the entire workspace and set up watchers for future changes.
+   */
+  public async initialRefresh() {
+    getOnDiskWorkspaceFoldersObjects().forEach((workspaceFolder) => {
+      this.changedFilePaths.addPath(workspaceFolder.uri.fsPath);
+    });
+
+    this.updateWatchers();
+    return this.refresh();
+  }
+
+  private workspaceFoldersChanged(event: WorkspaceFoldersChangeEvent) {
+    event.added.forEach((workspaceFolder) => {
+      this.changedFilePaths.addPath(workspaceFolder.uri.fsPath);
+    });
+    event.removed.forEach((workspaceFolder) => {
+      this.changedFilePaths.addPath(workspaceFolder.uri.fsPath);
+    });
+
+    this.updateWatchers();
+    void this.refresh();
+  }
+
+  private updateWatchers() {
+    this.watcher.clear();
+    for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
+      // Watch for changes to individual files
+      this.watcher.addWatch(
+        new RelativePattern(workspaceFolder, this.fileWatchPattern),
+      );
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(workspaceFolder, "**/"));
+    }
+  }
+
+  private fileChanged(uri: Uri) {
+    this.changedFilePaths.addPath(uri.fsPath);
+    void this.refresh();
+  }
+
+  protected async discover() {
+    let pathsUpdated = false;
+    let path: string | undefined;
+    while ((path = this.changedFilePaths.popPath()) !== undefined) {
+      if (await this.handledChangedPath(path)) {
+        pathsUpdated = true;
+      }
+    }
+
+    if (pathsUpdated) {
+      this.onDidChangePathsEmitter.fire();
+    }
+  }
+
+  private async handledChangedPath(path: string): Promise<boolean> {
+    if (!(await exists(path)) || !this.pathIsInWorkspace(path)) {
+      return this.handledRemovedPath(path);
+    }
+    if ((await lstat(path)).isDirectory()) {
+      return await this.handleChangedDirectory(path);
+    }
+    return this.handleChangedFile(path);
+  }
+
+  private pathIsInWorkspace(path: string): boolean {
+    return getOnDiskWorkspaceFoldersObjects().some((workspaceFolder) =>
+      containsPath(workspaceFolder.uri.fsPath, path),
+    );
+  }
+
+  private handledRemovedPath(path: string): boolean {
+    const oldLength = this.paths.length;
+    this.paths = this.paths.filter((q) => !containsPath(path, q.path));
+    return this.paths.length !== oldLength;
+  }
+
+  private async handleChangedDirectory(path: string): Promise<boolean> {
+    const newPaths = await workspace.findFiles(
+      new RelativePattern(path, this.fileWatchPattern),
+    );
+
+    let pathsUpdated = false;
+    for (const path of newPaths) {
+      if (await this.addOrUpdatePath(path.fsPath)) {
+        pathsUpdated = true;
+      }
+    }
+    return pathsUpdated;
+  }
+
+  private async handleChangedFile(path: string): Promise<boolean> {
+    if (this.pathIsRelevant(path)) {
+      return await this.addOrUpdatePath(path);
+    } else {
+      return false;
+    }
+  }
+
+  private async addOrUpdatePath(path: string): Promise<boolean> {
+    const data = await this.getDataForPath(path);
+    const existingDataIndex = this.paths.findIndex((x) => x.path === path);
+    if (existingDataIndex !== -1) {
+      if (
+        this.shouldOverwriteExistingData(data, this.paths[existingDataIndex])
+      ) {
+        this.paths.splice(existingDataIndex, 1, data);
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      this.paths.push(data);
+      return true;
+    }
+  }
+}

extensions/ql-vscode/test/vscode-tests/minimal-workspace/common/vscode/file-path-discovery.test.ts

@@ -0,0 +1,460 @@
+import {
+  EventEmitter,
+  FileSystemWatcher,
+  Uri,
+  workspace,
+  WorkspaceFolder,
+  WorkspaceFoldersChangeEvent,
+} from "vscode";
+import { FilePathDiscovery } from "../../../../../src/common/vscode/file-path-discovery";
+import { basename, dirname, join } from "path";
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
+import * as tmp from "tmp";
+import { expectArraysEqual } from "../../../utils/expect-arrays-equal";
+
+interface TestData {
+  path: string;
+  contents: string;
+}
+
+/**
+ * A test FilePathDiscovery that operates on files with the ".test" extension.
+ */
+class TestFilePathDiscovery extends FilePathDiscovery<TestData> {
+  constructor() {
+    super("TestFilePathDiscovery", "**/*.test");
+  }
+
+  public get onDidChangePaths() {
+    return this.onDidChangePathsEmitter.event;
+  }
+
+  public getPaths(): TestData[] {
+    return this.paths;
+  }
+
+  protected async getDataForPath(path: string): Promise<TestData> {
+    return {
+      path,
+      contents: readFileSync(path, "utf8"),
+    };
+  }
+
+  protected pathIsRelevant(path: string): boolean {
+    return path.endsWith(".test");
+  }
+
+  protected shouldOverwriteExistingData(
+    newData: TestData,
+    existingData: TestData,
+  ): boolean {
+    return newData.contents !== existingData.contents;
+  }
+}
+
+describe("FilePathDiscovery", () => {
+  let tmpDir: string;
+  let tmpDirRemoveCallback: (() => void) | undefined;
+
+  let workspaceFolder: WorkspaceFolder;
+  let workspacePath: string;
+  let workspaceFoldersSpy: jest.SpyInstance;
+
+  const onDidCreateFile = new EventEmitter<Uri>();
+  const onDidChangeFile = new EventEmitter<Uri>();
+  const onDidDeleteFile = new EventEmitter<Uri>();
+  let createFileSystemWatcherSpy: jest.SpyInstance;
+
+  const onDidChangeWorkspaceFolders =
+    new EventEmitter<WorkspaceFoldersChangeEvent>();
+
+  let discovery: TestFilePathDiscovery;
+
+  beforeEach(() => {
+    const t = tmp.dirSync();
+    tmpDir = t.name;
+    tmpDirRemoveCallback = t.removeCallback;
+
+    workspaceFolder = {
+      uri: Uri.file(join(tmpDir, "workspace")),
+      name: "test",
+      index: 0,
+    };
+    workspacePath = workspaceFolder.uri.fsPath;
+    workspaceFoldersSpy = jest
+      .spyOn(workspace, "workspaceFolders", "get")
+      .mockReturnValue([workspaceFolder]);
+
+    const watcher: FileSystemWatcher = {
+      ignoreCreateEvents: false,
+      ignoreChangeEvents: false,
+      ignoreDeleteEvents: false,
+      onDidCreate: onDidCreateFile.event,
+      onDidChange: onDidChangeFile.event,
+      onDidDelete: onDidDeleteFile.event,
+      dispose: () => undefined,
+    };
+    createFileSystemWatcherSpy = jest
+      .spyOn(workspace, "createFileSystemWatcher")
+      .mockReturnValue(watcher);
+
+    jest
+      .spyOn(workspace, "onDidChangeWorkspaceFolders")
+      .mockImplementation(onDidChangeWorkspaceFolders.event);
+
+    discovery = new TestFilePathDiscovery();
+  });
+
+  afterEach(() => {
+    tmpDirRemoveCallback?.();
+    discovery.dispose();
+  });
+
+  describe("initialRefresh", () => {
+    it("should handle no files being present", async () => {
+      await discovery.initialRefresh();
+      expect(discovery.getPaths()).toEqual([]);
+    });
+
+    it("should recursively discover all test files", async () => {
+      makeTestFile(join(workspacePath, "123.test"));
+      makeTestFile(join(workspacePath, "456.test"));
+      makeTestFile(join(workspacePath, "bar", "789.test"));
+
+      await discovery.initialRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+        { path: join(workspacePath, "456.test"), contents: "456" },
+        { path: join(workspacePath, "bar", "789.test"), contents: "789" },
+      ]);
+    });
+
+    it("should ignore non-test files", async () => {
+      makeTestFile(join(workspacePath, "1.test"));
+      makeTestFile(join(workspacePath, "2.foo"));
+      makeTestFile(join(workspacePath, "bar.ql"));
+
+      await discovery.initialRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+    });
+  });
+
+  describe("file added", () => {
+    it("should discover a single new file", async () => {
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expect(discovery.getPaths()).toEqual([]);
+
+      const newFile = join(workspacePath, "1.test");
+      makeTestFile(newFile);
+      onDidCreateFile.fire(Uri.file(newFile));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+      expect(didChangePathsListener).toHaveBeenCalled();
+    });
+
+    it("should do nothing if file doesnt actually exist", async () => {
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expect(discovery.getPaths()).toEqual([]);
+
+      onDidCreateFile.fire(Uri.file(join(workspacePath, "1.test")));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), []);
+      expect(didChangePathsListener).not.toHaveBeenCalled();
+    });
+
+    it("should recursively discover a directory of new files", async () => {
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expect(discovery.getPaths()).toEqual([]);
+
+      const newDir = join(workspacePath, "foo");
+      makeTestFile(join(newDir, "1.test"));
+      makeTestFile(join(newDir, "bar", "2.test"));
+      makeTestFile(join(newDir, "bar", "3.test"));
+      onDidCreateFile.fire(Uri.file(newDir));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(newDir, "1.test"), contents: "1" },
+        { path: join(newDir, "bar", "2.test"), contents: "2" },
+        { path: join(newDir, "bar", "3.test"), contents: "3" },
+      ]);
+      expect(didChangePathsListener).toHaveBeenCalled();
+    });
+
+    it("should do nothing if file is already known", async () => {
+      const testFile = join(workspacePath, "1.test");
+      makeTestFile(testFile);
+
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+
+      onDidCreateFile.fire(Uri.file(testFile));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+      expect(didChangePathsListener).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("file changed", () => {
+    it("should do nothing if nothing has actually changed", async () => {
+      const testFile = join(workspacePath, "123.test");
+      makeTestFile(testFile);
+
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+      ]);
+
+      onDidChangeFile.fire(Uri.file(testFile));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+      ]);
+      expect(didChangePathsListener).not.toHaveBeenCalled();
+    });
+
+    it("should update data if it has changed", async () => {
+      const testFile = join(workspacePath, "1.test");
+      makeTestFile(testFile, "foo");
+
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "foo" },
+      ]);
+
+      writeFileSync(testFile, "bar");
+      onDidChangeFile.fire(Uri.file(testFile));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "bar" },
+      ]);
+      expect(didChangePathsListener).toHaveBeenCalled();
+    });
+  });
+
+  describe("file deleted", () => {
+    it("should remove a file that has been deleted", async () => {
+      const testFile = join(workspacePath, "1.test");
+      makeTestFile(testFile);
+
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+
+      rmSync(testFile);
+      onDidDeleteFile.fire(Uri.file(testFile));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), []);
+      expect(didChangePathsListener).toHaveBeenCalled();
+    });
+
+    it("should do nothing if file still exists", async () => {
+      const testFile = join(workspacePath, "1.test");
+      makeTestFile(testFile);
+
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+
+      onDidDeleteFile.fire(Uri.file(testFile));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "1.test"), contents: "1" },
+      ]);
+      expect(didChangePathsListener).not.toHaveBeenCalled();
+    });
+
+    it("should remove a directory of files that has been deleted", async () => {
+      makeTestFile(join(workspacePath, "123.test"));
+      makeTestFile(join(workspacePath, "bar", "456.test"));
+      makeTestFile(join(workspacePath, "bar", "789.test"));
+
+      await discovery.initialRefresh();
+
+      const didChangePathsListener = jest.fn();
+      discovery.onDidChangePaths(didChangePathsListener);
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+        { path: join(workspacePath, "bar", "456.test"), contents: "456" },
+        { path: join(workspacePath, "bar", "789.test"), contents: "789" },
+      ]);
+
+      rmSync(join(workspacePath, "bar"), { recursive: true });
+
+      onDidDeleteFile.fire(Uri.file(join(workspacePath, "bar")));
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+      ]);
+      expect(didChangePathsListener).toHaveBeenCalled();
+    });
+  });
+
+  describe("workspaceFoldersChanged", () => {
+    it("initialRefresh establishes watchers", async () => {
+      await discovery.initialRefresh();
+
+      // Called twice for each workspace folder
+      expect(createFileSystemWatcherSpy).toHaveBeenCalledTimes(2);
+    });
+
+    it("should install watchers when workspace folders change", async () => {
+      await discovery.initialRefresh();
+
+      createFileSystemWatcherSpy.mockClear();
+
+      const previousWorkspaceFolders = workspace.workspaceFolders || [];
+      const newWorkspaceFolders: WorkspaceFolder[] = [
+        {
+          uri: Uri.file(join(tmpDir, "workspace2")),
+          name: "workspace2",
+          index: 0,
+        },
+        {
+          uri: Uri.file(join(tmpDir, "workspace3")),
+          name: "workspace3",
+          index: 1,
+        },
+      ];
+      workspaceFoldersSpy.mockReturnValue(newWorkspaceFolders);
+
+      onDidChangeWorkspaceFolders.fire({
+        added: newWorkspaceFolders,
+        removed: previousWorkspaceFolders,
+      });
+      await discovery.waitForCurrentRefresh();
+
+      // Called twice for each workspace folder
+      expect(createFileSystemWatcherSpy).toHaveBeenCalledTimes(4);
+    });
+
+    it("should discover files in new workspaces", async () => {
+      makeTestFile(join(workspacePath, "123.test"));
+
+      await discovery.initialRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+      ]);
+
+      const previousWorkspaceFolders = workspace.workspaceFolders || [];
+      const newWorkspaceFolders: WorkspaceFolder[] = [
+        workspaceFolder,
+        {
+          uri: Uri.file(join(tmpDir, "workspace2")),
+          name: "workspace2",
+          index: 1,
+        },
+      ];
+      workspaceFoldersSpy.mockReturnValue(newWorkspaceFolders);
+
+      makeTestFile(join(tmpDir, "workspace2", "456.test"));
+
+      onDidChangeWorkspaceFolders.fire({
+        added: newWorkspaceFolders,
+        removed: previousWorkspaceFolders,
+      });
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(workspacePath, "123.test"), contents: "123" },
+        { path: join(tmpDir, "workspace2", "456.test"), contents: "456" },
+      ]);
+    });
+
+    it("should forgot files in old workspaces, even if the files on disk still exist", async () => {
+      const workspaceFolders: WorkspaceFolder[] = [
+        {
+          uri: Uri.file(join(tmpDir, "workspace1")),
+          name: "workspace1",
+          index: 0,
+        },
+        {
+          uri: Uri.file(join(tmpDir, "workspace2")),
+          name: "workspace2",
+          index: 1,
+        },
+      ];
+      workspaceFoldersSpy.mockReturnValue(workspaceFolders);
+
+      makeTestFile(join(tmpDir, "workspace1", "123.test"));
+      makeTestFile(join(tmpDir, "workspace2", "456.test"));
+
+      await discovery.initialRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(tmpDir, "workspace1", "123.test"), contents: "123" },
+        { path: join(tmpDir, "workspace2", "456.test"), contents: "456" },
+      ]);
+
+      workspaceFoldersSpy.mockReturnValue([workspaceFolders[0]]);
+      onDidChangeWorkspaceFolders.fire({
+        added: [],
+        removed: [workspaceFolders[1]],
+      });
+      await discovery.waitForCurrentRefresh();
+
+      expectArraysEqual(discovery.getPaths(), [
+        { path: join(tmpDir, "workspace1", "123.test"), contents: "123" },
+      ]);
+    });
+  });
+});
+
+function makeTestFile(path: string, contents?: string) {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, contents ?? basename(path, ".test"));
+}

extensions/ql-vscode/test/vscode-tests/utils/expect-arrays-equal.ts

@@ -0,0 +1,6 @@
+export function expectArraysEqual<T>(actual: T[], expected: T[]) {
+  // Check that all of the expected values are present
+  expect(actual).toEqual(expect.arrayContaining(expected));
+  // Check that no extra un-expected values are present
+  expect(expected).toEqual(expect.arrayContaining(actual));
+}