Update QueryDiscovery to use FilePathDiscovery and QueryPackDiscovery

This commit is contained in:
Robert
2023-06-14 17:15:51 +01:00
parent a9d59aecb8
commit 5cbb7b49d7
5 changed files with 273 additions and 309 deletions

View File

@@ -5,6 +5,7 @@ import { isCanary, showQueriesPanel } from "../config";
import { DisposableObject } from "../pure/disposable-object";
import { QueriesPanel } from "./queries-panel";
import { QueryDiscovery } from "./query-discovery";
import { QueryPackDiscovery } from "./query-pack-discovery";
export class QueriesModule extends DisposableObject {
private constructor(readonly app: App) {
@@ -19,9 +20,16 @@ export class QueriesModule extends DisposableObject {
}
void extLogger.log("Initializing queries panel.");
const queryDiscovery = new QueryDiscovery(app.environment, cliServer);
const queryPackDiscovery = new QueryPackDiscovery(cliServer);
this.push(queryPackDiscovery);
void queryPackDiscovery.initialRefresh();
const queryDiscovery = new QueryDiscovery(
app.environment,
queryPackDiscovery,
);
this.push(queryDiscovery);
void queryDiscovery.refresh();
void queryDiscovery.initialRefresh();
const queriesPanel = new QueriesPanel(queryDiscovery);
this.push(queriesPanel);

View File

@@ -1,136 +1,116 @@
import { dirname, basename, normalize, relative } from "path";
import { Discovery } from "../common/discovery";
import { CodeQLCliServer } from "../codeql-cli/cli";
import {
Event,
EventEmitter,
RelativePattern,
Uri,
WorkspaceFolder,
workspace,
} from "vscode";
import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
import { Event } from "vscode";
import { EnvironmentContext } from "../common/app";
import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
import { AppEventEmitter } from "../common/events";
import {
FileTreeDirectory,
FileTreeLeaf,
FileTreeNode,
} from "../common/file-tree-nodes";
import { QueryDiscoverer } from "./query-tree-data-provider";
import { extLogger } from "../common";
import { FilePathDiscovery } from "../common/vscode/file-path-discovery";
import { containsPath } from "../pure/files";
import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
/**
* The results of discovering queries.
*/
export interface QueryDiscoveryResults {
/**
* A tree of directories and query files.
* May have multiple roots because of multiple workspaces.
*/
queries: Array<FileTreeDirectory<string>>;
const QUERY_FILE_EXTENSION = ".ql";
/**
* File system paths to watch. If any ql file changes in these directories
* or any subdirectories, then this could signify a change in queries.
*/
watchPaths: Uri[];
export interface QueryPackDiscoverer {
getLanguageForQueryFile(queryPath: string): string | undefined;
onDidChangeQueryPacks: Event<void>;
}
interface Query {
path: string;
language: string | undefined;
}
/**
* Discovers all query files contained in the QL packs in a given workspace folder.
* Discovers all query files in the workspace.
*/
export class QueryDiscovery extends Discovery implements QueryDiscoverer {
private results: Array<FileTreeDirectory<string>> | undefined;
private readonly onDidChangeQueriesEmitter: AppEventEmitter<void>;
private readonly watcher: MultiFileSystemWatcher = this.push(
new MultiFileSystemWatcher(),
);
export class QueryDiscovery
extends FilePathDiscovery<Query>
implements QueryDiscoverer
{
constructor(
private readonly env: EnvironmentContext,
private readonly cliServer: CodeQLCliServer,
private readonly queryPackDiscovery: QueryPackDiscoverer,
) {
super("Query Discovery", extLogger);
super("Query Discovery", `**/*${QUERY_FILE_EXTENSION}`);
this.onDidChangeQueriesEmitter = this.push(new EventEmitter<void>());
this.push(workspace.onDidChangeWorkspaceFolders(this.refresh.bind(this)));
this.push(this.watcher.onDidChange(this.refresh.bind(this)));
}
public get queries(): Array<FileTreeDirectory<string>> | undefined {
return this.results;
this.push(
this.queryPackDiscovery.onDidChangeQueryPacks(
this.recomputeAllQueryLanguages.bind(this),
),
);
}
/**
* Event to be fired when the set of discovered queries may have changed.
* Event that fires when the set of queries in the workspace changes.
*/
public get onDidChangeQueries(): Event<void> {
return this.onDidChangeQueriesEmitter.event;
}
protected async discover() {
const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
this.results = await this.discoverQueries(workspaceFolders);
this.watcher.clear();
for (const watchPath of workspaceFolders.map((f) => f.uri)) {
// Watch for changes to any `.ql` file
this.watcher.addWatch(new RelativePattern(watchPath, "**/*.{ql}"));
// need to explicitly watch for changes to directories themselves.
this.watcher.addWatch(new RelativePattern(watchPath, "**/"));
}
this.onDidChangeQueriesEmitter.fire();
return this.onDidChangePathsEmitter.event;
}
/**
* Discover all queries in the specified directory and its subdirectories.
* @returns A `QueryDirectory` object describing the contents of the directory, or `undefined` if
* no queries were found.
* Return all known queries, represented as a tree.
*
* Trivial directories where there is only one child will be collapsed into a single node.
*/
private async discoverQueries(
workspaceFolders: readonly WorkspaceFolder[],
): Promise<Array<FileTreeDirectory<string>>> {
const rootDirectories = [];
for (const workspaceFolder of workspaceFolders) {
const root = await this.discoverQueriesInWorkspace(workspaceFolder);
if (root !== undefined) {
rootDirectories.push(root);
public buildQueryTree(): Array<FileTreeNode<string>> {
const roots = [];
for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
const queriesInRoot = this.paths.filter((query) =>
containsPath(workspaceFolder.uri.fsPath, query.path),
);
if (queriesInRoot.length > 0) {
const root = new FileTreeDirectory<string>(
workspaceFolder.uri.fsPath,
workspaceFolder.name,
this.env,
);
for (const query of queriesInRoot) {
const dirName = dirname(normalize(relative(root.path, query.path)));
const parentDirectory = root.createDirectory(dirName);
parentDirectory.addChild(
new FileTreeLeaf<string>(
query.path,
basename(query.path),
query.language,
),
);
}
root.finish();
roots.push(root);
}
}
return rootDirectories;
return roots;
}
private async discoverQueriesInWorkspace(
workspaceFolder: WorkspaceFolder,
): Promise<FileTreeDirectory<string> | undefined> {
const fullPath = workspaceFolder.uri.fsPath;
const name = workspaceFolder.name;
protected async getDataForPath(path: string): Promise<Query> {
const language = this.determineQueryLanguage(path);
return { path, language };
}
// We don't want to log each invocation of resolveQueries, since it clutters up the log.
const silent = true;
const resolvedQueries = await this.cliServer.resolveQueries(
fullPath,
silent,
);
if (resolvedQueries.length === 0) {
return undefined;
protected pathIsRelevant(path: string): boolean {
return path.endsWith(QUERY_FILE_EXTENSION);
}
protected shouldOverwriteExistingData(
newData: Query,
existingData: Query,
): boolean {
return newData.language !== existingData.language;
}
private recomputeAllQueryLanguages() {
// All we know is that something has changed in the set of known query packs.
// We have no choice but to recompute the language for all queries.
for (const query of this.paths) {
query.language = this.determineQueryLanguage(query.path);
}
this.onDidChangePathsEmitter.fire();
}
const rootDirectory = new FileTreeDirectory<string>(
fullPath,
name,
this.env,
);
for (const queryPath of resolvedQueries) {
const relativePath = normalize(relative(fullPath, queryPath));
const dirName = dirname(relativePath);
const parentDirectory = rootDirectory.createDirectory(dirName);
parentDirectory.addChild(
new FileTreeLeaf<string>(queryPath, basename(queryPath), "language"),
);
}
rootDirectory.finish();
return rootDirectory;
private determineQueryLanguage(path: string): string | undefined {
return this.queryPackDiscovery.getLanguageForQueryFile(path);
}
}

View File

@@ -4,7 +4,7 @@ import { DisposableObject } from "../pure/disposable-object";
import { FileTreeNode } from "../common/file-tree-nodes";
export interface QueryDiscoverer {
readonly queries: Array<FileTreeNode<string>> | undefined;
readonly buildQueryTree: () => Array<FileTreeNode<string>>;
readonly onDidChangeQueries: Event<void>;
}
@@ -34,9 +34,9 @@ export class QueryTreeDataProvider
}
private createTree(): QueryTreeViewItem[] {
return (this.queryDiscoverer.queries || []).map(
this.convertFileTreeNode.bind(this),
);
return this.queryDiscoverer
.buildQueryTree()
.map(this.convertFileTreeNode.bind(this));
}
private convertFileTreeNode(

View File

@@ -1,203 +1,190 @@
import { EventEmitter, Uri, workspace } from "vscode";
import {
EventEmitter,
FileSystemWatcher,
Uri,
WorkspaceFoldersChangeEvent,
workspace,
} from "vscode";
import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import { QueryDiscovery } from "../../../../src/queries-panel/query-discovery";
QueryDiscovery,
QueryPackDiscoverer,
} from "../../../../src/queries-panel/query-discovery";
import { createMockEnvironmentContext } from "../../../__mocks__/appMock";
import { mockedObject } from "../../utils/mocking.helpers";
import { basename, join, sep } from "path";
import { dirname, join } from "path";
import * as tmp from "tmp";
import {
FileTreeDirectory,
FileTreeLeaf,
} from "../../../../src/common/file-tree-nodes";
import { mkdirSync, writeFileSync } from "fs";
describe("Query pack discovery", () => {
let tmpDir: string;
let tmpDirRemoveCallback: (() => void) | undefined;
let workspacePath: string;
const env = createMockEnvironmentContext();
const onDidChangeQueryPacks = new EventEmitter<void>();
let queryPackDiscoverer: QueryPackDiscoverer;
let discovery: QueryDiscovery;
describe("QueryDiscovery", () => {
beforeEach(() => {
expect(workspace.workspaceFolders?.length).toEqual(1);
const t = tmp.dirSync();
tmpDir = t.name;
tmpDirRemoveCallback = t.removeCallback;
const workspaceFolder = {
uri: Uri.file(join(tmpDir, "workspace")),
name: "workspace",
index: 0,
};
workspacePath = workspaceFolder.uri.fsPath;
jest
.spyOn(workspace, "workspaceFolders", "get")
.mockReturnValue([workspaceFolder]);
queryPackDiscoverer = {
getLanguageForQueryFile: () => "java",
onDidChangeQueryPacks: onDidChangeQueryPacks.event,
};
discovery = new QueryDiscovery(env, queryPackDiscoverer);
});
describe("queries", () => {
it("should return empty list when no QL files are present", async () => {
const resolveQueries = jest.fn().mockResolvedValue([]);
const cli = mockedObject<CodeQLCliServer>({
resolveQueries,
});
afterEach(() => {
tmpDirRemoveCallback?.();
discovery.dispose();
});
const discovery = new QueryDiscovery(createMockEnvironmentContext(), cli);
await discovery.refresh();
const queries = discovery.queries;
describe("buildQueryTree", () => {
it("returns an empty tree when there are no query files", async () => {
await discovery.initialRefresh();
expect(queries).toEqual([]);
expect(resolveQueries).toHaveBeenCalledTimes(1);
expect(discovery.buildQueryTree()).toEqual([]);
});
it("handles when query pack data is available", async () => {
makeTestFile(join(workspacePath, "query.ql"));
await discovery.initialRefresh();
expect(discovery.buildQueryTree()).toEqual([
new FileTreeDirectory(workspacePath, "workspace", env, [
new FileTreeLeaf(join(workspacePath, "query.ql"), "query.ql", "java"),
]),
]);
});
it("handles when query pack data is not available", async () => {
makeTestFile(join(workspacePath, "query.ql"));
queryPackDiscoverer.getLanguageForQueryFile = () => undefined;
await discovery.initialRefresh();
expect(discovery.buildQueryTree()).toEqual([
new FileTreeDirectory(workspacePath, "workspace", env, [
new FileTreeLeaf(
join(workspacePath, "query.ql"),
"query.ql",
undefined,
),
]),
]);
});
it("should organise query files into directories", async () => {
const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath;
const cli = mockedObject<CodeQLCliServer>({
resolveQueries: jest
.fn()
.mockResolvedValue([
join(workspaceRoot, "dir1/query1.ql"),
join(workspaceRoot, "dir2/query2.ql"),
join(workspaceRoot, "query3.ql"),
makeTestFile(join(workspacePath, "dir1", "query1.ql"));
makeTestFile(join(workspacePath, "dir1", "query2.ql"));
makeTestFile(join(workspacePath, "dir2", "query3.ql"));
makeTestFile(join(workspacePath, "query4.ql"));
await discovery.initialRefresh();
expect(discovery.buildQueryTree()).toEqual([
new FileTreeDirectory(workspacePath, "workspace", env, [
new FileTreeDirectory(join(workspacePath, "dir1"), "dir1", env, [
new FileTreeLeaf(
join(workspacePath, "dir1", "query1.ql"),
"query1.ql",
"java",
),
new FileTreeLeaf(
join(workspacePath, "dir1", "query2.ql"),
"query2.ql",
"java",
),
]),
});
const discovery = new QueryDiscovery(createMockEnvironmentContext(), cli);
await discovery.refresh();
const queries = discovery.queries;
expect(queries).toBeDefined();
expect(queries![0].children.length).toEqual(3);
expect(queries![0].children[0].name).toEqual("dir1");
expect(queries![0].children[0].children.length).toEqual(1);
expect(queries![0].children[0].children[0].name).toEqual("query1.ql");
expect(queries![0].children[1].name).toEqual("dir2");
expect(queries![0].children[1].children.length).toEqual(1);
expect(queries![0].children[1].children[0].name).toEqual("query2.ql");
expect(queries![0].children[2].name).toEqual("query3.ql");
new FileTreeDirectory(join(workspacePath, "dir2"), "dir2", env, [
new FileTreeLeaf(
join(workspacePath, "dir2", "query3.ql"),
"query3.ql",
"java",
),
]),
new FileTreeLeaf(
join(workspacePath, "query4.ql"),
"query4.ql",
"java",
),
]),
]);
});
it("should collapse directories containing only a single element", async () => {
const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath;
const cli = mockedObject<CodeQLCliServer>({
resolveQueries: jest
.fn()
.mockResolvedValue([
join(workspaceRoot, "dir1/query1.ql"),
join(workspaceRoot, "dir1/dir2/dir3/dir3/query2.ql"),
]),
});
makeTestFile(join(workspacePath, "query1.ql"));
makeTestFile(join(workspacePath, "foo", "bar", "baz", "query2.ql"));
const discovery = new QueryDiscovery(createMockEnvironmentContext(), cli);
await discovery.refresh();
const queries = discovery.queries;
expect(queries).toBeDefined();
await discovery.initialRefresh();
expect(queries![0].children.length).toEqual(1);
expect(queries![0].children[0].name).toEqual("dir1");
expect(queries![0].children[0].children.length).toEqual(2);
expect(queries![0].children[0].children[0].name).toEqual(
"dir2 / dir3 / dir3",
);
expect(queries![0].children[0].children[0].children.length).toEqual(1);
expect(queries![0].children[0].children[0].children[0].name).toEqual(
"query2.ql",
);
expect(queries![0].children[0].children[1].name).toEqual("query1.ql");
});
it("calls resolveQueries once for each workspace folder", async () => {
const workspaceRoots = [
`${sep}workspace1`,
`${sep}workspace2`,
`${sep}workspace3`,
];
jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValueOnce(
workspaceRoots.map((root, index) => ({
uri: Uri.file(root),
name: basename(root),
index,
})),
);
const resolveQueries = jest.fn().mockImplementation((queryDir) => {
const workspaceIndex = workspaceRoots.indexOf(queryDir);
if (workspaceIndex === -1) {
throw new Error("Unexpected workspace");
}
return Promise.resolve([
join(queryDir, `query${workspaceIndex + 1}.ql`),
]);
});
const cli = mockedObject<CodeQLCliServer>({
resolveQueries,
});
const discovery = new QueryDiscovery(createMockEnvironmentContext(), cli);
await discovery.refresh();
const queries = discovery.queries;
expect(queries).toBeDefined();
expect(queries!.length).toEqual(3);
expect(queries![0].children[0].name).toEqual("query1.ql");
expect(queries![1].children[0].name).toEqual("query2.ql");
expect(queries![2].children[0].name).toEqual("query3.ql");
expect(resolveQueries).toHaveBeenCalledTimes(3);
expect(discovery.buildQueryTree()).toEqual([
new FileTreeDirectory(workspacePath, "workspace", env, [
new FileTreeDirectory(
join(workspacePath, "foo", "bar", "baz"),
"foo / bar / baz",
env,
[
new FileTreeLeaf(
join(workspacePath, "foo", "bar", "baz", "query2.ql"),
"query2.ql",
"java",
),
],
),
new FileTreeLeaf(
join(workspacePath, "query1.ql"),
"query1.ql",
"java",
),
]),
]);
});
});
describe("onDidChangeQueries", () => {
it("should fire onDidChangeQueries when a watcher fires", async () => {
const onWatcherDidChangeEvent = new EventEmitter<Uri>();
const watcher: FileSystemWatcher = {
ignoreCreateEvents: false,
ignoreChangeEvents: false,
ignoreDeleteEvents: false,
onDidCreate: onWatcherDidChangeEvent.event,
onDidChange: onWatcherDidChangeEvent.event,
onDidDelete: onWatcherDidChangeEvent.event,
dispose: () => undefined,
};
const createFileSystemWatcherSpy = jest.spyOn(
workspace,
"createFileSystemWatcher",
);
createFileSystemWatcherSpy.mockReturnValue(watcher);
describe("recomputeAllQueryLanguages", () => {
it("should recompute the language of all query files", async () => {
makeTestFile(join(workspacePath, "query.ql"));
const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath;
const cli = mockedObject<CodeQLCliServer>({
resolveQueries: jest
.fn()
.mockResolvedValue([join(workspaceRoot, "query1.ql")]),
});
await discovery.initialRefresh();
const discovery = new QueryDiscovery(createMockEnvironmentContext(), cli);
expect(discovery.buildQueryTree()).toEqual([
new FileTreeDirectory(workspacePath, "workspace", env, [
new FileTreeLeaf(join(workspacePath, "query.ql"), "query.ql", "java"),
]),
]);
const onDidChangeQueriesSpy = jest.fn();
discovery.onDidChangeQueries(onDidChangeQueriesSpy);
queryPackDiscoverer.getLanguageForQueryFile = () => "python";
onDidChangeQueryPacks.fire();
await discovery.refresh();
expect(createFileSystemWatcherSpy).toHaveBeenCalledTimes(2);
expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1);
onWatcherDidChangeEvent.fire(workspace.workspaceFolders![0].uri);
await discovery.waitForCurrentRefresh();
expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
});
});
describe("onDidChangeWorkspaceFolders", () => {
it("should refresh when workspace folders change", async () => {
const onDidChangeWorkspaceFoldersEvent =
new EventEmitter<WorkspaceFoldersChangeEvent>();
jest
.spyOn(workspace, "onDidChangeWorkspaceFolders")
.mockImplementation(onDidChangeWorkspaceFoldersEvent.event);
const discovery = new QueryDiscovery(
createMockEnvironmentContext(),
mockedObject<CodeQLCliServer>({
resolveQueries: jest.fn().mockResolvedValue([]),
}),
);
const onDidChangeQueriesSpy = jest.fn();
discovery.onDidChangeQueries(onDidChangeQueriesSpy);
await discovery.refresh();
expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1);
onDidChangeWorkspaceFoldersEvent.fire({ added: [], removed: [] });
await discovery.waitForCurrentRefresh();
expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
expect(discovery.buildQueryTree()).toEqual([
new FileTreeDirectory(workspacePath, "workspace", env, [
new FileTreeLeaf(
join(workspacePath, "query.ql"),
"query.ql",
"python",
),
]),
]);
});
});
});
function makeTestFile(path: string) {
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, "");
}

View File

@@ -3,25 +3,13 @@ import {
FileTreeDirectory,
FileTreeLeaf,
} from "../../../../src/common/file-tree-nodes";
import {
QueryDiscoverer,
QueryTreeDataProvider,
} from "../../../../src/queries-panel/query-tree-data-provider";
import { QueryTreeDataProvider } from "../../../../src/queries-panel/query-tree-data-provider";
describe("QueryTreeDataProvider", () => {
describe("getChildren", () => {
it("returns no children when queries is undefined", async () => {
const dataProvider = new QueryTreeDataProvider({
queries: undefined,
onDidChangeQueries: jest.fn(),
});
expect(dataProvider.getChildren()).toEqual([]);
});
it("returns no children when there are no queries", async () => {
const dataProvider = new QueryTreeDataProvider({
queries: [],
buildQueryTree: () => [],
onDidChangeQueries: jest.fn(),
});
@@ -30,7 +18,7 @@ describe("QueryTreeDataProvider", () => {
it("converts FileTreeNode to QueryTreeViewItem", async () => {
const dataProvider = new QueryTreeDataProvider({
queries: [
buildQueryTree: () => [
new FileTreeDirectory<string>("dir1", "dir1", env, [
new FileTreeDirectory<string>("dir1/dir2", "dir2", env, [
new FileTreeLeaf<string>(
@@ -75,20 +63,21 @@ describe("QueryTreeDataProvider", () => {
describe("onDidChangeQueries", () => {
it("should update tree when the queries change", async () => {
const queryTree = [
new FileTreeDirectory<string>("dir1", "dir1", env, [
new FileTreeLeaf<string>("dir1/file1", "file1", "javascript"),
]),
];
const onDidChangeQueriesEmitter = new EventEmitter<void>();
const queryDiscoverer: QueryDiscoverer = {
queries: [
new FileTreeDirectory<string>("dir1", "dir1", env, [
new FileTreeLeaf<string>("dir1/file1", "file1", "javascript"),
]),
],
const queryDiscoverer = {
buildQueryTree: () => queryTree,
onDidChangeQueries: onDidChangeQueriesEmitter.event,
};
const dataProvider = new QueryTreeDataProvider(queryDiscoverer);
expect(dataProvider.getChildren().length).toEqual(1);
queryDiscoverer.queries?.push(
queryTree.push(
new FileTreeDirectory<string>("dir2", "dir2", env, [
new FileTreeLeaf<string>("dir2/file2", "file2", "javascript"),
]),