mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Move walkDirectory to pure files file

Browse Source
This commit is contained in:
Koen Vlaswinkel
2023-06-12 11:13:57 +02:00
parent 8c98401efe
commit bfead07592
6 changed files with 97 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/ql-vscode/src/codeql-cli/cli.ts
View File

@@ -22,11 +22,11 @@ import {
getErrorMessage,
getErrorStack,
} from "../pure/helpers-pure";
import { walkDirectory } from "../pure/files";
import { QueryMetadata, SortDirection } from "../pure/interface-types";
import { BaseLogger, Logger, ProgressReporter } from "../common";
import { CompilationMessage } from "../pure/legacy-messages";
import { sarifParser } from "../common/sarif-parser";
import { walkDirectory } from "../helpers";
import { App } from "../common/app";
import { QueryLanguage } from "../common/query-language";

31
extensions/ql-vscode/src/helpers.ts
View File

@@ -1,10 +1,4 @@
import {
ensureDirSync,
pathExists,
ensureDir,
writeFile,
opendir,
} from "fs-extra";
import { ensureDirSync, pathExists, ensureDir, writeFile } from "fs-extra";
import { join, dirname } from "path";
import { dirSync } from "tmp-promise";
import { Uri, window as Window, workspace, env, WorkspaceFolder } from "vscode";
@@ -455,29 +449,6 @@ export async function createTimestampFile(storagePath: string) {
await writeFile(timestampPath, Date.now().toString(), "utf8");
}
/**
* Recursively walk a directory and return the full path to all files found.
* Symbolic links are ignored.
*
* @param dir the directory to walk
*
* @return An iterator of the full path to all files recursively found in the directory.
*/
export async function* walkDirectory(
dir: string,
): AsyncIterableIterator<string> {
const seenFiles = new Set<string>();
for await (const d of await opendir(dir)) {
const entry = join(dir, d.name);
seenFiles.add(entry);
if (d.isDirectory()) {
yield* walkDirectory(entry);
} else if (d.isFile()) {
yield entry;
}
}
}
/**
* Returns the path of the first folder in the workspace.
* This is used to decide where to create skeleton QL packs.

25
extensions/ql-vscode/src/pure/files.ts
View File

@@ -1,4 +1,4 @@
import { pathExists, stat, readdir } from "fs-extra";
import { pathExists, stat, readdir, opendir } from "fs-extra";
import { join, resolve } from "path";
/**
@@ -88,3 +88,26 @@ export async function readDirFullPaths(path: string): Promise<string[]> {
const baseNames = await readdir(path);
return baseNames.map((baseName) => join(path, baseName));
}
/**
* Recursively walk a directory and return the full path to all files found.
* Symbolic links are ignored.
*
* @param dir the directory to walk
*
* @return An iterator of the full path to all files recursively found in the directory.
*/
export async function* walkDirectory(
dir: string,
): AsyncIterableIterator<string> {
const seenFiles = new Set<string>();
for await (const d of await opendir(dir)) {
const entry = join(dir, d.name);
seenFiles.add(entry);
if (d.isDirectory()) {
yield* walkDirectory(entry);
} else if (d.isFile()) {
yield entry;
}
}
}

68
extensions/ql-vscode/test/unit-tests/pure/files.test.ts
View File

@@ -6,7 +6,11 @@ import {
getDirectoryNamesInsidePath,
pathsEqual,
readDirFullPaths,
walkDirectory,
} from "../../../src/pure/files";
import { DirResult } from "tmp";
import * as tmp from "tmp";
import { ensureDirSync, symlinkSync, writeFileSync } from "fs-extra";
describe("files", () => {
const dataDir = join(__dirname, "../../data");
@@ -299,3 +303,67 @@ describe("containsPath", () => {
},
);
});
describe("walkDirectory", () => {
let tmpDir: DirResult;
let dir: string;
let dir2: string;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
dir = join(tmpDir.name, "dir");
ensureDirSync(dir);
dir2 = join(tmpDir.name, "dir2");
});
afterEach(() => {
tmpDir.removeCallback();
});
it("should walk a directory", async () => {
const file1 = join(dir, "file1");
const file2 = join(dir, "file2");
const file3 = join(dir, "file3");
const dir3 = join(dir, "dir3");
const file4 = join(dir, "file4");
const file5 = join(dir, "file5");
const file6 = join(dir, "file6");
// These symlinks link back to paths that are already existing, so ignore.
const symLinkFile7 = join(dir, "symlink0");
const symlinkDir = join(dir2, "symlink1");
// some symlinks that point outside of the base dir.
const file8 = join(tmpDir.name, "file8");
const file9 = join(dir2, "file8");
const symlinkDir2 = join(dir2, "symlink2");
const symlinkFile2 = join(dir2, "symlinkFile3");
ensureDirSync(dir2);
ensureDirSync(dir3);
writeFileSync(file1, "file1");
writeFileSync(file2, "file2");
writeFileSync(file3, "file3");
writeFileSync(file4, "file4");
writeFileSync(file5, "file5");
writeFileSync(file6, "file6");
writeFileSync(file8, "file8");
writeFileSync(file9, "file9");
// We don't really need to be testing all of these variants of symlinks,
// but it doesn't hurt, and will help us if we ever do decide to support them.
symlinkSync(file6, symLinkFile7, "file");
symlinkSync(dir3, symlinkDir, "dir");
symlinkSync(file8, symlinkFile2, "file");
symlinkSync(dir2, symlinkDir2, "dir");
const files = [];
for await (const file of walkDirectory(dir)) {
files.push(file);
}
// Only real files should be returned.
expect(files.sort()).toEqual([file1, file2, file3, file4, file5, file6]);
});
});

74
extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts
View File

@@ -1,14 +1,7 @@
import { Uri, window, workspace, WorkspaceFolder } from "vscode";
import * as tmp from "tmp";
import { join } from "path";
import {
writeFileSync,
ensureDirSync,
symlinkSync,
writeFile,
mkdir,
} from "fs-extra";
import { DirResult } from "tmp";
import { writeFile, mkdir } from "fs-extra";
import {
getFirstWorkspaceFolder,
@@ -18,7 +11,6 @@ import {
showBinaryChoiceWithUrlDialog,
showInformationMessageWithAction,
showNeverAskAgainDialog,
walkDirectory,
} from "../../../src/helpers";
import { reportStreamProgress } from "../../../src/common/vscode/progress";
import { Setting } from "../../../src/config";
@@ -243,70 +235,6 @@ describe("helpers", () => {
});
});
describe("walkDirectory", () => {
let tmpDir: DirResult;
let dir: string;
let dir2: string;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
dir = join(tmpDir.name, "dir");
ensureDirSync(dir);
dir2 = join(tmpDir.name, "dir2");
});
afterEach(() => {
tmpDir.removeCallback();
});
it("should walk a directory", async () => {
const file1 = join(dir, "file1");
const file2 = join(dir, "file2");
const file3 = join(dir, "file3");
const dir3 = join(dir, "dir3");
const file4 = join(dir, "file4");
const file5 = join(dir, "file5");
const file6 = join(dir, "file6");
// These symlinks link back to paths that are already existing, so ignore.
const symLinkFile7 = join(dir, "symlink0");
const symlinkDir = join(dir2, "symlink1");
// some symlinks that point outside of the base dir.
const file8 = join(tmpDir.name, "file8");
const file9 = join(dir2, "file8");
const symlinkDir2 = join(dir2, "symlink2");
const symlinkFile2 = join(dir2, "symlinkFile3");
ensureDirSync(dir2);
ensureDirSync(dir3);
writeFileSync(file1, "file1");
writeFileSync(file2, "file2");
writeFileSync(file3, "file3");
writeFileSync(file4, "file4");
writeFileSync(file5, "file5");
writeFileSync(file6, "file6");
writeFileSync(file8, "file8");
writeFileSync(file9, "file9");
// We don't really need to be testing all of these variants of symlinks,
// but it doesn't hurt, and will help us if we ever do decide to support them.
symlinkSync(file6, symLinkFile7, "file");
symlinkSync(dir3, symlinkDir, "dir");
symlinkSync(file8, symlinkFile2, "file");
symlinkSync(dir2, symlinkDir2, "dir");
const files = [];
for await (const file of walkDirectory(dir)) {
files.push(file);
}
// Only real files should be returned.
expect(files.sort()).toEqual([file1, file2, file3, file4, file5, file6]);
});
});
describe("isFolderAlreadyInWorkspace", () => {
beforeEach(() => {
const folders = [

3
extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/variant-analysis-history.test.ts
View File

@@ -10,7 +10,8 @@ import { join } from "path";
import { ExtensionContext, Uri } from "vscode";
import { DatabaseManager } from "../../../../src/databases/local-databases";
import { tmpDir, walkDirectory } from "../../../../src/helpers";
import { tmpDir } from "../../../../src/helpers";
import { walkDirectory } from "../../../../src/pure/files";
import { DisposableBucket } from "../../disposable-bucket";
import { testDisposeHandler } from "../../test-dispose-handler";
import { HistoryItemLabelProvider } from "../../../../src/query-history/history-item-label-provider";