mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into koesie10/yauzl-concurrent

Browse Source
This commit is contained in:
Koen Vlaswinkel
2023-12-20 10:10:38 +01:00
parent 7a1f157225 e824fda9e7
commit 4444951093
7 changed files with 223 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
extensions/ql-vscode/package-lock.json generated
View File

@@ -38,7 +38,6 @@
"tmp": "^0.2.1",
"tmp-promise": "^3.0.2",
"tree-kill": "^1.2.2",
"unzipper": "^0.10.5",
"vscode-extension-telemetry": "^0.1.6",
"vscode-jsonrpc": "^8.0.2",
"vscode-languageclient": "^8.0.2",
@@ -10293,6 +10292,7 @@
"version": "1.6.52",
"resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz",
"integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==",
"dev": true,
"engines": {
"node": ">=0.6"
}
@@ -10306,18 +10306,6 @@
"node": "*"
}
},
"node_modules/binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
"dependencies": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
},
"engines": {
"node": "*"
}
},
"node_modules/binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -10371,11 +10359,6 @@
"node": ">= 6"
}
},
"node_modules/bluebird": {
"version": "3.4.7",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA=="
},
"node_modules/body-parser": {
"version": "1.20.1",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@@ -10607,22 +10590,6 @@
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
},
"node_modules/buffer-indexof-polyfill": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz",
"integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==",
"engines": {
"node": ">=0.2.0"
}
},
"node_modules/bundle-name": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz",
@@ -10746,17 +10713,6 @@
"node": ">=4"
}
},
"node_modules/chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
"dependencies": {
"traverse": ">=0.3.0 <0.4"
},
"engines": {
"node": "*"
}
},
"node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -13097,14 +13053,6 @@
"node": ">=12"
}
},
"node_modules/duplexer2": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
"integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==",
"dependencies": {
"readable-stream": "^2.0.2"
}
},
"node_modules/duplexify": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@@ -15860,50 +15808,6 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/fstream": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
"integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
"dependencies": {
"graceful-fs": "^4.1.2",
"inherits": "~2.0.0",
"mkdirp": ">=0.5 0",
"rimraf": "2"
},
"engines": {
"node": ">=0.6"
}
},
"node_modules/fstream/node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/fstream/node_modules/rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"dependencies": {
"glob": "^7.1.3"
},
"bin": {
"rimraf": "bin.js"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -21876,11 +21780,6 @@
"node": ">= 8"
}
},
"node_modules/listenercount": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz",
"integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ=="
},
"node_modules/listr2": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/listr2/-/listr2-8.0.0.tgz",
@@ -23092,6 +22991,7 @@
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
"dev": true,
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -23152,6 +23052,7 @@
"version": "0.5.6",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
"integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
"dev": true,
"dependencies": {
"minimist": "^1.2.6"
},
@@ -26812,7 +26713,8 @@
"node_modules/setimmediate": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
"integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="
"integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==",
"dev": true
},
"node_modules/setprototypeof": {
"version": "1.2.0",
@@ -28512,14 +28414,6 @@
"node": ">=12"
}
},
"node_modules/traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==",
"engines": {
"node": "*"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@@ -29456,23 +29350,6 @@
"node": ">=8"
}
},
"node_modules/unzipper": {
"version": "0.10.14",
"resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz",
"integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==",
"dependencies": {
"big-integer": "^1.6.17",
"binary": "~0.3.0",
"bluebird": "~3.4.1",
"buffer-indexof-polyfill": "~1.0.0",
"duplexer2": "~0.1.4",
"fstream": "^1.0.12",
"graceful-fs": "^4.2.2",
"listenercount": "~1.0.1",
"readable-stream": "~2.3.6",
"setimmediate": "~1.0.4"
}
},
"node_modules/upath": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz",

1
extensions/ql-vscode/package.json
View File

@@ -1940,7 +1940,6 @@
"tmp": "^0.2.1",
"tmp-promise": "^3.0.2",
"tree-kill": "^1.2.2",
"unzipper": "^0.10.5",
"vscode-extension-telemetry": "^0.1.6",
"vscode-jsonrpc": "^8.0.2",
"vscode-languageclient": "^8.0.2",

6
extensions/ql-vscode/scripts/source-map.ts
View File

@@ -20,7 +20,7 @@ import { spawnSync } from "child_process";
import { basename, resolve } from "path";
import { pathExists, readJSON } from "fs-extra";
import { RawSourceMap, SourceMapConsumer } from "source-map";
import { Open } from "unzipper";
import { unzipToDirectorySequentially } from "../src/common/unzip";
if (process.argv.length !== 4) {
console.error(
@@ -78,10 +78,10 @@ async function extractSourceMap() {
releaseAssetsDirectory,
]);
const file = await Open.file(
await unzipToDirectorySequentially(
resolve(releaseAssetsDirectory, sourcemapAsset.name),
sourceMapsDirectory,
);
await file.extract({ path: sourceMapsDirectory });
} else {
const workflowRuns = runGhJSON<WorkflowRunListItem[]>([
"run",

7
extensions/ql-vscode/src/common/files.ts
View File

@@ -91,18 +91,23 @@ export async function readDirFullPaths(path: string): Promise<string[]> {
* Symbolic links are ignored.
*
* @param dir the directory to walk
* @param includeDirectories whether to include directories in the results
*
* @return An iterator of the full path to all files recursively found in the directory.
*/
export async function* walkDirectory(
dir: string,
includeDirectories = false,
): AsyncIterableIterator<string> {
const seenFiles = new Set<string>();
for await (const d of await opendir(dir)) {
const entry = join(dir, d.name);
seenFiles.add(entry);
if (d.isDirectory()) {
yield* walkDirectory(entry);
if (includeDirectories) {
yield entry;
}
yield* walkDirectory(entry, includeDirectories);
} else if (d.isFile()) {
yield entry;
}

7
extensions/ql-vscode/src/common/unzip.ts
View File

@@ -31,12 +31,7 @@ export function readZipEntries(zipFile: ZipFile): Promise<ZipEntry[]> {
zipFile.readEntry();
zipFile.on("entry", (entry: ZipEntry) => {
if (/\/$/.test(entry.fileName)) {
// Directory file names end with '/'
// We don't need to do anything for directories.
} else {
files.push(entry);
}
files.push(entry);
zipFile.readEntry();
});

208
extensions/ql-vscode/test/unit-tests/common/unzip.test.ts Normal file
View File

@@ -0,0 +1,208 @@
import { createHash } from "crypto";
import { open } from "fs/promises";
import { join, relative, resolve, sep } from "path";
import { chmod, pathExists, readdir } from "fs-extra";
import { dir, DirectoryResult } from "tmp-promise";
import {
excludeDirectories,
openZip,
openZipBuffer,
readZipEntries,
unzipToDirectorySequentially,
} from "../../../src/common/unzip";
import { walkDirectory } from "../../../src/common/files";
import { unzipToDirectoryConcurrently } from "../../../src/common/unzip-concurrently";
const zipPath = resolve(__dirname, "../data/unzip/test-zip.zip");
describe("openZip", () => {
it("can open a zip file", async () => {
const zipFile = await openZip(zipPath, {
lazyEntries: false,
});
expect(zipFile.entryCount).toEqual(11);
});
});
describe("readZipEntries", () => {
it("can read the entries when there are multiple directories", async () => {
const zipFile = await openZip(zipPath, {
lazyEntries: true,
});
const entries = await readZipEntries(zipFile);
expect(entries.map((entry) => entry.fileName).sort()).toEqual([
"directory/",
"directory/file.txt",
"directory/file2.txt",
"directory2/",
"directory2/file.txt",
"empty-directory/",
"tools/",
"tools/osx64/",
"tools/osx64/java-aarch64/",
"tools/osx64/java-aarch64/bin/",
"tools/osx64/java-aarch64/bin/java",
]);
});
});
describe("excludeDirectories", () => {
it("excludes directories", async () => {
const zipFile = await openZip(zipPath, {
lazyEntries: true,
});
const entries = await readZipEntries(zipFile);
const entriesWithoutDirectories = excludeDirectories(entries);
expect(
entriesWithoutDirectories.map((entry) => entry.fileName).sort(),
).toEqual([
"directory/file.txt",
"directory/file2.txt",
"directory2/file.txt",
"tools/osx64/java-aarch64/bin/java",
]);
});
});
describe("openZipBuffer", () => {
it("can read an entry in the zip file", async () => {
const zipFile = await openZip(zipPath, {
lazyEntries: true,
autoClose: false,
});
const entries = await readZipEntries(zipFile);
const entry = entries.find(
(entry) => entry.fileName === "directory/file.txt",
);
expect(entry).toBeDefined();
if (!entry) {
return;
}
const buffer = await openZipBuffer(zipFile, entry);
expect(buffer).toHaveLength(13);
expect(buffer.toString("utf8")).toEqual("I am a file\n\n");
});
});
describe.each([
{
name: "unzipToDirectorySequentially",
unzipToDirectory: unzipToDirectorySequentially,
},
{
name: "unzipToDirectoryConcurrently",
unzipToDirectory: unzipToDirectoryConcurrently,
},
])("$name", ({ unzipToDirectory }) => {
let tmpDir: DirectoryResult;
beforeEach(async () => {
tmpDir = await dir({
unsafeCleanup: true,
});
});
afterEach(async () => {
for await (const file of walkDirectory(tmpDir.path)) {
await chmod(file, 0o777);
}
await tmpDir?.cleanup();
});
it("extracts all files", async () => {
await unzipToDirectory(zipPath, tmpDir.path);
const allFiles = [];
for await (const file of walkDirectory(tmpDir.path, true)) {
allFiles.push(file);
}
expect(
allFiles
.map((filePath) => relative(tmpDir.path, filePath).split(sep).join("/"))
.sort(),
).toEqual([
"directory",
"directory/file.txt",
"directory/file2.txt",
"directory2",
"directory2/file.txt",
"empty-directory",
"tools",
"tools/osx64",
"tools/osx64/java-aarch64",
"tools/osx64/java-aarch64/bin",
"tools/osx64/java-aarch64/bin/java",
]);
await expectFile(join(tmpDir.path, "directory", "file.txt"), {
mode: 0o100644,
contents: "I am a file\n\n",
});
await expectFile(join(tmpDir.path, "directory", "file2.txt"), {
mode: 0o100644,
contents: "I am another file\n\n",
});
await expectFile(join(tmpDir.path, "directory2", "file.txt"), {
mode: 0o100600,
contents: "I am secret\n",
});
await expectFile(
join(tmpDir.path, "tools", "osx64", "java-aarch64", "bin", "java"),
{
mode: 0o100755,
hash: "68b832b5c0397c5baddbbb0a76cf5407b7ea5eee8f84f9ab9488f04a52e529eb",
},
);
expect(await pathExists(join(tmpDir.path, "empty-directory"))).toBe(true);
expect(await readdir(join(tmpDir.path, "empty-directory"))).toEqual([]);
});
});
async function expectFile(
filePath: string,
{
mode: expectedMode,
hash: expectedHash,
contents: expectedContents,
}: {
mode: number;
hash?: string;
contents?: string;
},
) {
const file = await open(filePath, "r");
// Windows doesn't really support file modes
if (process.platform !== "win32") {
const stats = await file.stat();
expect(stats.mode).toEqual(expectedMode);
}
const contents = await file.readFile();
if (expectedHash) {
const hash = await computeHash(contents);
expect(hash).toEqual(expectedHash);
}
if (expectedContents) {
expect(contents.toString("utf-8")).toEqual(expectedContents);
}
await file.close();
}
async function computeHash(contents: Buffer) {
const hash = createHash("sha256");
hash.update(contents);
return hash.digest("hex");
}

BIN
extensions/ql-vscode/test/unit-tests/data/unzip/test-zip.zip Normal file
View File

Binary file not shown.