mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into koesie10/cleanup-distributions

Browse Source
This commit is contained in:
Koen Vlaswinkel
2024-10-22 15:21:57 +02:00
parent 408df85a38 f19b0dfc8b
commit afdef0029e
8 changed files with 939 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/CHANGELOG.md
View File

@@ -3,6 +3,7 @@
## [UNRELEASED]
- Support result columns of type `QlBuiltins::BigInt` in quick evaluations. [#3647](https://github.com/github/vscode-codeql/pull/3647)
- Fix a bug where the CodeQL CLI would be re-downloaded if you switched to a different filesystem (for example Codespaces or a remote SSH host). [#3762](https://github.com/github/vscode-codeql/pull/3762)
## 1.16.0 - 10 October 2024

665
extensions/ql-vscode/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

14
extensions/ql-vscode/package.json
View File

@@ -1963,7 +1963,7 @@
"generate": "npm-run-all -p generate:*",
"generate:schemas": "vite-node scripts/generate-schemas.ts",
"generate:chromium-version": "vite-node scripts/generate-chromium-version.ts",
"check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" -not -path \"./.vscode-test/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
"check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" -not -path \"*/.vscode-test/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
"postinstall": "patch-package",
"prepare": "cd ../.. && husky"
},
@@ -1986,6 +1986,7 @@
"msw": "^2.2.13",
"nanoid": "^5.0.7",
"p-queue": "^8.0.1",
"proper-lockfile": "^4.1.2",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"semver": "^7.6.2",
@@ -2026,7 +2027,7 @@
"@storybook/react-vite": "^8.3.5",
"@storybook/theming": "^8.2.4",
"@testing-library/dom": "^10.4.0",
"@testing-library/jest-dom": "^6.5.0",
"@testing-library/jest-dom": "^6.6.1",
"@testing-library/react": "^16.0.1",
"@testing-library/user-event": "^14.5.2",
"@types/child-process-promise": "^2.2.1",
@@ -2040,6 +2041,7 @@
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "20.16.*",
"@types/proper-lockfile": "^4.1.4",
"@types/react": "^18.3.1",
"@types/react-dom": "^18.3.0",
"@types/sarif": "^2.1.2",
@@ -2051,8 +2053,8 @@
"@types/tmp": "^0.2.6",
"@types/vscode": "1.90.0",
"@types/yauzl": "^2.10.3",
"@typescript-eslint/eslint-plugin": "^8.8.1",
"@typescript-eslint/parser": "^8.8.1",
"@typescript-eslint/eslint-plugin": "^8.9.0",
"@typescript-eslint/parser": "^8.9.0",
"@vscode/test-electron": "^2.3.9",
"@vscode/vsce": "^2.24.0",
"ansi-colors": "^4.1.1",
@@ -2066,10 +2068,10 @@
"eslint-plugin-deprecation": "^3.0.0",
"eslint-plugin-etc": "^2.0.2",
"eslint-plugin-github": "^5.0.1",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-import": "^2.31.0",
"eslint-plugin-jest-dom": "^5.4.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.34.1",
"eslint-plugin-react": "^7.37.1",
"eslint-plugin-react-hooks": "^4.6.2",
"eslint-plugin-storybook": "^0.8.0",
"glob": "^11.0.0",

167
extensions/ql-vscode/src/codeql-cli/distribution.ts
View File

@@ -1,5 +1,12 @@
import type { WriteStream } from "fs";
import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
import {
createWriteStream,
mkdtemp,
outputJson,
pathExists,
readJson,
remove,
} from "fs-extra";
import { tmpdir } from "os";
import { delimiter, dirname, join } from "path";
import { Range, satisfies } from "semver";
@@ -21,6 +28,7 @@ import {
} from "../common/invocation-rate-limiter";
import type { NotificationLogger } from "../common/logging";
import {
showAndLogExceptionWithTelemetry,
showAndLogErrorMessage,
showAndLogWarningMessage,
} from "../common/logging";
@@ -29,6 +37,11 @@ import { reportUnzipProgress } from "../common/vscode/unzip-progress";
import type { Release } from "./distribution/release";
import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";
import { createTimeoutSignal } from "../common/fetch-stream";
import { withDistributionUpdateLock } from "./lock";
import { asError, getErrorMessage } from "../common/helpers-pure";
import { isIOError } from "../common/files";
import { telemetryListener } from "../common/vscode/telemetry";
import { redactableError } from "../common/errors";
import { ExtensionManagedDistributionCleaner } from "./distribution/cleaner";
/**
@@ -55,6 +68,11 @@ const NIGHTLY_DISTRIBUTION_REPOSITORY_NWO = "dsp-testing/codeql-cli-nightlies";
*/
export const DEFAULT_DISTRIBUTION_VERSION_RANGE: Range = new Range("2.x");
export interface DistributionState {
folderIndex: number;
release: Release | null;
}
export interface DistributionProvider {
getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
onDidChangeDistribution?: Event<void>;
@@ -74,6 +92,7 @@ export class DistributionManager implements DistributionProvider {
config,
versionRange,
extensionContext,
logger,
);
this.updateCheckRateLimiter = new InvocationRateLimiter(
extensionContext.globalState,
@@ -89,6 +108,10 @@ export class DistributionManager implements DistributionProvider {
);
}
public async initialize(): Promise<void> {
await this.extensionSpecificDistributionManager.initialize();
}
/**
* Look up a CodeQL launcher binary.
*/
@@ -294,14 +317,58 @@ export class DistributionManager implements DistributionProvider {
}
class ExtensionSpecificDistributionManager {
private distributionState: DistributionState | undefined;
constructor(
private readonly config: DistributionConfig,
private readonly versionRange: Range,
private readonly extensionContext: ExtensionContext,
private readonly logger: NotificationLogger,
) {
/**/
}
public async initialize() {
await this.ensureDistributionStateExists();
}
private async ensureDistributionStateExists() {
const distributionStatePath = this.getDistributionStatePath();
try {
this.distributionState = await readJson(distributionStatePath);
} catch (e: unknown) {
if (isIOError(e) && e.code === "ENOENT") {
// If the file doesn't exist, that just means we need to create it
this.distributionState = {
folderIndex:
this.extensionContext.globalState.get(
"distributionFolderIndex",
0,
) ?? 0,
release: (this.extensionContext.globalState.get(
"distributionRelease",
) ?? null) as Release | null,
};
// This may result in a race condition, but when this happens both processes should write the same file.
await outputJson(distributionStatePath, this.distributionState);
} else {
void showAndLogExceptionWithTelemetry(
this.logger,
telemetryListener,
redactableError(
asError(e),
)`Failed to read distribution state from ${distributionStatePath}: ${getErrorMessage(e)}`,
);
this.distributionState = {
folderIndex: 0,
release: null,
};
}
}
}
public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
if (this.getInstalledRelease() !== undefined) {
// An extension specific distribution has been installed.
@@ -364,9 +431,21 @@ class ExtensionSpecificDistributionManager {
release: Release,
progressCallback?: ProgressCallback,
): Promise<void> {
await this.downloadDistribution(release, progressCallback);
// Store the installed release within the global extension state.
await this.storeInstalledRelease(release);
if (!this.distributionState) {
await this.ensureDistributionStateExists();
}
const distributionStatePath = this.getDistributionStatePath();
await withDistributionUpdateLock(
// .lock will be appended to this filename
distributionStatePath,
async () => {
await this.downloadDistribution(release, progressCallback);
// Store the installed release within the global extension state.
await this.storeInstalledRelease(release);
},
);
}
private async downloadDistribution(
@@ -578,23 +657,19 @@ class ExtensionSpecificDistributionManager {
}
private async bumpDistributionFolderIndex(): Promise<void> {
const index = this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
0,
);
await this.extensionContext.globalState.update(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
index + 1,
);
await this.updateState((oldState) => {
return {
...oldState,
folderIndex: (oldState.folderIndex ?? 0) + 1,
};
});
}
private getDistributionStoragePath(): string {
const distributionState = this.getDistributionState();
// Use an empty string for the initial distribution for backwards compatibility.
const distributionFolderIndex =
this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
0,
) || "";
const distributionFolderIndex = distributionState.folderIndex || "";
return join(
this.extensionContext.globalStorageUri.fsPath,
ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
@@ -609,28 +684,56 @@ class ExtensionSpecificDistributionManager {
);
}
private getInstalledRelease(): Release | undefined {
return this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._installedReleaseStateKey,
private getDistributionStatePath(): string {
return join(
this.extensionContext.globalStorageUri.fsPath,
ExtensionSpecificDistributionManager._distributionStateFilename,
);
}
private getInstalledRelease(): Release | undefined {
return this.getDistributionState().release ?? undefined;
}
private async storeInstalledRelease(
release: Release | undefined,
): Promise<void> {
await this.extensionContext.globalState.update(
ExtensionSpecificDistributionManager._installedReleaseStateKey,
release,
);
await this.updateState((oldState) => ({
...oldState,
release: release ?? null,
}));
}
private getDistributionState(): DistributionState {
const distributionState = this.distributionState;
if (distributionState === undefined) {
throw new Error(
"Invariant violation: distribution state not initialized",
);
}
return distributionState;
}
private async updateState(
f: (oldState: DistributionState) => DistributionState,
) {
const oldState = this.distributionState;
if (oldState === undefined) {
throw new Error(
"Invariant violation: distribution state not initialized",
);
}
const newState = f(oldState);
this.distributionState = newState;
const distributionStatePath = this.getDistributionStatePath();
await outputJson(distributionStatePath, newState);
}
public get folderIndex() {
return (
this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
0,
) ?? 0
);
const distributionState = this.getDistributionState();
return distributionState.folderIndex;
}
public get distributionFolderPrefix() {
@@ -638,10 +741,8 @@ class ExtensionSpecificDistributionManager {
}
private static readonly _currentDistributionFolderBaseName = "distribution";
private static readonly _currentDistributionFolderIndexStateKey =
"distributionFolderIndex";
private static readonly _installedReleaseStateKey = "distributionRelease";
private static readonly _codeQlExtractedFolderName = "codeql";
private static readonly _distributionStateFilename = "distribution.json";
}
/*

22
extensions/ql-vscode/src/codeql-cli/lock.ts Normal file
View File

@@ -0,0 +1,22 @@
import { lock } from "proper-lockfile";
export async function withDistributionUpdateLock(
lockFile: string,
f: () => Promise<void>,
) {
const release = await lock(lockFile, {
stale: 60_000, // 1 minute. We can take the lock longer than this because that's based on the update interval.
update: 10_000, // 10 seconds
retries: {
minTimeout: 10_000,
maxTimeout: 60_000,
retries: 100,
},
});
try {
await f();
} finally {
await release();
}
}

1
extensions/ql-vscode/src/extension.ts
View File

@@ -364,6 +364,7 @@ export async function activate(
ctx,
app.logger,
);
await distributionManager.initialize();
registerErrorStubs([checkForUpdatesCommand], (command) => async () => {
void showAndLogErrorMessage(

2
extensions/ql-vscode/supported_cli_versions.json
View File

@@ -1,5 +1,5 @@
[
"v2.19.1",
"v2.19.2",
"v2.18.4",
"v2.17.6",
"v2.16.6",

365
extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts
View File

@@ -1,12 +1,21 @@
import * as log from "../../../../src/common/logging/notifications";
import { extLogger } from "../../../../src/common/logging/vscode";
import { writeFile } from "fs-extra";
import {
outputFile,
outputJson,
readFile,
readJson,
writeFile,
} from "fs-extra";
import { join } from "path";
import * as os from "os";
import type { DirectoryResult } from "tmp-promise";
import { dir } from "tmp-promise";
import type { DistributionState } from "../../../../src/codeql-cli/distribution";
import {
DEFAULT_DISTRIBUTION_VERSION_RANGE,
DistributionManager,
DistributionUpdateCheckResultKind,
getExecutableFromDirectory,
} from "../../../../src/codeql-cli/distribution";
import type {
@@ -14,6 +23,19 @@ import type {
showAndLogWarningMessage,
} from "../../../../src/common/logging";
import { createMockLogger } from "../../../__mocks__/loggerMock";
import { mockedObject } from "../../../mocked-object";
import type { DistributionConfig } from "../../../../src/config";
import type { ExtensionContext } from "vscode";
import { Uri } from "vscode";
import { setupServer } from "msw/node";
import { http, HttpResponse } from "msw";
import {
codeQlLauncherName,
getRequiredAssetName,
} from "../../../../src/common/distribution";
import type { GithubRelease } from "../../../../src/codeql-cli/distribution/releases-api-consumer";
import type { Release } from "../../../../src/codeql-cli/distribution/release";
import { zip } from "zip-a-folder";
jest.mock("os", () => {
const original = jest.requireActual("os");
@@ -155,3 +177,344 @@ describe("Launcher path", () => {
expect(errorSpy).toHaveBeenCalledTimes(1);
});
});
describe("Distribution updates", () => {
const server = setupServer();
beforeAll(() =>
server.listen({
onUnhandledRequest: "error",
}),
);
afterEach(() => server.resetHandlers());
afterAll(() => server.close());
let manager: DistributionManager;
let globalStorageDirectory: DirectoryResult;
beforeEach(async () => {
globalStorageDirectory = await dir({
unsafeCleanup: true,
});
manager = new DistributionManager(
mockedObject<DistributionConfig>({
customCodeQlPath: undefined,
channel: "stable",
includePrerelease: false,
personalAccessToken: undefined,
downloadTimeout: 100,
onDidChangeConfiguration: () => {},
}),
DEFAULT_DISTRIBUTION_VERSION_RANGE,
mockedObject<ExtensionContext>({
globalState: {
get: () => {},
update: () => {},
},
globalStorageUri: Uri.file(globalStorageDirectory.path),
}),
createMockLogger(),
);
await manager.initialize();
});
afterEach(async () => {
await globalStorageDirectory.cleanup();
});
it("should have an empty distribution.json file after initialization", async () => {
expect(
await readJson(join(globalStorageDirectory.path, "distribution.json")),
).toEqual({
folderIndex: 0,
release: null,
} satisfies DistributionState);
});
describe("checkForUpdatesToDistribution", () => {
beforeEach(() => {
server.resetHandlers(
http.get(
"https://api.github.com/repos/github/codeql-cli-binaries/releases",
async () => {
return HttpResponse.json([
{
id: 1335,
name: "v2.2.0",
tag_name: "v2.2.0",
created_at: "2024-02-02T02:02:02Z",
prerelease: false,
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
{
id: 1,
name: "v2.1.0",
tag_name: "v2.1.0",
created_at: "2022-02-02T02:02:02Z",
prerelease: false,
assets: [
{
id: 1,
name: getRequiredAssetName(),
size: 100,
},
],
},
] satisfies GithubRelease[]);
},
),
);
});
it("should have an update when no distribution is installed", async () => {
expect(
await manager.checkForUpdatesToExtensionManagedDistribution(0),
).toEqual({
kind: DistributionUpdateCheckResultKind.UpdateAvailable,
updatedRelease: {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
} satisfies Awaited<
ReturnType<typeof manager.checkForUpdatesToExtensionManagedDistribution>
>);
});
it("should not have an update when the latest distribution is installed", async () => {
await outputJson(join(globalStorageDirectory.path, "distribution.json"), {
folderIndex: 1,
release: {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
} satisfies DistributionState);
await outputFile(
join(
globalStorageDirectory.path,
"distribution1",
"codeql",
codeQlLauncherName(),
),
"",
);
// Re-initialize manager to read the state from the file
await manager.initialize();
expect(
await manager.checkForUpdatesToExtensionManagedDistribution(0),
).toEqual({
kind: DistributionUpdateCheckResultKind.AlreadyUpToDate,
} satisfies Awaited<
ReturnType<typeof manager.checkForUpdatesToExtensionManagedDistribution>
>);
});
it("should have an update when an older distribution is installed", async () => {
await outputJson(join(globalStorageDirectory.path, "distribution.json"), {
folderIndex: 1,
release: {
id: 1,
name: "v2.1.0",
createdAt: "2022-02-02T02:02:02Z",
assets: [
{
id: 1,
name: getRequiredAssetName(),
size: 100,
},
],
},
} satisfies DistributionState);
await outputFile(
join(
globalStorageDirectory.path,
"distribution1",
"codeql",
codeQlLauncherName(),
),
"",
);
// Re-initialize manager to read the state from the file
await manager.initialize();
expect(
await manager.checkForUpdatesToExtensionManagedDistribution(0),
).toEqual({
kind: DistributionUpdateCheckResultKind.UpdateAvailable,
updatedRelease: {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
} satisfies Awaited<
ReturnType<typeof manager.checkForUpdatesToExtensionManagedDistribution>
>);
});
});
describe("installExtensionManagedDistributionRelease", () => {
const release: Release = {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
};
let codeqlReleaseZipTempDir: DirectoryResult;
let codeqlReleaseZipPath: string;
beforeAll(async () => {
codeqlReleaseZipTempDir = await dir({
unsafeCleanup: true,
});
await outputFile(
join(
codeqlReleaseZipTempDir.path,
"distribution",
"codeql",
codeQlLauncherName(),
),
"launcher!",
);
codeqlReleaseZipPath = join(codeqlReleaseZipTempDir.path, "codeql.zip");
await zip(
join(codeqlReleaseZipTempDir.path, "distribution"),
codeqlReleaseZipPath,
);
server.resetHandlers(
http.get(
"https://api.github.com/repos/github/codeql-cli-binaries/releases/assets/783",
async () => {
const file = await readFile(codeqlReleaseZipPath);
return HttpResponse.arrayBuffer(file, {
headers: {
"Content-Type": "application/octet-stream",
},
});
},
),
);
});
afterAll(async () => {
await codeqlReleaseZipTempDir?.cleanup();
});
it("installs a distribution when no distribution exists", async () => {
await manager.installExtensionManagedDistributionRelease(release);
expect(
await readJson(join(globalStorageDirectory.path, "distribution.json")),
).toEqual({
folderIndex: 1,
release,
} satisfies DistributionState);
expect(
await readFile(
join(
globalStorageDirectory.path,
"distribution1",
"codeql",
codeQlLauncherName(),
),
"utf-8",
),
).toEqual("launcher!");
});
it("installs a distribution when a distribution already exists", async () => {
await outputJson(join(globalStorageDirectory.path, "distribution.json"), {
folderIndex: 78,
release: {
id: 1,
name: "v2.1.0",
createdAt: "2022-02-02T02:02:02Z",
assets: [
{
id: 1,
name: getRequiredAssetName(),
size: 100,
},
],
},
} satisfies DistributionState);
await outputFile(
join(
globalStorageDirectory.path,
"distribution78",
"codeql",
codeQlLauncherName(),
),
"",
);
// Re-initialize manager to read the state from the file
await manager.initialize();
await manager.installExtensionManagedDistributionRelease(release);
expect(
await readJson(join(globalStorageDirectory.path, "distribution.json")),
).toEqual({
folderIndex: 79,
release,
} satisfies DistributionState);
expect(
await readFile(
join(
globalStorageDirectory.path,
"distribution79",
"codeql",
codeQlLauncherName(),
),
"utf-8",
),
).toEqual("launcher!");
});
});
});