mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3762 from github/koesie10/fix-codeql-download

Browse Source 
Store state of CodeQL distribution on filesystem instead of in `globalState`
This commit is contained in:
Koen Vlaswinkel
2024-10-22 15:13:05 +02:00
committed by GitHub
parent 5b69404da8 8a7f710876
commit f19b0dfc8b
7 changed files with 573 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/ql-vscode/CHANGELOG.md
View File

@@ -3,6 +3,7 @@
## [UNRELEASED]
- Support result columns of type `QlBuiltins::BigInt` in quick evaluations. [#3647](https://github.com/github/vscode-codeql/pull/3647)
- Fix a bug where the CodeQL CLI would be re-downloaded if you switched to a different filesystem (for example Codespaces or a remote SSH host). [#3762](https://github.com/github/vscode-codeql/pull/3762)
## 1.16.0 - 10 October 2024

45
extensions/ql-vscode/package-lock.json generated
View File

@@ -28,6 +28,7 @@
"msw": "^2.2.13",
"nanoid": "^5.0.7",
"p-queue": "^8.0.1",
"proper-lockfile": "^4.1.2",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"semver": "^7.6.2",
@@ -82,6 +83,7 @@
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "20.16.*",
"@types/proper-lockfile": "^4.1.4",
"@types/react": "^18.3.1",
"@types/react-dom": "^18.3.0",
"@types/sarif": "^2.1.2",
@@ -6348,6 +6350,16 @@
"integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==",
"dev": true
},
"node_modules/@types/proper-lockfile": {
"version": "4.1.4",
"resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz",
"integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/retry": "*"
}
},
"node_modules/@types/qs": {
"version": "6.9.16",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz",
@@ -6385,6 +6397,13 @@
"integrity": "sha512-A4STmOXPhMUtHH+S6ymgE2GiBSMqf4oTvcQZMcHzokuTLVYzXTB8ttjcgxOVaAp2lGwEdzZ0J+cRbbeevQj1UQ==",
"dev": true
},
"node_modules/@types/retry": {
"version": "0.12.5",
"resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz",
"integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/sarif": {
"version": "2.1.7",
"resolved": "https://registry.npmjs.org/@types/sarif/-/sarif-2.1.7.tgz",
@@ -19837,6 +19856,23 @@
"react-is": "^16.13.1"
}
},
"node_modules/proper-lockfile": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
"integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==",
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.2.4",
"retry": "^0.12.0",
"signal-exit": "^3.0.2"
}
},
"node_modules/proper-lockfile/node_modules/signal-exit": {
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
"license": "ISC"
},
"node_modules/proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -20661,6 +20697,15 @@
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
"dev": true
},
"node_modules/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/reusify": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",

2
extensions/ql-vscode/package.json
View File

@@ -1986,6 +1986,7 @@
"msw": "^2.2.13",
"nanoid": "^5.0.7",
"p-queue": "^8.0.1",
"proper-lockfile": "^4.1.2",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"semver": "^7.6.2",
@@ -2040,6 +2041,7 @@
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "20.16.*",
"@types/proper-lockfile": "^4.1.4",
"@types/react": "^18.3.1",
"@types/react-dom": "^18.3.0",
"@types/sarif": "^2.1.2",

160
extensions/ql-vscode/src/codeql-cli/distribution.ts
View File

@@ -1,5 +1,12 @@
import type { WriteStream } from "fs";
import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
import {
createWriteStream,
mkdtemp,
outputJson,
pathExists,
readJson,
remove,
} from "fs-extra";
import { tmpdir } from "os";
import { delimiter, dirname, join } from "path";
import { Range, satisfies } from "semver";
@@ -19,7 +26,9 @@ import {
InvocationRateLimiter,
InvocationRateLimiterResultKind,
} from "../common/invocation-rate-limiter";
import type { NotificationLogger } from "../common/logging";
import {
showAndLogExceptionWithTelemetry,
showAndLogErrorMessage,
showAndLogWarningMessage,
} from "../common/logging";
@@ -28,6 +37,11 @@ import { reportUnzipProgress } from "../common/vscode/unzip-progress";
import type { Release } from "./distribution/release";
import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";
import { createTimeoutSignal } from "../common/fetch-stream";
import { withDistributionUpdateLock } from "./lock";
import { asError, getErrorMessage } from "../common/helpers-pure";
import { isIOError } from "../common/files";
import { telemetryListener } from "../common/vscode/telemetry";
import { redactableError } from "../common/errors";
/**
* distribution.ts
@@ -53,6 +67,11 @@ const NIGHTLY_DISTRIBUTION_REPOSITORY_NWO = "dsp-testing/codeql-cli-nightlies";
*/
export const DEFAULT_DISTRIBUTION_VERSION_RANGE: Range = new Range("2.x");
export interface DistributionState {
folderIndex: number;
release: Release | null;
}
export interface DistributionProvider {
getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
onDidChangeDistribution?: Event<void>;
@@ -64,6 +83,7 @@ export class DistributionManager implements DistributionProvider {
public readonly config: DistributionConfig,
private readonly versionRange: Range,
extensionContext: ExtensionContext,
logger: NotificationLogger,
) {
this._onDidChangeDistribution = config.onDidChangeConfiguration;
this.extensionSpecificDistributionManager =
@@ -71,6 +91,7 @@ export class DistributionManager implements DistributionProvider {
config,
versionRange,
extensionContext,
logger,
);
this.updateCheckRateLimiter = new InvocationRateLimiter(
extensionContext.globalState,
@@ -80,6 +101,10 @@ export class DistributionManager implements DistributionProvider {
);
}
public async initialize(): Promise<void> {
await this.extensionSpecificDistributionManager.initialize();
}
/**
* Look up a CodeQL launcher binary.
*/
@@ -280,14 +305,58 @@ export class DistributionManager implements DistributionProvider {
}
class ExtensionSpecificDistributionManager {
private distributionState: DistributionState | undefined;
constructor(
private readonly config: DistributionConfig,
private readonly versionRange: Range,
private readonly extensionContext: ExtensionContext,
private readonly logger: NotificationLogger,
) {
/**/
}
public async initialize() {
await this.ensureDistributionStateExists();
}
private async ensureDistributionStateExists() {
const distributionStatePath = this.getDistributionStatePath();
try {
this.distributionState = await readJson(distributionStatePath);
} catch (e: unknown) {
if (isIOError(e) && e.code === "ENOENT") {
// If the file doesn't exist, that just means we need to create it
this.distributionState = {
folderIndex:
this.extensionContext.globalState.get(
"distributionFolderIndex",
0,
) ?? 0,
release: (this.extensionContext.globalState.get(
"distributionRelease",
) ?? null) as Release | null,
};
// This may result in a race condition, but when this happens both processes should write the same file.
await outputJson(distributionStatePath, this.distributionState);
} else {
void showAndLogExceptionWithTelemetry(
this.logger,
telemetryListener,
redactableError(
asError(e),
)`Failed to read distribution state from ${distributionStatePath}: ${getErrorMessage(e)}`,
);
this.distributionState = {
folderIndex: 0,
release: null,
};
}
}
}
public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
if (this.getInstalledRelease() !== undefined) {
// An extension specific distribution has been installed.
@@ -350,9 +419,21 @@ class ExtensionSpecificDistributionManager {
release: Release,
progressCallback?: ProgressCallback,
): Promise<void> {
await this.downloadDistribution(release, progressCallback);
// Store the installed release within the global extension state.
await this.storeInstalledRelease(release);
if (!this.distributionState) {
await this.ensureDistributionStateExists();
}
const distributionStatePath = this.getDistributionStatePath();
await withDistributionUpdateLock(
// .lock will be appended to this filename
distributionStatePath,
async () => {
await this.downloadDistribution(release, progressCallback);
// Store the installed release within the global extension state.
await this.storeInstalledRelease(release);
},
);
}
private async downloadDistribution(
@@ -564,23 +645,19 @@ class ExtensionSpecificDistributionManager {
}
private async bumpDistributionFolderIndex(): Promise<void> {
const index = this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
0,
);
await this.extensionContext.globalState.update(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
index + 1,
);
await this.updateState((oldState) => {
return {
...oldState,
folderIndex: (oldState.folderIndex ?? 0) + 1,
};
});
}
private getDistributionStoragePath(): string {
const distributionState = this.getDistributionState();
// Use an empty string for the initial distribution for backwards compatibility.
const distributionFolderIndex =
this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey,
0,
) || "";
const distributionFolderIndex = distributionState.folderIndex || "";
return join(
this.extensionContext.globalStorageUri.fsPath,
ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
@@ -595,26 +672,55 @@ class ExtensionSpecificDistributionManager {
);
}
private getInstalledRelease(): Release | undefined {
return this.extensionContext.globalState.get(
ExtensionSpecificDistributionManager._installedReleaseStateKey,
private getDistributionStatePath(): string {
return join(
this.extensionContext.globalStorageUri.fsPath,
ExtensionSpecificDistributionManager._distributionStateFilename,
);
}
private getInstalledRelease(): Release | undefined {
return this.getDistributionState().release ?? undefined;
}
private async storeInstalledRelease(
release: Release | undefined,
): Promise<void> {
await this.extensionContext.globalState.update(
ExtensionSpecificDistributionManager._installedReleaseStateKey,
release,
);
await this.updateState((oldState) => ({
...oldState,
release: release ?? null,
}));
}
private getDistributionState(): DistributionState {
const distributionState = this.distributionState;
if (distributionState === undefined) {
throw new Error(
"Invariant violation: distribution state not initialized",
);
}
return distributionState;
}
private async updateState(
f: (oldState: DistributionState) => DistributionState,
) {
const oldState = this.distributionState;
if (oldState === undefined) {
throw new Error(
"Invariant violation: distribution state not initialized",
);
}
const newState = f(oldState);
this.distributionState = newState;
const distributionStatePath = this.getDistributionStatePath();
await outputJson(distributionStatePath, newState);
}
private static readonly _currentDistributionFolderBaseName = "distribution";
private static readonly _currentDistributionFolderIndexStateKey =
"distributionFolderIndex";
private static readonly _installedReleaseStateKey = "distributionRelease";
private static readonly _codeQlExtractedFolderName = "codeql";
private static readonly _distributionStateFilename = "distribution.json";
}
/*

22
extensions/ql-vscode/src/codeql-cli/lock.ts Normal file
View File

@@ -0,0 +1,22 @@
import { lock } from "proper-lockfile";
export async function withDistributionUpdateLock(
lockFile: string,
f: () => Promise<void>,
) {
const release = await lock(lockFile, {
stale: 60_000, // 1 minute. We can take the lock longer than this because that's based on the update interval.
update: 10_000, // 10 seconds
retries: {
minTimeout: 10_000,
maxTimeout: 60_000,
retries: 100,
},
});
try {
await f();
} finally {
await release();
}
}

2
extensions/ql-vscode/src/extension.ts
View File

@@ -362,7 +362,9 @@ export async function activate(
distributionConfigListener,
codeQlVersionRange,
ctx,
app.logger,
);
await distributionManager.initialize();
registerErrorStubs([checkForUpdatesCommand], (command) => async () => {
void showAndLogErrorMessage(

369
extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts
View File

@@ -1,18 +1,41 @@
import * as log from "../../../../src/common/logging/notifications";
import { extLogger } from "../../../../src/common/logging/vscode";
import { writeFile } from "fs-extra";
import {
outputFile,
outputJson,
readFile,
readJson,
writeFile,
} from "fs-extra";
import { join } from "path";
import * as os from "os";
import type { DirectoryResult } from "tmp-promise";
import { dir } from "tmp-promise";
import type { DistributionState } from "../../../../src/codeql-cli/distribution";
import {
DEFAULT_DISTRIBUTION_VERSION_RANGE,
DistributionManager,
DistributionUpdateCheckResultKind,
getExecutableFromDirectory,
} from "../../../../src/codeql-cli/distribution";
import type {
showAndLogErrorMessage,
showAndLogWarningMessage,
} from "../../../../src/common/logging";
import { createMockLogger } from "../../../__mocks__/loggerMock";
import { mockedObject } from "../../../mocked-object";
import type { DistributionConfig } from "../../../../src/config";
import type { ExtensionContext } from "vscode";
import { Uri } from "vscode";
import { setupServer } from "msw/node";
import { http, HttpResponse } from "msw";
import {
codeQlLauncherName,
getRequiredAssetName,
} from "../../../../src/common/distribution";
import type { GithubRelease } from "../../../../src/codeql-cli/distribution/releases-api-consumer";
import type { Release } from "../../../../src/codeql-cli/distribution/release";
import { zip } from "zip-a-folder";
jest.mock("os", () => {
const original = jest.requireActual("os");
@@ -108,6 +131,7 @@ describe("Launcher path", () => {
{ customCodeQlPath: pathToCmd } as any,
{} as any,
{} as any,
createMockLogger(),
);
const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -126,6 +150,7 @@ describe("Launcher path", () => {
{ customCodeQlPath: pathToCmd } as any,
{} as any,
{} as any,
createMockLogger(),
);
const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -141,6 +166,7 @@ describe("Launcher path", () => {
{ customCodeQlPath: pathToCmd } as any,
{} as any,
{} as any,
createMockLogger(),
);
const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -151,3 +177,344 @@ describe("Launcher path", () => {
expect(errorSpy).toHaveBeenCalledTimes(1);
});
});
describe("Distribution updates", () => {
const server = setupServer();
beforeAll(() =>
server.listen({
onUnhandledRequest: "error",
}),
);
afterEach(() => server.resetHandlers());
afterAll(() => server.close());
let manager: DistributionManager;
let globalStorageDirectory: DirectoryResult;
beforeEach(async () => {
globalStorageDirectory = await dir({
unsafeCleanup: true,
});
manager = new DistributionManager(
mockedObject<DistributionConfig>({
customCodeQlPath: undefined,
channel: "stable",
includePrerelease: false,
personalAccessToken: undefined,
downloadTimeout: 100,
onDidChangeConfiguration: () => {},
}),
DEFAULT_DISTRIBUTION_VERSION_RANGE,
mockedObject<ExtensionContext>({
globalState: {
get: () => {},
update: () => {},
},
globalStorageUri: Uri.file(globalStorageDirectory.path),
}),
createMockLogger(),
);
await manager.initialize();
});
afterEach(async () => {
await globalStorageDirectory.cleanup();
});
it("should have an empty distribution.json file after initialization", async () => {
expect(
await readJson(join(globalStorageDirectory.path, "distribution.json")),
).toEqual({
folderIndex: 0,
release: null,
} satisfies DistributionState);
});
describe("checkForUpdatesToDistribution", () => {
beforeEach(() => {
server.resetHandlers(
http.get(
"https://api.github.com/repos/github/codeql-cli-binaries/releases",
async () => {
return HttpResponse.json([
{
id: 1335,
name: "v2.2.0",
tag_name: "v2.2.0",
created_at: "2024-02-02T02:02:02Z",
prerelease: false,
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
{
id: 1,
name: "v2.1.0",
tag_name: "v2.1.0",
created_at: "2022-02-02T02:02:02Z",
prerelease: false,
assets: [
{
id: 1,
name: getRequiredAssetName(),
size: 100,
},
],
},
] satisfies GithubRelease[]);
},
),
);
});
it("should have an update when no distribution is installed", async () => {
expect(
await manager.checkForUpdatesToExtensionManagedDistribution(0),
).toEqual({
kind: DistributionUpdateCheckResultKind.UpdateAvailable,
updatedRelease: {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
} satisfies Awaited<
ReturnType<typeof manager.checkForUpdatesToExtensionManagedDistribution>
>);
});
it("should not have an update when the latest distribution is installed", async () => {
await outputJson(join(globalStorageDirectory.path, "distribution.json"), {
folderIndex: 1,
release: {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
} satisfies DistributionState);
await outputFile(
join(
globalStorageDirectory.path,
"distribution1",
"codeql",
codeQlLauncherName(),
),
"",
);
// Re-initialize manager to read the state from the file
await manager.initialize();
expect(
await manager.checkForUpdatesToExtensionManagedDistribution(0),
).toEqual({
kind: DistributionUpdateCheckResultKind.AlreadyUpToDate,
} satisfies Awaited<
ReturnType<typeof manager.checkForUpdatesToExtensionManagedDistribution>
>);
});
it("should have an update when an older distribution is installed", async () => {
await outputJson(join(globalStorageDirectory.path, "distribution.json"), {
folderIndex: 1,
release: {
id: 1,
name: "v2.1.0",
createdAt: "2022-02-02T02:02:02Z",
assets: [
{
id: 1,
name: getRequiredAssetName(),
size: 100,
},
],
},
} satisfies DistributionState);
await outputFile(
join(
globalStorageDirectory.path,
"distribution1",
"codeql",
codeQlLauncherName(),
),
"",
);
// Re-initialize manager to read the state from the file
await manager.initialize();
expect(
await manager.checkForUpdatesToExtensionManagedDistribution(0),
).toEqual({
kind: DistributionUpdateCheckResultKind.UpdateAvailable,
updatedRelease: {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
},
} satisfies Awaited<
ReturnType<typeof manager.checkForUpdatesToExtensionManagedDistribution>
>);
});
});
describe("installExtensionManagedDistributionRelease", () => {
const release: Release = {
id: 1335,
name: "v2.2.0",
createdAt: "2024-02-02T02:02:02Z",
assets: [
{
id: 783,
name: getRequiredAssetName(),
size: 2378,
},
],
};
let codeqlReleaseZipTempDir: DirectoryResult;
let codeqlReleaseZipPath: string;
beforeAll(async () => {
codeqlReleaseZipTempDir = await dir({
unsafeCleanup: true,
});
await outputFile(
join(
codeqlReleaseZipTempDir.path,
"distribution",
"codeql",
codeQlLauncherName(),
),
"launcher!",
);
codeqlReleaseZipPath = join(codeqlReleaseZipTempDir.path, "codeql.zip");
await zip(
join(codeqlReleaseZipTempDir.path, "distribution"),
codeqlReleaseZipPath,
);
server.resetHandlers(
http.get(
"https://api.github.com/repos/github/codeql-cli-binaries/releases/assets/783",
async () => {
const file = await readFile(codeqlReleaseZipPath);
return HttpResponse.arrayBuffer(file, {
headers: {
"Content-Type": "application/octet-stream",
},
});
},
),
);
});
afterAll(async () => {
await codeqlReleaseZipTempDir?.cleanup();
});
it("installs a distribution when no distribution exists", async () => {
await manager.installExtensionManagedDistributionRelease(release);
expect(
await readJson(join(globalStorageDirectory.path, "distribution.json")),
).toEqual({
folderIndex: 1,
release,
} satisfies DistributionState);
expect(
await readFile(
join(
globalStorageDirectory.path,
"distribution1",
"codeql",
codeQlLauncherName(),
),
"utf-8",
),
).toEqual("launcher!");
});
it("installs a distribution when a distribution already exists", async () => {
await outputJson(join(globalStorageDirectory.path, "distribution.json"), {
folderIndex: 78,
release: {
id: 1,
name: "v2.1.0",
createdAt: "2022-02-02T02:02:02Z",
assets: [
{
id: 1,
name: getRequiredAssetName(),
size: 100,
},
],
},
} satisfies DistributionState);
await outputFile(
join(
globalStorageDirectory.path,
"distribution78",
"codeql",
codeQlLauncherName(),
),
"",
);
// Re-initialize manager to read the state from the file
await manager.initialize();
await manager.installExtensionManagedDistributionRelease(release);
expect(
await readJson(join(globalStorageDirectory.path, "distribution.json")),
).toEqual({
folderIndex: 79,
release,
} satisfies DistributionState);
expect(
await readFile(
join(
globalStorageDirectory.path,
"distribution79",
"codeql",
codeQlLauncherName(),
),
"utf-8",
),
).toEqual("launcher!");
});
});
});