Merge pull request #2877 from github/robertbrignull/upgrade_msw

Upgrade MSW
This commit is contained in:
Koen Vlaswinkel
2023-10-03 10:08:34 +02:00
committed by GitHub
18 changed files with 622 additions and 630 deletions

View File

@@ -62,7 +62,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json

View File

@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json
@@ -64,7 +64,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json
@@ -110,7 +110,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json
@@ -149,7 +149,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json
@@ -183,7 +183,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json
@@ -251,7 +251,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json

View File

@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.15.0'
- name: Install dependencies
run: |

View File

@@ -21,6 +21,7 @@ The following files will need to be updated:
- `extensions/ql-vscode/.nvmrc` - this will enable nvm to automatically switch to the correct node version when you're in the project folder
- `extensions/ql-vscode/package-lock.json` - the "engines.node: '[VERSION]'" setting
- `extensions/ql-vscode/package.json` - the "engines.node: '[VERSION]'" setting
- `extensions/ql-vscode/package.json` - the "@types/node: '[VERSION]'" dependency
## Node.js version used in tests

View File

@@ -1 +1 @@
v16.17.1
v18.15.0

View File

@@ -2,6 +2,7 @@
## [UNRELEASED]
- Increase the required version of VS Code to 1.82.0. [#2877](https://github.com/github/vscode-codeql/pull/2877)
- Fix a bug where the query server was restarted twice after configuration changes. [#2884](https://github.com/github/vscode-codeql/pull/2884).
## 1.9.1 - 29 September 2023

File diff suppressed because it is too large Load Diff

View File

@@ -13,8 +13,8 @@
"url": "https://github.com/github/vscode-codeql"
},
"engines": {
"vscode": "^1.67.0",
"node": "^16.17.1",
"vscode": "^1.82.0",
"node": "^18.15.0",
"npm": ">=7.20.6"
},
"categories": [
@@ -2070,8 +2070,8 @@
"prepare": "cd ../.. && husky install"
},
"dependencies": {
"@octokit/plugin-retry": "^4.1.6",
"@octokit/rest": "^19.0.4",
"@octokit/plugin-retry": "^6.0.1",
"@octokit/rest": "^20.0.2",
"@vscode/codicons": "^0.0.31",
"@vscode/debugadapter": "^1.59.0",
"@vscode/debugprotocol": "^1.59.0",
@@ -2085,7 +2085,7 @@
"fs-extra": "^11.1.1",
"immutable": "^4.0.0",
"js-yaml": "^4.1.0",
"msw": "^1.2.0",
"msw": "^0.0.0-fetch.rc-20",
"nanoid": "^3.2.0",
"node-fetch": "^2.6.7",
"p-queue": "^6.0.0",
@@ -2115,7 +2115,7 @@
"@babel/preset-typescript": "^7.21.4",
"@faker-js/faker": "^8.0.2",
"@github/markdownlint-github": "^0.3.0",
"@octokit/plugin-throttling": "^5.0.1",
"@octokit/plugin-throttling": "^8.0.0",
"@storybook/addon-actions": "^7.1.0",
"@storybook/addon-essentials": "^7.1.0",
"@storybook/addon-interactions": "^7.1.0",
@@ -2141,7 +2141,7 @@
"@types/jest": "^29.0.2",
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "^16.11.25",
"@types/node": "18.15.0",
"@types/node-fetch": "^2.5.2",
"@types/react": "^18.0.28",
"@types/react-dom": "^18.0.11",
@@ -2153,7 +2153,7 @@
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "^0.10.1",
"@types/vscode": "^1.67.0",
"@types/vscode": "^1.82.0",
"@types/webpack": "^5.28.0",
"@types/webpack-env": "^1.18.0",
"@typescript-eslint/eslint-plugin": "^6.2.1",

View File

@@ -14,7 +14,8 @@
import { pathExists, readJson, writeJson } from "fs-extra";
import { resolve, relative } from "path";
import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
import { Octokit } from "@octokit/core";
import { type RestEndpointMethodTypes } from "@octokit/rest";
import { throttling } from "@octokit/plugin-throttling";
import { getFiles } from "./util/files";
@@ -22,6 +23,7 @@ import type { GitHubApiRequest } from "../src/common/mock-gh-api/gh-api-request"
import { isGetVariantAnalysisRequest } from "../src/common/mock-gh-api/gh-api-request";
import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
import { AppOctokit } from "../src/common/octokit";
const extensionDirectory = resolve(__dirname, "..");
const scenariosDirectory = resolve(
@@ -31,7 +33,7 @@ const scenariosDirectory = resolve(
// Make sure we don't run into rate limits by automatically waiting until we can
// make another request.
const MyOctokit = Octokit.plugin(throttling);
const MyOctokit = AppOctokit.plugin(throttling);
const auth = process.env.GITHUB_TOKEN;

View File

@@ -17,7 +17,7 @@ export enum RequestKind {
AutoModel = "autoModel",
}
interface BasicErorResponse {
export interface BasicErrorResponse {
message: string;
}
@@ -27,7 +27,7 @@ interface GetRepoRequest {
};
response: {
status: number;
body: Repository | BasicErorResponse | undefined;
body: Repository | BasicErrorResponse | undefined;
};
}
@@ -37,7 +37,7 @@ interface SubmitVariantAnalysisRequest {
};
response: {
status: number;
body?: VariantAnalysis | BasicErorResponse;
body?: VariantAnalysis | BasicErrorResponse;
};
}
@@ -47,7 +47,7 @@ interface GetVariantAnalysisRequest {
};
response: {
status: number;
body?: VariantAnalysis | BasicErorResponse;
body?: VariantAnalysis | BasicErrorResponse;
};
}
@@ -58,7 +58,7 @@ interface GetVariantAnalysisRepoRequest {
};
response: {
status: number;
body?: VariantAnalysisRepoTask | BasicErorResponse;
body?: VariantAnalysisRepoTask | BasicErrorResponse;
};
}
@@ -74,6 +74,13 @@ export interface GetVariantAnalysisRepoResultRequest {
};
}
export interface CodeSearchResponse {
total_count: number;
items: Array<{
repository: Repository;
}>;
}
interface CodeSearchRequest {
request: {
kind: RequestKind.CodeSearch;
@@ -81,16 +88,14 @@ interface CodeSearchRequest {
};
response: {
status: number;
body?: {
total_count?: number;
items?: Array<{
repository: Repository;
}>;
};
message?: string;
body?: CodeSearchResponse | BasicErrorResponse;
};
}
export interface AutoModelResponse {
models: string;
}
interface AutoModelRequest {
request: {
kind: RequestKind.AutoModel;
@@ -100,10 +105,7 @@ interface AutoModelRequest {
};
response: {
status: number;
body?: {
models: string;
};
message?: string;
body?: AutoModelResponse | BasicErrorResponse;
};
}

View File

@@ -1,30 +1,33 @@
import { ensureDir, writeFile } from "fs-extra";
import { join } from "path";
import { MockedRequest } from "msw";
import { SetupServer } from "msw/node";
import { IsomorphicResponse } from "@mswjs/interceptors";
import { Headers } from "headers-polyfill";
import fetch from "node-fetch";
import { SetupServer } from "msw/node";
import { DisposableObject } from "../disposable-object";
import { gzipDecode } from "../zlib";
import {
AutoModelResponse,
BasicErrorResponse,
CodeSearchResponse,
GetVariantAnalysisRepoResultRequest,
GitHubApiRequest,
RequestKind,
} from "./gh-api-request";
import {
VariantAnalysis,
VariantAnalysisRepoTask,
} from "../../variant-analysis/gh-api/variant-analysis";
import { Repository } from "../../variant-analysis/gh-api/repository";
export class Recorder extends DisposableObject {
private readonly allRequests = new Map<string, MockedRequest>();
private currentRecordedScenario: GitHubApiRequest[] = [];
private _isRecording = false;
constructor(private readonly server: SetupServer) {
super();
this.onRequestStart = this.onRequestStart.bind(this);
this.onResponseBypass = this.onResponseBypass.bind(this);
}
@@ -45,7 +48,6 @@ export class Recorder extends DisposableObject {
this.clear();
this.server.events.on("request:start", this.onRequestStart);
this.server.events.on("response:bypass", this.onResponseBypass);
}
@@ -56,13 +58,11 @@ export class Recorder extends DisposableObject {
this._isRecording = false;
this.server.events.removeListener("request:start", this.onRequestStart);
this.server.events.removeListener("response:bypass", this.onResponseBypass);
}
public clear() {
this.currentRecordedScenario = [];
this.allRequests.clear();
}
public async save(scenariosPath: string, name: string): Promise<string> {
@@ -91,7 +91,7 @@ export class Recorder extends DisposableObject {
let bodyFileLink = undefined;
if (writtenRequest.response.body) {
await writeFile(bodyFilePath, writtenRequest.response.body || "");
await writeFile(bodyFilePath, writtenRequest.response.body);
bodyFileLink = `file:${bodyFileName}`;
}
@@ -112,33 +112,18 @@ export class Recorder extends DisposableObject {
return scenarioDirectory;
}
private onRequestStart(request: MockedRequest): void {
private async onResponseBypass(
response: Response,
request: Request,
_requestId: string,
): Promise<void> {
if (request.headers.has("x-vscode-codeql-msw-bypass")) {
return;
}
this.allRequests.set(request.id, request);
}
private async onResponseBypass(
response: IsomorphicResponse,
requestId: string,
): Promise<void> {
const request = this.allRequests.get(requestId);
this.allRequests.delete(requestId);
if (!request) {
return;
}
if (response.body === undefined) {
return;
}
const gitHubApiRequest = await createGitHubApiRequest(
request.url.toString(),
response.status,
response.body,
response.headers,
request.url,
response,
);
if (!gitHubApiRequest) {
return;
@@ -150,14 +135,14 @@ export class Recorder extends DisposableObject {
async function createGitHubApiRequest(
url: string,
status: number,
body: string,
headers: Headers,
response: Response,
): Promise<GitHubApiRequest | undefined> {
if (!url) {
return undefined;
}
const status = response.status;
if (url.match(/\/repos\/[a-zA-Z0-9-_.]+\/[a-zA-Z0-9-_.]+$/)) {
return {
request: {
@@ -165,7 +150,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
Repository | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -179,7 +166,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
VariantAnalysis | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -195,7 +184,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
VariantAnalysis | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -211,7 +202,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
VariantAnalysisRepoTask | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -238,9 +231,10 @@ async function createGitHubApiRequest(
repositoryId: parseInt(repoDownloadMatch.groups.repositoryId, 10),
},
response: {
status,
status: response.status,
body: responseBuffer,
contentType: headers.get("content-type") ?? "application/octet-stream",
contentType:
response.headers.get("content-type") ?? "application/octet-stream",
},
};
}
@@ -254,7 +248,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
CodeSearchResponse | BasicErrorResponse | undefined
>(response),
},
};
}
@@ -269,7 +265,9 @@ async function createGitHubApiRequest(
},
response: {
status,
body: JSON.parse(body),
body: await jsonResponseBody<
BasicErrorResponse | AutoModelResponse | undefined
>(response),
},
};
}
@@ -277,6 +275,26 @@ async function createGitHubApiRequest(
return undefined;
}
async function responseBody(response: Response): Promise<Uint8Array> {
const body = await response.arrayBuffer();
const view = new Uint8Array(body);
if (view[0] === 0x1f && view[1] === 0x8b) {
// Response body is gzipped, so we need to un-gzip it.
return await gzipDecode(view);
} else {
return view;
}
}
async function jsonResponseBody<T>(response: Response): Promise<T> {
const body = await responseBody(response);
const text = new TextDecoder("utf-8").decode(body);
return JSON.parse(text);
}
function shouldWriteBodyToFile(
request: GitHubApiRequest,
): request is GetVariantAnalysisRepoResultRequest {

View File

@@ -1,6 +1,6 @@
import { join } from "path";
import { readdir, readJson, readFile } from "fs-extra";
import { DefaultBodyType, MockedRequest, rest, RestHandler } from "msw";
import { RequestHandler, rest } from "msw";
import {
GitHubApiRequest,
isAutoModelRequest,
@@ -14,7 +14,19 @@ import {
const baseUrl = "https://api.github.com";
type RequestHandler = RestHandler<MockedRequest<DefaultBodyType>>;
const jsonResponse = <T>(
body: T,
init?: ResponseInit,
contentType = "application/json",
): Response => {
return new Response(JSON.stringify(body), {
...init,
headers: {
"Content-Type": contentType,
...init?.headers,
},
});
};
export async function createRequestHandlers(
scenarioDirPath: string,
@@ -82,11 +94,10 @@ function createGetRepoRequestHandler(
const getRepoRequest = getRepoRequests[0];
return rest.get(`${baseUrl}/repos/:owner/:name`, (_req, res, ctx) => {
return res(
ctx.status(getRepoRequest.response.status),
ctx.json(getRepoRequest.response.body),
);
return rest.get(`${baseUrl}/repos/:owner/:name`, () => {
return jsonResponse(getRepoRequest.response.body, {
status: getRepoRequest.response.status,
});
});
}
@@ -105,11 +116,10 @@ function createSubmitVariantAnalysisRequestHandler(
return rest.post(
`${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses`,
(_req, res, ctx) => {
return res(
ctx.status(getRepoRequest.response.status),
ctx.json(getRepoRequest.response.body),
);
() => {
return jsonResponse(getRepoRequest.response.body, {
status: getRepoRequest.response.status,
});
},
);
}
@@ -127,7 +137,7 @@ function createGetVariantAnalysisRequestHandler(
// request, so keep an index of the request and return the appropriate response.
return rest.get(
`${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId`,
(_req, res, ctx) => {
() => {
const request = getVariantAnalysisRequests[requestIndex];
if (requestIndex < getVariantAnalysisRequests.length - 1) {
@@ -135,10 +145,9 @@ function createGetVariantAnalysisRequestHandler(
requestIndex++;
}
return res(
ctx.status(request.response.status),
ctx.json(request.response.body),
);
return jsonResponse(request.response.body, {
status: request.response.status,
});
},
);
}
@@ -152,18 +161,17 @@ function createGetVariantAnalysisRepoRequestHandler(
return rest.get(
`${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId/repositories/:repoId`,
(req, res, ctx) => {
({ request, params }) => {
const scenarioRequest = getVariantAnalysisRepoRequests.find(
(r) => r.request.repositoryId.toString() === req.params.repoId,
(r) => r.request.repositoryId.toString() === params.repoId,
);
if (!scenarioRequest) {
throw Error(`No scenario request found for ${req.url}`);
throw Error(`No scenario request found for ${request.url}`);
}
return res(
ctx.status(scenarioRequest.response.status),
ctx.json(scenarioRequest.response.body),
);
return jsonResponse(scenarioRequest.response.body, {
status: scenarioRequest.response.status,
});
},
);
}
@@ -177,22 +185,23 @@ function createGetVariantAnalysisRepoResultRequestHandler(
return rest.get(
"https://objects-origin.githubusercontent.com/codeql-query-console/codeql-variant-analysis-repo-tasks/:variantAnalysisId/:repoId/*",
(req, res, ctx) => {
({ request, params }) => {
const scenarioRequest = getVariantAnalysisRepoResultRequests.find(
(r) => r.request.repositoryId.toString() === req.params.repoId,
(r) => r.request.repositoryId.toString() === params.repoId,
);
if (!scenarioRequest) {
throw Error(`No scenario request found for ${req.url}`);
throw Error(`No scenario request found for ${request.url}`);
}
if (scenarioRequest.response.body) {
return res(
ctx.status(scenarioRequest.response.status),
ctx.set("Content-Type", scenarioRequest.response.contentType),
ctx.body(scenarioRequest.response.body),
);
return new Response(scenarioRequest.response.body, {
status: scenarioRequest.response.status,
headers: {
"Content-Type": scenarioRequest.response.contentType,
},
});
} else {
return res(ctx.status(scenarioRequest.response.status));
return new Response(null, { status: scenarioRequest.response.status });
}
},
);
@@ -207,7 +216,7 @@ function createCodeSearchRequestHandler(
// During a code search, there are multiple request to get pages of results. We
// need to return different responses for each request, so keep an index of the
// request and return the appropriate response.
return rest.get(`${baseUrl}/search/code?q=*`, (_req, res, ctx) => {
return rest.get(`${baseUrl}/search/code`, () => {
const request = codeSearchRequests[requestIndex];
if (requestIndex < codeSearchRequests.length - 1) {
@@ -215,10 +224,9 @@ function createCodeSearchRequestHandler(
requestIndex++;
}
return res(
ctx.status(request.response.status),
ctx.json(request.response.body),
);
return jsonResponse(request.response.body, {
status: request.response.status,
});
});
}
@@ -233,7 +241,7 @@ function createAutoModelRequestHandler(
// so keep an index of the request and return the appropriate response.
return rest.post(
`${baseUrl}/repos/github/codeql/code-scanning/codeql/auto-model`,
(_req, res, ctx) => {
() => {
const request = autoModelRequests[requestIndex];
if (requestIndex < autoModelRequests.length - 1) {
@@ -241,10 +249,9 @@ function createAutoModelRequestHandler(
requestIndex++;
}
return res(
ctx.status(request.response.status),
ctx.json(request.response.body),
);
return jsonResponse(request.response.body, {
status: request.response.status,
});
},
);
}

View File

@@ -0,0 +1,10 @@
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
import fetch from "node-fetch";
export const AppOctokit = Octokit.Octokit.defaults({
request: {
fetch,
},
retry,
});

View File

@@ -1,7 +1,7 @@
import * as vscode from "vscode";
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
import { Credentials } from "../authentication";
import { AppOctokit } from "../octokit";
export const GITHUB_AUTH_PROVIDER_ID = "github";
@@ -32,9 +32,8 @@ export class VSCodeCredentials implements Credentials {
const accessToken = await this.getAccessToken();
return new Octokit.Octokit({
return new AppOctokit({
auth: accessToken,
retry,
});
}

View File

@@ -1,9 +1,9 @@
import { retry } from "@octokit/plugin-retry";
import { throttling } from "@octokit/plugin-throttling";
import { Octokit } from "@octokit/rest";
import { Progress, CancellationToken } from "vscode";
import { Credentials } from "../common/authentication";
import { BaseLogger } from "../common/logging";
import { AppOctokit } from "../common/octokit";
export async function getCodeSearchRepositories(
query: string,
@@ -46,12 +46,11 @@ async function provideOctokitWithThrottling(
credentials: Credentials,
logger: BaseLogger,
): Promise<Octokit> {
const MyOctokit = Octokit.plugin(throttling);
const MyOctokit = AppOctokit.plugin(throttling);
const auth = await credentials.getAccessToken();
const octokit = new MyOctokit({
auth,
retry,
throttle: {
onRateLimit: (retryAfter: number, options: any): boolean => {
void logger.log(

View File

@@ -14,7 +14,6 @@ import {
} from "fs-extra";
import { basename, join } from "path";
import * as Octokit from "@octokit/rest";
import { retry } from "@octokit/plugin-retry";
import { DatabaseManager, DatabaseItem } from "./local-databases";
import { tmpDir } from "../tmp-dir";
@@ -32,6 +31,7 @@ import { Credentials } from "../common/authentication";
import { AppCommandManager } from "../common/commands";
import { allowHttp } from "../config";
import { showAndLogInformationMessage } from "../common/logging";
import { AppOctokit } from "../common/octokit";
/**
* Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -186,7 +186,7 @@ export async function downloadGitHubDatabase(
const octokit = credentials
? await credentials.getOctokit()
: new Octokit.Octokit({ retry });
: new AppOctokit();
const result = await convertGithubNwoToDatabaseUrl(
nwo,

View File

@@ -1,8 +1,8 @@
import { retry } from "@octokit/plugin-retry";
import * as Octokit from "@octokit/rest";
import { RequestInterface } from "@octokit/types/dist-types/RequestInterface";
import { Credentials } from "../../src/common/authentication";
import { AppOctokit } from "../../src/common/octokit";
function makeTestOctokit(octokit: Octokit.Octokit): Credentials {
return {
@@ -38,5 +38,5 @@ export function testCredentialsWithStub(
* optionally authenticated with a given token.
*/
export function testCredentialsWithRealOctokit(token?: string): Credentials {
return makeTestOctokit(new Octokit.Octokit({ auth: token, retry }));
return makeTestOctokit(new AppOctokit({ auth: token }));
}

View File

@@ -12,9 +12,7 @@ const rootDir = path.resolve(__dirname, "../..");
/** @type import("jest-runner-vscode").RunnerOptions */
const config = {
// Temporary until https://github.com/github/vscode-codeql/issues/2402 is fixed
// version: "stable",
version: "1.77.3",
version: "stable",
launchArgs: [
"--disable-gpu",
"--extensions-dir=" + path.join(rootDir, ".vscode-test", "extensions"),