Merge pull request #1648 from github/koesie10/store-scenario-bodies-as-files

Store binary scenario bodies as files

extensions/ql-vscode/src/mocks/gh-api-request.ts

@@ -64,7 +64,8 @@ export interface GetVariantAnalysisRepoResultRequest {
   },
   response: {
     status: number,
-    body?: ArrayBuffer
+    body?: Buffer | string,
+    contentType: string,
   }
 }
 

extensions/ql-vscode/src/mocks/recorder.ts

@@ -5,9 +5,12 @@ import { MockedRequest } from 'msw';
 import { SetupServerApi } from 'msw/node';
 import { IsomorphicResponse } from '@mswjs/interceptors';
 
+import { Headers } from 'headers-polyfill';
+import fetch from 'node-fetch';
+
 import { DisposableObject } from '../pure/disposable-object';
 
-import { GitHubApiRequest, RequestKind } from './gh-api-request';
+import { GetVariantAnalysisRepoResultRequest, GitHubApiRequest, RequestKind } from './gh-api-request';
 
 export class Recorder extends DisposableObject {
   private readonly allRequests = new Map<string, MockedRequest>();
@@ -70,7 +73,28 @@ export class Recorder extends DisposableObject {
 
       const fileName = `${i}-${request.request.kind}.json`;
       const filePath = path.join(scenarioDirectory, fileName);
-      await fs.writeFile(filePath, JSON.stringify(request, null, 2));
+
+      let writtenRequest = {
+        ...request
+      };
+
+      if (shouldWriteBodyToFile(writtenRequest)) {
+        const extension = writtenRequest.response.contentType === 'application/zip' ? 'zip' : 'bin';
+
+        const bodyFileName = `${i}-${writtenRequest.request.kind}.body.${extension}`;
+        const bodyFilePath = path.join(scenarioDirectory, bodyFileName);
+        await fs.writeFile(bodyFilePath, writtenRequest.response.body);
+
+        writtenRequest = {
+          ...writtenRequest,
+          response: {
+            ...writtenRequest.response,
+            body: `file:${bodyFileName}`,
+          },
+        };
+      }
+
+      await fs.writeFile(filePath, JSON.stringify(writtenRequest, null, 2));
     }
 
     this.stop();
@@ -79,10 +103,14 @@ export class Recorder extends DisposableObject {
   }
 
   private onRequestStart(request: MockedRequest): void {
+    if (request.headers.has('x-vscode-codeql-msw-bypass')) {
+      return;
+    }
+
     this.allRequests.set(request.id, request);
   }
 
-  private onResponseBypass(response: IsomorphicResponse, requestId: string): void {
+  private async onResponseBypass(response: IsomorphicResponse, requestId: string): Promise<void> {
     const request = this.allRequests.get(requestId);
     this.allRequests.delete(requestId);
     if (!request) {
@@ -93,7 +121,7 @@ export class Recorder extends DisposableObject {
       return;
     }
 
-    const gitHubApiRequest = createGitHubApiRequest(request.url.toString(), response.status, response.body);
+    const gitHubApiRequest = await createGitHubApiRequest(request.url.toString(), response.status, response.body, response.headers);
     if (!gitHubApiRequest) {
       return;
     }
@@ -102,7 +130,7 @@ export class Recorder extends DisposableObject {
   }
 }
 
-function createGitHubApiRequest(url: string, status: number, body: string): GitHubApiRequest | undefined {
+async function createGitHubApiRequest(url: string, status: number, body: string, headers: Headers): Promise<GitHubApiRequest | undefined> {
   if (!url) {
     return undefined;
   }
@@ -160,6 +188,17 @@ function createGitHubApiRequest(url: string, status: number, body: string): GitH
   // if url is a download URL for a variant analysis result, then it's a get-variant-analysis-repoResult.
   const repoDownloadMatch = url.match(/objects-origin\.githubusercontent\.com\/codeql-query-console\/codeql-variant-analysis-repo-tasks\/\d+\/(?<repositoryId>\d+)/);
   if (repoDownloadMatch?.groups?.repositoryId) {
+    // msw currently doesn't support binary response bodies, so we need to download this separately
+    // see https://github.com/mswjs/interceptors/blob/15eafa6215a328219999403e3ff110e71699b016/src/interceptors/ClientRequest/utils/getIncomingMessageBody.ts#L24-L33
+    // Essentially, mws is trying to decode a ZIP file as UTF-8 which changes the bytes and corrupts the file.
+    const response = await fetch(url, {
+      headers: {
+        // We need to ensure we don't end up in an infinite loop, since this request will also be intercepted
+        'x-vscode-codeql-msw-bypass': 'true',
+      },
+    });
+    const responseBuffer = await response.buffer();
+
     return {
       request: {
         kind: RequestKind.GetVariantAnalysisRepoResult,
@@ -167,10 +206,15 @@ function createGitHubApiRequest(url: string, status: number, body: string): GitH
       },
       response: {
         status,
-        body: body as unknown as ArrayBuffer,
+        body: responseBuffer,
+        contentType: headers.get('content-type') ?? 'application/octet-stream',
       }
     };
   }
 
   return undefined;
 }
+
+function shouldWriteBodyToFile(request: GitHubApiRequest): request is GetVariantAnalysisRepoResultRequest {
+  return request.response.body instanceof Buffer;
+}

extensions/ql-vscode/src/mocks/request-handlers.ts

@@ -39,8 +39,17 @@ async function readRequestFiles(scenarioDirPath: string): Promise<GitHubApiReque
 
   const requests: GitHubApiRequest[] = [];
   for (const file of orderedFiles) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
     const filePath = path.join(scenarioDirPath, file);
     const request: GitHubApiRequest = await fs.readJson(filePath, { encoding: 'utf8' });
+
+    if (typeof request.response.body === 'string' && request.response.body.startsWith('file:')) {
+      request.response.body = await fs.readFile(path.join(scenarioDirPath, request.response.body.substring(5)));
+    }
+
     requests.push(request);
   }
 
@@ -135,6 +144,7 @@ function createGetVariantAnalysisRepoResultRequestHandler(requests: GitHubApiReq
       if (scenarioRequest.response.body) {
         return res(
           ctx.status(scenarioRequest.response.status),
+          ctx.set('Content-Type', scenarioRequest.response.contentType),
           ctx.body(scenarioRequest.response.body),
         );
       } else {