Monitor remote query run and render results (#1033)

extensions/ql-vscode/package.json

@@ -284,10 +284,6 @@
         "command": "codeQL.runRemoteQuery",
         "title": "CodeQL: Run Remote Query"
       },
-      {
-        "command": "codeQL.openRemoteQueriesView",
-        "title": "CodeQL: Open Remote Queries View"
-      },
       {
         "command": "codeQL.runQueries",
         "title": "CodeQL: Run Queries in Selected Files"
@@ -750,10 +746,6 @@
           "command": "codeQL.runRemoteQuery",
           "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
         },
-        {
-          "command": "codeQL.openRemoteQueriesView",
-          "when": "config.codeQL.canary"
-        },
         {
           "command": "codeQL.runQueries",
           "when": "false"

extensions/ql-vscode/src/extension.ts

@@ -74,8 +74,8 @@ import {
 import { CodeQlStatusBarHandler } from './status-bar';
 
 import { Credentials } from './authentication';
-import { runRemoteQuery } from './remote-queries/run-remote-query';
-import { RemoteQueriesInterfaceManager } from './remote-queries/remote-queries-interface';
+import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
+import { RemoteQuery } from './remote-queries/remote-query';
 
 /**
  * extension.ts
@@ -745,12 +745,8 @@ async function activateWithInstalledDistribution(
     )
   );
 
-  void logger.log('Initializing remote queries panel interface.');
-  const rmpm = new RemoteQueriesInterfaceManager(
-    ctx,
-    logger
-  );
-  ctx.subscriptions.push(rmpm);
+  void logger.log('Initializing remote queries interface.');
+  const rqm = new RemoteQueriesManager(ctx, logger, cliServer);
 
   // The "runRemoteQuery" command is internal-only.
   ctx.subscriptions.push(
@@ -765,8 +761,11 @@ async function activateWithInstalledDistribution(
           step: 0,
           message: 'Getting credentials'
         });
-        const credentials = await Credentials.initialize(ctx);
-        await runRemoteQuery(cliServer, credentials, uri || window.activeTextEditor?.document.uri, false, progress, token);
+        await rqm.runRemoteQuery(
+          uri || window.activeTextEditor?.document.uri,
+          progress,
+          token
+        );
       } else {
         throw new Error('Remote queries require the CodeQL Canary version to run.');
       }
@@ -775,6 +774,14 @@ async function activateWithInstalledDistribution(
       cancellable: true
     })
   );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.monitorRemoteQuery', async (
+      query: RemoteQuery,
+      token: CancellationToken) => {
+      await rqm.monitorRemoteQuery(query, token);
+    }));
+
   ctx.subscriptions.push(
     commandRunner(
       'codeQL.openReferencedFile',

extensions/ql-vscode/src/pure/interface-types.ts

@@ -1,4 +1,5 @@
 import * as sarif from 'sarif';
+import { RemoteQueryResult } from '../remote-queries/shared/remote-query-result';
 import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationValue } from './bqrs-cli-types';
 
 /**
@@ -370,14 +371,15 @@ export type FromRemoteQueriesMessage =
   | RemoteQueryErrorMessage;
 
 export type ToRemoteQueriesMessage =
-  | OpenRemoteQueriesViewMessage;
+  | SetRemoteQueryResultMessage;
 
 export interface RemoteQueryLoadedMessage {
   t: 'remoteQueryLoaded';
 }
 
-export interface OpenRemoteQueriesViewMessage {
-  t: 'openRemoteQueriesView';
+export interface SetRemoteQueryResultMessage {
+  t: 'setRemoteQueryResult';
+  queryResult: RemoteQueryResult
 }
 
 export interface RemoteQueryErrorMessage {

extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts

@@ -1,4 +1,3 @@
-import { DisposableObject } from '../pure/disposable-object';
 import {
   WebviewPanel,
   ExtensionContext,
@@ -16,10 +15,12 @@ import {
 import { Logger } from '../logging';
 import { getHtmlForWebview } from '../interface-utils';
 import { assertNever } from '../pure/helpers-pure';
-import { commandRunner } from '../commandRunner';
+import { AnalysisResult, RemoteQueryResult } from './remote-query-result';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
+import { AnalysisResult as AnalysisResultViewModel } from './shared/remote-query-result';
 
-
-export class RemoteQueriesInterfaceManager extends DisposableObject {
+export class RemoteQueriesInterfaceManager {
   private panel: WebviewPanel | undefined;
   private panelLoaded = false;
   private panelLoadedCallBacks: (() => void)[] = [];
@@ -28,22 +29,49 @@ export class RemoteQueriesInterfaceManager extends DisposableObject {
     private ctx: ExtensionContext,
     private logger: Logger,
   ) {
-    super();
-    commandRunner('codeQL.openRemoteQueriesView', () => this.handleOpenRemoteQueriesView());
     this.panelLoadedCallBacks.push(() => {
       void logger.log('Remote queries view loaded');
     });
   }
 
-  async showResults() {
+  async showResults(query: RemoteQuery, queryResult: RemoteQueryResult) {
     this.getPanel().reveal(undefined, true);
 
     await this.waitForPanelLoaded();
     await this.postMessage({
-      t: 'openRemoteQueriesView',
+      t: 'setRemoteQueryResult',
+      queryResult: this.buildViewModel(query, queryResult)
     });
   }
 
+  /**
+   * Builds up a model tailored to the view based on the query and result domain entities.
+   * The data is cleaned up, sorted where necessary, and transformed to a format that
+   * the view model can use. 
+   * @param query Information about the query that was run.
+   * @param queryResult The result of the query.
+   * @returns A fully created view model.
+   */
+  private buildViewModel(query: RemoteQuery, queryResult: RemoteQueryResult): RemoteQueryResultViewModel {
+    const queryFile = path.basename(query.queryFilePath);
+    const totalResultCount = queryResult.analysisResults.reduce((acc, cur) => acc + cur.resultCount, 0);
+    const executionDuration = this.getDuration(queryResult.executionEndTime, query.executionStartTime);
+    const analysisResults = this.buildAnalysisResults(queryResult.analysisResults);
+    const affectedRepositories = queryResult.analysisResults.filter(r => r.resultCount > 0);
+
+    return {
+      queryTitle: query.queryName,
+      queryFile: queryFile,
+      totalRepositoryCount: query.repositories.length,
+      affectedRepositoryCount: affectedRepositories.length,
+      totalResultCount: totalResultCount,
+      executionTimestamp: this.formatDate(query.executionStartTime),
+      executionDuration: executionDuration,
+      downloadLink: queryResult.allResultsDownloadUri,
+      results: analysisResults
+    };
+  }
+
   getPanel(): WebviewPanel {
     if (this.panel == undefined) {
       const { ctx } = this;
@@ -124,11 +152,65 @@ export class RemoteQueriesInterfaceManager extends DisposableObject {
     return this.getPanel().webview.postMessage(msg);
   }
 
-  async handleOpenRemoteQueriesView() {
-    this.getPanel().reveal(undefined, true);
-
-    await this.waitForPanelLoaded();
+  private getDuration(startTime: Date, endTime: Date): string {
+    const diffInMs = startTime.getTime() - endTime.getTime();
+    return this.formatDuration(diffInMs);
   }
 
+  private formatDuration(ms: number): string {
+    const seconds = ms / 1000;
+    const minutes = seconds / 60;
+    const hours = minutes / 60;
+    const days = hours / 24;
+    if (days > 1) {
+      return `${days.toFixed(2)} days`;
+    } else if (hours > 1) {
+      return `${hours.toFixed(2)} hours`;
+    } else if (minutes > 1) {
+      return `${minutes.toFixed(2)} minutes`;
+    } else {
+      return `${seconds.toFixed(2)} seconds`;
+    }
+  }
+
+  private formatDate = (d: Date): string => {
+    const datePart = d.toLocaleDateString(undefined, { day: 'numeric', month: 'short' });
+    const timePart = d.toLocaleTimeString(undefined, { hour: 'numeric', minute: 'numeric', hour12: true });
+    return `${datePart} at ${timePart}`;
+  };
+
+  private formatFileSize(bytes: number): string {
+    const kb = bytes / 1024;
+    const mb = kb / 1024;
+    const gb = mb / 1024;
+
+    if (bytes < 1024) {
+      return `${bytes} bytes`;
+    } else if (kb < 1024) {
+      return `${kb.toFixed(2)} KB`;
+    } else if (mb < 1024) {
+      return `${mb.toFixed(2)} MB`;
+    } else {
+      return `${gb.toFixed(2)} GB`;
+    }
+  }
+
+  /**
+   * Builds up a list of analysis results, in a data structure tailored to the view.
+   * @param analysisResults The results of a specific analysis.
+   * @returns A fully created view model.
+   */
+  private buildAnalysisResults(analysisResults: AnalysisResult[]): AnalysisResultViewModel[] {
+    const filteredAnalysisResults = analysisResults.filter(r => r.resultCount > 0);
+
+    const sortedAnalysisResults = filteredAnalysisResults.sort((a, b) => b.resultCount - a.resultCount);
+
+    return sortedAnalysisResults.map((analysisResult) => ({
+      nwo: analysisResult.nwo,
+      resultCount: analysisResult.resultCount,
+      downloadLink: analysisResult.downloadUri,
+      fileSize: this.formatFileSize(analysisResult.fileSizeInBytes)
+    }));
+  }
 }
 

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -0,0 +1,100 @@
+import { CancellationToken, commands, ExtensionContext, Uri, window } from 'vscode';
+import { Credentials } from '../authentication';
+import { CodeQLCliServer } from '../cli';
+import { ProgressCallback } from '../commandRunner';
+import { showAndLogErrorMessage, showInformationMessageWithAction } from '../helpers';
+import { Logger } from '../logging';
+import { getResultIndex, ResultIndexItem, runRemoteQuery } from './run-remote-query';
+import { RemoteQueriesInterfaceManager } from './remote-queries-interface';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueriesMonitor } from './remote-queries-monitor';
+import { RemoteQueryResult } from './remote-query-result';
+
+export class RemoteQueriesManager {
+  private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
+
+  constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly logger: Logger,
+    private readonly cliServer: CodeQLCliServer
+  ) {
+    this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
+  }
+
+  public async runRemoteQuery(
+    uri: Uri | undefined,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<void> {
+    const credentials = await Credentials.initialize(this.ctx);
+
+    const querySubmission = await runRemoteQuery(
+      this.cliServer,
+      credentials, uri || window.activeTextEditor?.document.uri,
+      false,
+      progress,
+      token);
+
+    if (querySubmission && querySubmission.query) {
+      void commands.executeCommand('codeQL.monitorRemoteQuery', querySubmission.query);
+    }
+  }
+
+  public async monitorRemoteQuery(
+    query: RemoteQuery,
+    cancellationToken: CancellationToken
+  ): Promise<void> {
+    const credentials = await Credentials.initialize(this.ctx);
+
+    const queryResult = await this.remoteQueriesMonitor.monitorQuery(query, cancellationToken);
+
+    const executionEndTime = new Date();
+
+    if (queryResult.status === 'CompletedSuccessfully') {
+      const resultIndexItems = await this.downloadResultIndex(credentials, query);
+
+      const totalResultCount = resultIndexItems.reduce((acc, cur) => acc + cur.results_count, 0);
+      const message = `Query "${query.queryName}" run on ${query.repositories.length} repositories and returned ${totalResultCount} results`;
+
+      const shouldOpenView = await showInformationMessageWithAction(message, 'View');
+      if (shouldOpenView) {
+        const queryResult = this.mapQueryResult(executionEndTime, resultIndexItems);
+        const rqim = new RemoteQueriesInterfaceManager(this.ctx, this.logger);
+        await rqim.showResults(query, queryResult);
+      }
+    } else if (queryResult.status === 'CompletedUnsuccessfully') {
+      await showAndLogErrorMessage(`Remote query execution failed. Error: ${queryResult.error}`);
+      return;
+    } else if (queryResult.status === 'Cancelled') {
+      await showAndLogErrorMessage('Remote query monitoring was cancelled');
+    }
+  }
+
+  private async downloadResultIndex(credentials: Credentials, query: RemoteQuery) {
+    return await getResultIndex(
+      credentials,
+      query.controllerRepository.owner,
+      query.controllerRepository.name,
+      query.actionsWorkflowRunId);
+  }
+
+  private mapQueryResult(executionEndTime: Date, resultindexItems: ResultIndexItem[]): RemoteQueryResult {
+    // Example URIs are used for now, but a solution for downloading the results will soon be implemented.
+    const allResultsDownloadUri = 'www.example.com';
+    const analysisDownloadUri = 'www.example.com';
+
+    const analysisResults = resultindexItems.map(ri => ({
+      nwo: ri.nwo,
+      resultCount: ri.results_count,
+      downloadUri: analysisDownloadUri,
+      fileSizeInBytes: ri.sarif_file_size || ri.bqrs_file_size,
+    })
+    );
+
+    return {
+      executionEndTime,
+      analysisResults,
+      allResultsDownloadUri,
+    };
+  }
+}

extensions/ql-vscode/src/remote-queries/remote-queries-monitor.ts

@@ -0,0 +1,88 @@
+import * as vscode from 'vscode';
+import { Credentials } from '../authentication';
+import { Logger } from '../logging';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
+
+export class RemoteQueriesMonitor {
+  // With a sleep of 5 seconds, the maximum number of attempts takes
+  // us to just over 2 days worth of monitoring.
+  private static readonly maxAttemptCount = 17280;
+  private static readonly sleepTime = 5000;
+
+  constructor(
+    private readonly extensionContext: vscode.ExtensionContext,
+    private readonly logger: Logger
+  ) {
+  }
+
+  public async monitorQuery(
+    remoteQuery: RemoteQuery,
+    cancellationToken: vscode.CancellationToken
+  ): Promise<RemoteQueryWorkflowResult> {
+    const credentials = await Credentials.initialize(this.extensionContext);
+
+    if (!credentials) {
+      throw Error('Error authenticating with GitHub');
+    }
+
+    let attemptCount = 0;
+
+    const octokit = await credentials.getOctokit();
+
+    while (attemptCount <= RemoteQueriesMonitor.maxAttemptCount) {
+      if (cancellationToken && cancellationToken.isCancellationRequested) {
+        return { status: 'Cancelled' };
+      }
+
+      const workflowRun = await octokit.rest.actions.getWorkflowRun({
+        owner: remoteQuery.controllerRepository.owner,
+        repo: remoteQuery.controllerRepository.name,
+        run_id: remoteQuery.actionsWorkflowRunId
+      });
+
+      if (workflowRun.data.status === 'completed') {
+        if (workflowRun.data.conclusion === 'success') {
+          return { status: 'CompletedSuccessfully' };
+        } else {
+          const error = this.getWorkflowError(workflowRun.data.conclusion);
+          return { status: 'CompletedUnsuccessfully', error };
+        }
+      }
+
+      await this.sleep(RemoteQueriesMonitor.sleepTime);
+      attemptCount++;
+    }
+
+    void this.logger.log('Remote query monitoring timed out after 2 days');
+    return { status: 'Cancelled' };
+  }
+
+  private async sleep(ms: number) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  private getWorkflowError(conclusion: string | null): string {
+    if (!conclusion) {
+      return 'Workflow finished without a conclusion';
+    }
+
+    if (conclusion === 'cancelled') {
+      return 'The remote query execution was cancelled.';
+    }
+
+    if (conclusion === 'timed_out') {
+      return 'The remote query execution timed out.';
+    }
+
+    if (conclusion === 'failure') {
+      // TODO: Get the actual error from the workflow or potentially
+      // from an artifact from the action itself.
+      return 'The remote query execution has failed.';
+    }
+
+    return `Unexpected query execution conclusion: ${conclusion}`;
+  }
+}
+
+

extensions/ql-vscode/src/remote-queries/remote-query-result.ts

@@ -0,0 +1,12 @@
+export interface RemoteQueryResult {
+  executionEndTime: Date;
+  analysisResults: AnalysisResult[];
+  allResultsDownloadUri: string;
+}
+
+export interface AnalysisResult {
+  nwo: string,
+  resultCount: number,
+  downloadUri: string,
+  fileSizeInBytes: number
+}

extensions/ql-vscode/src/remote-queries/remote-query-workflow-result.ts

@@ -0,0 +1,10 @@
+export type RemoteQueryWorkflowStatus =
+  | 'InProgress'
+  | 'CompletedSuccessfully'
+  | 'CompletedUnsuccessfully'
+  | 'Cancelled';
+
+export interface RemoteQueryWorkflowResult {
+  status: RemoteQueryWorkflowStatus;
+  error?: string;
+}

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -521,7 +521,7 @@ async function downloadArtifact(
   return artifactPath;
 }
 
-interface ResultIndexItem {
+export interface ResultIndexItem {
   nwo: string;
   id: string;
   results_count: number;

extensions/ql-vscode/src/remote-queries/shared/remote-query-result.ts

@@ -0,0 +1,18 @@
+export interface RemoteQueryResult {
+  queryTitle: string;
+  queryFile: string;
+  totalRepositoryCount: number;
+  affectedRepositoryCount: number;
+  totalResultCount: number;
+  executionTimestamp: string;
+  executionDuration: string;
+  downloadLink: string;
+  results: AnalysisResult[]
+}
+
+export interface AnalysisResult {
+  nwo: string,
+  resultCount: number,
+  downloadLink: string,
+  fileSize: string,
+}

extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx

@@ -1,28 +1,25 @@
 import * as React from 'react';
+import { useEffect, useState } from 'react';
 import * as Rdom from 'react-dom';
+import { SetRemoteQueryResultMessage } from '../../pure/interface-types';
+import { AnalysisResult, RemoteQueryResult } from '../shared/remote-query-result';
 import * as octicons from '../../view/octicons';
 
 import { vscode } from '../../view/vscode-api';
 
-interface AnalysisResult {
-  nwo: string,
-  resultCount: number,
-  downloadLink: string,
-  fileSize: string,
-}
+const emptyQueryResult: RemoteQueryResult = {
+  queryTitle: '',
+  queryFile: '',
+  totalRepositoryCount: 0,
+  affectedRepositoryCount: 0,
+  totalResultCount: 0,
+  executionTimestamp: '',
+  executionDuration: '',
+  downloadLink: '',
+  results: []
+};
 
-interface Props {
-  queryTitle: string;
-  queryFile: string;
-  totalRepositoryCount: number;
-  totalResultCount: number;
-  executionTimestamp: string;
-  executionDuration: string;
-  downloadLink: string;
-  results: AnalysisResult[]
-}
-
-const AnalysisResult = (props: AnalysisResult) => (
+const AnalysisResultItem = (props: AnalysisResult) => (
   <span>
     <span className="vscode-codeql__analysis-item">{octicons.repo}</span>
     <span className="vscode-codeql__analysis-item">{props.nwo}</span>
@@ -39,74 +36,64 @@ const AnalysisResult = (props: AnalysisResult) => (
   </span>
 );
 
-export function RemoteQueries(props: Props): JSX.Element {
-  return <div className="vscode-codeql__remote-queries-view">
-    <h1 className="vscode-codeql__query-title">{props.queryTitle}</h1>
+export function RemoteQueries(): JSX.Element {
+  const [queryResult, setQueryResult] = useState<RemoteQueryResult>(emptyQueryResult);
 
-    <p className="vscode-codeql__paragraph">
-      {props.totalResultCount} results in {props.totalRepositoryCount} repositories
-      ({props.executionDuration}), {props.executionTimestamp}
-    </p>
-    <p className="vscode-codeql__paragraph">
-      <span className="vscode-codeql__query-file">{octicons.file} <span>{props.queryFile}</span></span>
-      <span>{octicons.codeSquare} <span>query</span></span>
-    </p>
+  useEffect(() => {
+    window.addEventListener('message', (evt: MessageEvent) => {
+      if (evt.origin === window.origin) {
+        const msg: SetRemoteQueryResultMessage = evt.data;
+        if (msg.t === 'setRemoteQueryResult') {
+          setQueryResult(msg.queryResult);
+        }
+      } else {
+        // sanitize origin
+        const origin = evt.origin.replace(/\n|\r/g, '');
+        console.error(`Invalid event origin ${origin}`);
+      }
+    });
+  });
 
-    <div className="vscode-codeql__query-summary-container">
-      <h2 className="vscode-codeql__query-summary-title">Repositories with results ({props.totalRepositoryCount}):</h2>
-      <a className="vscode-codeql__summary-download-link vscode-codeql__download-link" href={props.downloadLink}>
-        {octicons.download}Download all
-      </a>
-    </div>
+  if (!queryResult) {
+    return <div>Waiting for results to load.</div>;
+  }
 
-    <ul className="vscode-codeql__results-list">
-      {props.results.map(result =>
-        <li key={result.nwo} className="vscode-codeql__results-list-item">
-          <AnalysisResult {...result} />
-        </li>
-      )}
-    </ul>
-  </div>;
+  try {
+    return <div className="vscode-codeql__remote-queries-view">
+      <h1 className="vscode-codeql__query-title">{queryResult.queryTitle}</h1>
+
+      <p className="vscode-codeql__paragraph">
+        {queryResult.totalResultCount} results in {queryResult.totalRepositoryCount} repositories
+        ({queryResult.executionDuration}), {queryResult.executionTimestamp}
+      </p>
+      <p className="vscode-codeql__paragraph">
+        <span className="vscode-codeql__query-file">{octicons.file} <span>{queryResult.queryFile}</span></span>
+        <span>{octicons.codeSquare} <span>query</span></span>
+      </p>
+
+      <div className="vscode-codeql__query-summary-container">
+        <h2 className="vscode-codeql__query-summary-title">Repositories with results ({queryResult.affectedRepositoryCount}):</h2>
+        <a className="vscode-codeql__summary-download-link vscode-codeql__download-link" href={queryResult.downloadLink}>
+          {octicons.download}Download all
+        </a>
+      </div>
+
+      <ul className="vscode-codeql__results-list">
+        {queryResult.results.map(result =>
+          <li key={result.nwo} className="vscode-codeql__results-list-item">
+            <AnalysisResultItem {...result} />
+          </li>
+        )}
+      </ul>
+    </div>;
+  } catch (err) {
+    console.error(err);
+    return <div>There was an error displaying the view.</div>;
+  }
 }
 
-const formatDate = (d: Date): string => {
-  const datePart = d.toLocaleDateString(undefined, { day: 'numeric', month: 'short' });
-  const timePart = d.toLocaleTimeString(undefined, { hour: 'numeric', minute: 'numeric', hour12: true });
-  return `${datePart} at ${timePart}`;
-};
-
-const data: Props = {
-  queryTitle: 'Empty block',
-  queryFile: 'example.ql',
-  totalRepositoryCount: 13,
-  totalResultCount: 72,
-  executionTimestamp: formatDate(new Date()),
-  executionDuration: '0.6 seconds',
-  downloadLink: 'www.example.com',
-  results: [
-    {
-      nwo: 'github/foo',
-      resultCount: 35,
-      downloadLink: 'www.example.com',
-      fileSize: '12.3mb'
-    },
-    {
-      nwo: 'github/bar',
-      resultCount: 9,
-      downloadLink: 'www.example.com',
-      fileSize: '10.1mb'
-    },
-    {
-      nwo: 'github/baz',
-      resultCount: 80,
-      downloadLink: 'www.example.com',
-      fileSize: '11.2mb'
-    }
-  ]
-};
-
 Rdom.render(
-  <RemoteQueries {...data} />,
+  <RemoteQueries />,
   document.getElementById('root'),
   // Post a message to the extension when fully loaded.
   () => vscode.postMessage({ t: 'remoteQueryLoaded' })