v1.17.1

Bump CLI Version to v2.20.2 for integration tests

docs/releasing.md

@@ -1,6 +1,5 @@
 # Releasing (write access required)
 
-1. Make sure the needed authentication keys are valid. Most likely the Azure DevOps PAT needs to be regenerated. See below.
 1. Determine the new version number. We default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
     - Making substantial new features available to all users. This can include lifting a feature flag.
     - Breakage in compatibility with recent versions of the CLI.
@@ -61,7 +60,7 @@
 
 ## Secrets and authentication for publishing
 
-Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token.
+Repository administrators will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token.
 
 To regenerate the Open VSX token:
 

extensions/ql-vscode/.vscodeignore

@@ -1,17 +1 @@
-.vs/**
-.vscode/**
-.vscode-test/**
-typings/**
-out/test/**
-out/vscode-tests/**
-**/@types/**
-**/*.ts
-test/**
-src/**
 **/*.map
-.gitignore
-gulpfile.js/**
-tsconfig.json
-.prettierrc
-vsc-extension-quickstart.md
-node_modules/**

extensions/ql-vscode/CHANGELOG.md

@@ -1,6 +1,10 @@
 # CodeQL for Visual Studio Code: Changelog
 
-## 1.7.0 - 20 December 2024
+## 1.17.1 - 23 January 2025
+
+- Remove support for CodeQL CLI versions older than 2.18.4. [#3895](https://github.com/github/vscode-codeql/pull/3895)
+
+## 1.17.0 - 20 December 2024
 
 - Add a palette command that allows importing all databases directly inside of a parent folder. [#3797](https://github.com/github/vscode-codeql/pull/3797)
 - Only use VS Code telemetry settings instead of using `codeQL.telemetry.enableTelemetry` [#3853](https://github.com/github/vscode-codeql/pull/3853)

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.17.0",
+  "version": "1.17.1",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -42,13 +42,6 @@
     "workspaceContains:.git"
   ],
   "main": "./out/extension",
-  "files": [
-    "gen/*.js",
-    "media/**",
-    "out/**",
-    "package.json",
-    "language-configuration.json"
-  ],
   "contributes": {
     "configurationDefaults": {
       "[ql]": {
@@ -949,6 +942,10 @@
         "command": "codeQLQueryHistory.compareWith",
         "title": "Compare Results"
       },
+      {
+        "command": "codeQLQueryHistory.comparePerformanceWith",
+        "title": "Compare Performance"
+      },
       {
         "command": "codeQLQueryHistory.openOnGithub",
         "title": "View Logs"
@@ -1220,6 +1217,11 @@
           "group": "3_queryHistory@0",
           "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
         },
+        {
+          "command": "codeQLQueryHistory.comparePerformanceWith",
+          "group": "3_queryHistory@1",
+          "when": "viewItem == rawResultsItem && config.codeQL.canary || viewItem == interpretedResultsItem && config.codeQL.canary"
+        },
         {
           "command": "codeQLQueryHistory.showQueryLog",
           "group": "4_queryHistory@4",
@@ -1723,6 +1725,10 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.comparePerformanceWith",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.sortByName",
           "when": "false"
@@ -2006,18 +2012,18 @@
     "@github/markdownlint-github": "^0.6.3",
     "@microsoft/eslint-formatter-sarif": "^3.1.0",
     "@playwright/test": "^1.49.0",
-    "@storybook/addon-a11y": "^8.4.7",
-    "@storybook/addon-actions": "^8.4.7",
-    "@storybook/addon-essentials": "^8.4.7",
-    "@storybook/addon-interactions": "^8.4.7",
-    "@storybook/addon-links": "^8.4.7",
+    "@storybook/addon-a11y": "^8.5.0",
+    "@storybook/addon-actions": "^8.5.0",
+    "@storybook/addon-essentials": "^8.5.0",
+    "@storybook/addon-interactions": "^8.5.0",
+    "@storybook/addon-links": "^8.5.0",
     "@storybook/blocks": "^8.0.2",
-    "@storybook/components": "^8.4.7",
-    "@storybook/csf": "^0.1.12",
+    "@storybook/components": "^8.5.0",
+    "@storybook/csf": "^0.1.13",
     "@storybook/icons": "^1.3.0",
-    "@storybook/manager-api": "^8.4.7",
-    "@storybook/react": "^8.4.7",
-    "@storybook/react-vite": "^8.4.7",
+    "@storybook/manager-api": "^8.5.0",
+    "@storybook/react": "^8.5.0",
+    "@storybook/react-vite": "^8.5.0",
     "@storybook/theming": "^8.2.4",
     "@testing-library/dom": "^10.4.0",
     "@testing-library/jest-dom": "^6.6.3",
@@ -2045,10 +2051,10 @@
     "@types/tmp": "^0.2.6",
     "@types/vscode": "1.90.0",
     "@types/yauzl": "^2.10.3",
-    "@typescript-eslint/eslint-plugin": "^8.18.1",
-    "@typescript-eslint/parser": "^8.18.1",
+    "@typescript-eslint/eslint-plugin": "^8.19.0",
+    "@typescript-eslint/parser": "^8.19.0",
     "@vscode/test-electron": "^2.3.9",
-    "@vscode/vsce": "^2.24.0",
+    "@vscode/vsce": "^3.2.1",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.9.5",
     "cosmiconfig": "^9.0.0",
@@ -2069,20 +2075,20 @@
     "eslint-plugin-storybook": "^0.8.0",
     "glob": "^11.0.0",
     "gulp": "^5.0.0",
-    "gulp-esbuild": "^0.12.1",
+    "gulp-esbuild": "^0.14.0",
     "gulp-replace": "^1.1.3",
     "gulp-typescript": "^5.0.1",
     "husky": "^9.1.5",
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "^15.2.10",
-    "markdownlint-cli2": "^0.13.0",
+    "lint-staged": "^15.3.0",
+    "markdownlint-cli2": "^0.17.0",
     "markdownlint-cli2-formatter-pretty": "^0.0.7",
     "npm-run-all": "^4.1.5",
     "patch-package": "^8.0.0",
     "prettier": "^3.2.5",
-    "storybook": "^8.4.7",
+    "storybook": "^8.5.0",
     "tar-stream": "^3.1.7",
     "through2": "^4.0.2",
     "ts-jest": "^29.2.5",

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -1904,7 +1904,7 @@ function shouldDebugCliServer() {
 export class CliVersionConstraint {
   // The oldest version of the CLI that we support. This is used to determine
   // whether to show a warning about the CLI being too old on startup.
-  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.16.6");
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.18.4");
 
   constructor(private readonly cli: CodeQLCliServer) {
     /**/

extensions/ql-vscode/src/common/commands.ts

@@ -180,6 +180,7 @@ export type QueryHistoryCommands = {
   "codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.comparePerformanceWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;

extensions/ql-vscode/src/common/interface-types.ts

@@ -27,6 +27,7 @@ import type {
 } from "./raw-result-types";
 import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
 import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
+import type { PerformanceComparisonDataFromLog } from "../log-insights/performance-comparison";
 
 /**
  * This module contains types and code that are shared between
@@ -396,6 +397,17 @@ export interface SetComparisonsMessage {
   readonly message: string | undefined;
 }
 
+export type ToComparePerformanceViewMessage = SetPerformanceComparisonQueries;
+
+export interface SetPerformanceComparisonQueries {
+  readonly t: "setPerformanceComparison";
+  readonly from: PerformanceComparisonDataFromLog;
+  readonly to: PerformanceComparisonDataFromLog;
+  readonly comparison: boolean;
+}
+
+export type FromComparePerformanceViewMessage = CommonFromViewMessages;
+
 export type QueryCompareResult =
   | RawQueryCompareResult
   | InterpretedQueryCompareResult;

extensions/ql-vscode/src/common/jsonl-reader.ts

@@ -24,7 +24,12 @@ export async function readJsonlFile<T>(
   return new Promise((resolve, reject) => {
     const stream = createReadStream(path, { encoding: "utf8" });
     let buffer = "";
-    stream.on("data", async (chunk: string) => {
+    stream.on("data", async (chunk: string | Buffer) => {
+      if (typeof chunk !== "string") {
+        // This should never happen because we specify the encoding as "utf8".
+        throw new Error("Invalid chunk");
+      }
+
       const parts = (buffer + chunk).split(doubleLineBreakRegexp);
       buffer = parts.pop()!;
       if (parts.length > 0) {

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -41,6 +41,13 @@ export abstract class AbstractWebview<
 
   constructor(protected readonly app: App) {}
 
+  public hidePanel() {
+    if (this.panel !== undefined) {
+      this.panel.dispose();
+      this.panel = undefined;
+    }
+  }
+
   public async restoreView(panel: WebviewPanel): Promise<void> {
     this.panel = panel;
     const config = await this.getPanelConfig();

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -7,6 +7,7 @@ import type { App } from "../app";
 export type WebviewKind =
   | "results"
   | "compare"
+  | "compare-performance"
   | "variant-analysis"
   | "data-flow-paths"
   | "model-editor"

extensions/ql-vscode/src/compare-performance/compare-performance-view.ts

@@ -0,0 +1,108 @@
+import { statSync } from "fs";
+import { ViewColumn } from "vscode";
+
+import type { App } from "../common/app";
+import { redactableError } from "../common/errors";
+import type {
+  FromComparePerformanceViewMessage,
+  ToComparePerformanceViewMessage,
+} from "../common/interface-types";
+import type { Logger } from "../common/logging";
+import { showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
+import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
+import { AbstractWebview } from "../common/vscode/abstract-webview";
+import { withProgress } from "../common/vscode/progress";
+import { telemetryListener } from "../common/vscode/telemetry";
+import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
+import { PerformanceOverviewScanner } from "../log-insights/performance-comparison";
+import { scanLog } from "../log-insights/log-scanner";
+import type { ResultsView } from "../local-queries";
+
+export class ComparePerformanceView extends AbstractWebview<
+  ToComparePerformanceViewMessage,
+  FromComparePerformanceViewMessage
+> {
+  constructor(
+    app: App,
+    public logger: Logger,
+    public labelProvider: HistoryItemLabelProvider,
+    private resultsView: ResultsView,
+  ) {
+    super(app);
+  }
+
+  async showResults(fromJsonLog: string, toJsonLog: string) {
+    const panel = await this.getPanel();
+    panel.reveal(undefined, false);
+
+    // Close the results viewer as it will have opened when the user clicked the query in the history view
+    // (which they must do as part of the UI interaction for opening the performance view).
+    // The performance view generally needs a lot of width so it's annoying to have the result viewer open.
+    this.resultsView.hidePanel();
+
+    await this.waitForPanelLoaded();
+
+    function scanLogWithProgress(log: string, logDescription: string) {
+      const bytes = statSync(log).size;
+      return withProgress(
+        async (progress) =>
+          scanLog(log, new PerformanceOverviewScanner(), progress),
+
+        {
+          title: `Scanning evaluator log ${logDescription} (${(bytes / 1024 / 1024).toFixed(1)} MB)`,
+        },
+      );
+    }
+
+    const [fromPerf, toPerf] = await Promise.all([
+      fromJsonLog === ""
+        ? new PerformanceOverviewScanner()
+        : scanLogWithProgress(fromJsonLog, "1/2"),
+      scanLogWithProgress(toJsonLog, fromJsonLog === "" ? "1/1" : "2/2"),
+    ]);
+
+    await this.postMessage({
+      t: "setPerformanceComparison",
+      from: fromPerf.getData(),
+      to: toPerf.getData(),
+      comparison: fromJsonLog !== "",
+    });
+  }
+
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: "comparePerformanceView",
+      title: "Compare CodeQL Performance",
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: "compare-performance",
+    };
+  }
+
+  protected onPanelDispose(): void {}
+
+  protected async onMessage(
+    msg: FromComparePerformanceViewMessage,
+  ): Promise<void> {
+    switch (msg.t) {
+      case "viewLoaded":
+        this.onWebViewLoaded();
+        break;
+
+      case "telemetry":
+        telemetryListener?.sendUIInteraction(msg.action);
+        break;
+
+      case "unhandledError":
+        void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
+          redactableError(
+            msg.error,
+          )`Unhandled error in performance comparison view: ${msg.error.message}`,
+        );
+        break;
+    }
+  }
+}

extensions/ql-vscode/src/extension.ts

@@ -135,6 +135,7 @@ import { LanguageContextStore } from "./language-context-store";
 import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
 import { GitHubDatabasesModule } from "./databases/github-databases";
 import { DatabaseFetcher } from "./databases/database-fetcher";
+import { ComparePerformanceView } from "./compare-performance/compare-performance-view";
 
 /**
  * extension.ts
@@ -928,6 +929,11 @@ async function activateWithInstalledDistribution(
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo,
     ): Promise<void> => showResultsForComparison(compareView, from, to),
+    async (
+      from: CompletedLocalQueryInfo,
+      to: CompletedLocalQueryInfo | undefined,
+    ): Promise<void> =>
+      showPerformanceComparison(comparePerformanceView, from, to),
   );
 
   ctx.subscriptions.push(qhm);
@@ -953,6 +959,15 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(compareView);
 
+  void extLogger.log("Initializing performance comparison view.");
+  const comparePerformanceView = new ComparePerformanceView(
+    app,
+    queryServerLogger,
+    labelProvider,
+    localQueryResultsView,
+  );
+  ctx.subscriptions.push(comparePerformanceView);
+
   void extLogger.log("Initializing source archive filesystem provider.");
   archiveFilesystemProvider_activate(ctx, dbm);
 
@@ -1191,6 +1206,30 @@ async function showResultsForComparison(
   }
 }
 
+async function showPerformanceComparison(
+  view: ComparePerformanceView,
+  from: CompletedLocalQueryInfo,
+  to: CompletedLocalQueryInfo | undefined,
+): Promise<void> {
+  let fromLog = from.evaluatorLogPaths?.jsonSummary;
+  let toLog = to?.evaluatorLogPaths?.jsonSummary;
+
+  if (to === undefined) {
+    toLog = fromLog;
+    fromLog = "";
+  }
+  if (fromLog === undefined || toLog === undefined) {
+    return extLogger.showWarningMessage(
+      `Cannot compare performance as the structured logs are missing. Did they queries complete normally?`,
+    );
+  }
+  await extLogger.log(
+    `Comparing performance of ${from.getQueryName()} and ${to?.getQueryName() ?? "baseline"}`,
+  );
+
+  await view.showResults(fromLog, toLog);
+}
+
 function addUnhandledRejectionListener() {
   const handler = (error: unknown) => {
     // This listener will be triggered for errors from other extensions as

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -1,6 +1,7 @@
-import type { SummaryEvent } from "./log-summary";
-import { readJsonlFile } from "../common/jsonl-reader";
 import type { Disposable } from "../common/disposable-object";
+import { readJsonlFile } from "../common/jsonl-reader";
+import type { ProgressCallback } from "../common/vscode/progress";
+import type { SummaryEvent } from "./log-summary";
 
 /**
  * Callback interface used to report diagnostics from a log scanner.
@@ -112,3 +113,27 @@ export class EvaluationLogScannerSet {
     scanners.forEach((scanner) => scanner.onDone());
   }
 }
+
+/**
+ * Scan the evaluator summary log using the given scanner. For convenience, returns the scanner.
+ *
+ * @param jsonSummaryLocation The file path of the JSON summary log.
+ * @param scanner The scanner to process events from the log
+ */
+export async function scanLog<T extends EvaluationLogScanner>(
+  jsonSummaryLocation: string,
+  scanner: T,
+  progress?: ProgressCallback,
+): Promise<T> {
+  progress?.({
+    // all scans have step 1 - the backing progress tracker allows increments instead of steps - but for now we are happy with a tiny UI that says what is happening
+    message: `Scanning ...`,
+    step: 1,
+    maxStep: 2,
+  });
+  await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
+    scanner.onEvent(obj);
+  });
+  scanner.onDone();
+  return scanner;
+}

extensions/ql-vscode/src/log-insights/log-summary.ts

@@ -33,6 +33,7 @@ interface ResultEventBase extends SummaryEventBase {
 export interface ComputeSimple extends ResultEventBase {
   evaluationStrategy: "COMPUTE_SIMPLE";
   ra: Ra;
+  millis: number;
   pipelineRuns?: [PipelineRun];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };
@@ -42,6 +43,7 @@ export interface ComputeRecursive extends ResultEventBase {
   evaluationStrategy: "COMPUTE_RECURSIVE";
   deltaSizes: number[];
   ra: Ra;
+  millis: number;
   pipelineRuns: PipelineRun[];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };

extensions/ql-vscode/src/log-insights/performance-comparison.ts

@@ -0,0 +1,183 @@
+import type { EvaluationLogScanner } from "./log-scanner";
+import type { SummaryEvent } from "./log-summary";
+
+export interface PipelineSummary {
+  steps: string[];
+  /** Total counts for each step in the RA array, across all iterations */
+  counts: number[];
+}
+
+/**
+ * Data extracted from a log for the purpose of doing a performance comparison.
+ *
+ * Memory compactness is important since we keep this data in memory; once for
+ * each side of the comparison.
+ *
+ * This object must be able to survive a `postMessage` transfer from the extension host
+ * to a web view (which rules out `Map` values, for example).
+ */
+export interface PerformanceComparisonDataFromLog {
+  /**
+   * Names of predicates mentioned in the log.
+   *
+   * For compactness, details of these predicates are stored in a "struct of arrays" style.
+   *
+   * All fields (except those ending with `Indices`) should contain an array of the same length as `names`;
+   * details of a given predicate should be stored at the same index in each of those arrays.
+   */
+  names: string[];
+
+  /** Number of milliseconds spent evaluating the `i`th predicate from the `names` array. */
+  timeCosts: number[];
+
+  /** Number of tuples seen in pipelines evaluating the `i`th predicate from the `names` array. */
+  tupleCosts: number[];
+
+  /** Number of iterations seen when evaluating the `i`th predicate from the `names` array. */
+  iterationCounts: number[];
+
+  /** Number of executions of pipelines evaluating the `i`th predicate from the `names` array. */
+  evaluationCounts: number[];
+
+  /**
+   * List of indices into the `names` array for which we have seen a cache hit.
+   */
+  cacheHitIndices: number[];
+
+  /**
+   * List of indices into the `names` array where the predicate was deemed empty due to a sentinel check.
+   */
+  sentinelEmptyIndices: number[];
+
+  /**
+   * All the pipeline runs seen for the `i`th predicate from the `names` array.
+   */
+  pipelineSummaryList: Array<Record<string, PipelineSummary>>;
+}
+
+export class PerformanceOverviewScanner implements EvaluationLogScanner {
+  private readonly nameToIndex = new Map<string, number>();
+  private readonly data: PerformanceComparisonDataFromLog = {
+    names: [],
+    timeCosts: [],
+    tupleCosts: [],
+    cacheHitIndices: [],
+    sentinelEmptyIndices: [],
+    pipelineSummaryList: [],
+    evaluationCounts: [],
+    iterationCounts: [],
+  };
+
+  private getPredicateIndex(name: string): number {
+    const { nameToIndex } = this;
+    let index = nameToIndex.get(name);
+    if (index === undefined) {
+      index = nameToIndex.size;
+      nameToIndex.set(name, index);
+      const {
+        names,
+        timeCosts,
+        tupleCosts,
+        iterationCounts,
+        evaluationCounts,
+        pipelineSummaryList,
+      } = this.data;
+      names.push(name);
+      timeCosts.push(0);
+      tupleCosts.push(0);
+      iterationCounts.push(0);
+      evaluationCounts.push(0);
+      pipelineSummaryList.push({});
+    }
+    return index;
+  }
+
+  getData(): PerformanceComparisonDataFromLog {
+    return this.data;
+  }
+
+  onEvent(event: SummaryEvent): void {
+    if (
+      event.completionType !== undefined &&
+      event.completionType !== "SUCCESS"
+    ) {
+      return; // Skip any evaluation that wasn't successful
+    }
+
+    switch (event.evaluationStrategy) {
+      case "EXTENSIONAL":
+      case "COMPUTED_EXTENSIONAL": {
+        break;
+      }
+      case "CACHE_HIT":
+      case "CACHACA": {
+        // Record a cache hit, but only if the predicate has not been seen before.
+        // We're mainly interested in the reuse of caches from an earlier query run as they can distort comparisons.
+        if (!this.nameToIndex.has(event.predicateName)) {
+          this.data.cacheHitIndices.push(
+            this.getPredicateIndex(event.predicateName),
+          );
+        }
+        break;
+      }
+      case "SENTINEL_EMPTY": {
+        this.data.sentinelEmptyIndices.push(
+          this.getPredicateIndex(event.predicateName),
+        );
+        break;
+      }
+      case "COMPUTE_RECURSIVE":
+      case "COMPUTE_SIMPLE":
+      case "IN_LAYER": {
+        const index = this.getPredicateIndex(event.predicateName);
+        let totalTime = 0;
+        let totalTuples = 0;
+        if (event.evaluationStrategy !== "IN_LAYER") {
+          totalTime += event.millis;
+        } else {
+          // IN_LAYER events do no record of their total time.
+          // Make a best-effort estimate by adding up the positive iteration times (they can be negative).
+          for (const millis of event.predicateIterationMillis ?? []) {
+            if (millis > 0) {
+              totalTime += millis;
+            }
+          }
+        }
+        const {
+          timeCosts,
+          tupleCosts,
+          iterationCounts,
+          evaluationCounts,
+          pipelineSummaryList,
+        } = this.data;
+        const pipelineSummaries = pipelineSummaryList[index];
+        for (const { counts, raReference } of event.pipelineRuns ?? []) {
+          // Get or create the pipeline summary for this RA
+          const pipelineSummary = (pipelineSummaries[raReference] ??= {
+            steps: event.ra[raReference],
+            counts: counts.map(() => 0),
+          });
+          const { counts: totalTuplesPerStep } = pipelineSummary;
+          for (let i = 0, length = counts.length; i < length; ++i) {
+            const count = counts[i];
+            if (count < 0) {
+              // Empty RA lines have a tuple count of -1. Do not count them when aggregating.
+              // But retain the fact that this step had a negative count for rendering purposes.
+              totalTuplesPerStep[i] = count;
+              continue;
+            }
+            totalTuples += count;
+            totalTuplesPerStep[i] += count;
+          }
+        }
+        timeCosts[index] += totalTime;
+        tupleCosts[index] += totalTuples;
+        iterationCounts[index] += event.pipelineRuns?.length ?? 0;
+        evaluationCounts[index] += 1;
+        break;
+      }
+    }
+  }
+
+  onDone(): void {}
+}

extensions/ql-vscode/src/query-history/query-history-manager.ts

@@ -149,6 +149,10 @@ export class QueryHistoryManager extends DisposableObject {
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo,
     ) => Promise<void>,
+    private readonly doComparePerformanceCallback: (
+      from: CompletedLocalQueryInfo,
+      to: CompletedLocalQueryInfo | undefined,
+    ) => Promise<void>,
   ) {
     super();
 
@@ -263,6 +267,8 @@ export class QueryHistoryManager extends DisposableObject {
         "query",
       ),
       "codeQLQueryHistory.compareWith": this.handleCompareWith.bind(this),
+      "codeQLQueryHistory.comparePerformanceWith":
+        this.handleComparePerformanceWith.bind(this),
       "codeQLQueryHistory.showEvalLog": createSingleSelectionCommand(
         this.app.logger,
         this.handleShowEvalLog.bind(this),
@@ -679,6 +685,39 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
+  async handleComparePerformanceWith(
+    singleItem: QueryHistoryInfo,
+    multiSelect: QueryHistoryInfo[] | undefined,
+  ) {
+    multiSelect ||= [singleItem];
+
+    if (
+      !this.isSuccessfulCompletedLocalQueryInfo(singleItem) ||
+      !multiSelect.every(this.isSuccessfulCompletedLocalQueryInfo)
+    ) {
+      throw new Error(
+        "Please only select local queries that have completed successfully.",
+      );
+    }
+
+    const fromItem = this.getFromQueryToCompare(singleItem, multiSelect);
+
+    let toItem: CompletedLocalQueryInfo | undefined = undefined;
+    try {
+      toItem = await this.findOtherQueryToComparePerformance(
+        fromItem,
+        multiSelect,
+      );
+    } catch (e) {
+      void showAndLogErrorMessage(
+        this.app.logger,
+        `Failed to compare queries: ${getErrorMessage(e)}`,
+      );
+    }
+
+    await this.doComparePerformanceCallback(fromItem, toItem);
+  }
+
   async handleItemClicked(item: QueryHistoryInfo) {
     this.treeDataProvider.setCurrentItem(item);
 
@@ -1076,6 +1115,7 @@ export class QueryHistoryManager extends DisposableObject {
         detail: item.completedQuery.message,
         query: item,
       }));
+
     if (comparableQueryLabels.length < 1) {
       throw new Error("No other queries available to compare with.");
     }
@@ -1084,6 +1124,52 @@ export class QueryHistoryManager extends DisposableObject {
     return choice?.query;
   }
 
+  private async findOtherQueryToComparePerformance(
+    fromItem: CompletedLocalQueryInfo,
+    allSelectedItems: CompletedLocalQueryInfo[],
+  ): Promise<CompletedLocalQueryInfo | undefined> {
+    // If exactly 2 items are selected, return the one that
+    // isn't being used as the "from" item.
+    if (allSelectedItems.length === 2) {
+      const otherItem =
+        fromItem === allSelectedItems[0]
+          ? allSelectedItems[1]
+          : allSelectedItems[0];
+      return otherItem;
+    }
+
+    if (allSelectedItems.length > 2) {
+      throw new Error("Please select no more than 2 queries.");
+    }
+
+    // Otherwise, present a dialog so the user can choose the item they want to use.
+    const comparableQueryLabels = this.treeDataProvider.allHistory
+      .filter(this.isSuccessfulCompletedLocalQueryInfo)
+      .filter((otherItem) => otherItem !== fromItem)
+      .map((item) => ({
+        label: this.labelProvider.getLabel(item),
+        description: item.databaseName,
+        detail: item.completedQuery.message,
+        query: item,
+      }));
+    const comparableQueryLabelsWithDefault = [
+      {
+        label: "Single run",
+        description:
+          "Look at the performance of this run, compared to a trivial baseline",
+        detail: undefined,
+        query: undefined,
+      },
+      ...comparableQueryLabels,
+    ];
+    if (comparableQueryLabelsWithDefault.length < 1) {
+      throw new Error("No other queries available to compare with.");
+    }
+    const choice = await window.showQuickPick(comparableQueryLabelsWithDefault);
+
+    return choice?.query;
+  }
+
   /**
    * Updates the compare with source query. This ensures that all compare command invocations
    * when exactly 2 queries are selected always have the proper _from_ query. Always use

extensions/ql-vscode/src/view/common/WarningBox.tsx

@@ -0,0 +1,31 @@
+import { styled } from "styled-components";
+import { WarningIcon } from "./icon/WarningIcon";
+
+const WarningBoxDiv = styled.div`
+  max-width: 100em;
+  padding: 0.5em 1em;
+  border: 1px solid var(--vscode-widget-border);
+  box-shadow: var(--vscode-widget-shadow) 0px 3px 8px;
+  display: flex;
+`;
+
+const IconPane = styled.p`
+  width: 3em;
+  flex-shrink: 0;
+  text-align: center;
+`;
+
+export interface WarningBoxProps {
+  children: React.ReactNode;
+}
+
+export function WarningBox(props: WarningBoxProps) {
+  return (
+    <WarningBoxDiv>
+      <IconPane>
+        <WarningIcon />
+      </IconPane>
+      <p>{props.children}</p>
+    </WarningBoxDiv>
+  );
+}

extensions/ql-vscode/src/view/common/index.ts

@@ -6,3 +6,4 @@ export * from "./HorizontalSpace";
 export * from "./SectionTitle";
 export * from "./VerticalSpace";
 export * from "./ViewTitle";
+export * from "./WarningBox";

extensions/ql-vscode/src/view/compare-performance/ComparePerformance.tsx

@@ -0,0 +1,862 @@
+import type { ChangeEvent } from "react";
+import {
+  Fragment,
+  memo,
+  useDeferredValue,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import type {
+  SetPerformanceComparisonQueries,
+  ToComparePerformanceViewMessage,
+} from "../../common/interface-types";
+import { useMessageFromExtension } from "../common/useMessageFromExtension";
+import type {
+  PerformanceComparisonDataFromLog,
+  PipelineSummary,
+} from "../../log-insights/performance-comparison";
+import { formatDecimal } from "../../common/number";
+import { styled } from "styled-components";
+import { Codicon, ViewTitle, WarningBox } from "../common";
+import { abbreviateRANames, abbreviateRASteps } from "./RAPrettyPrinter";
+import { Renaming, RenamingInput } from "./RenamingInput";
+
+const enum AbsentReason {
+  NotSeen = "NotSeen",
+  CacheHit = "CacheHit",
+  Sentinel = "Sentinel",
+}
+
+type Optional<T> = AbsentReason | T;
+
+function isPresent<T>(x: Optional<T>): x is T {
+  return typeof x !== "string";
+}
+
+interface PredicateInfo {
+  tuples: number;
+  evaluationCount: number;
+  iterationCount: number;
+  timeCost: number;
+  pipelines: Record<string, PipelineSummary>;
+}
+
+class ComparisonDataset {
+  public nameToIndex = new Map<string, number>();
+  public cacheHitIndices: Set<number>;
+  public sentinelEmptyIndices: Set<number>;
+
+  constructor(public data: PerformanceComparisonDataFromLog) {
+    const { names } = data;
+    const { nameToIndex } = this;
+    for (let i = 0; i < names.length; i++) {
+      nameToIndex.set(names[i], i);
+    }
+    this.cacheHitIndices = new Set(data.cacheHitIndices);
+    this.sentinelEmptyIndices = new Set(data.sentinelEmptyIndices);
+  }
+
+  getTupleCountInfo(name: string): Optional<PredicateInfo> {
+    const { data, nameToIndex, cacheHitIndices, sentinelEmptyIndices } = this;
+    const index = nameToIndex.get(name);
+    if (index == null) {
+      return AbsentReason.NotSeen;
+    }
+    const tupleCost = data.tupleCosts[index];
+    if (tupleCost === 0) {
+      if (sentinelEmptyIndices.has(index)) {
+        return AbsentReason.Sentinel;
+      } else if (cacheHitIndices.has(index)) {
+        return AbsentReason.CacheHit;
+      }
+    }
+    return {
+      evaluationCount: data.evaluationCounts[index],
+      iterationCount: data.iterationCounts[index],
+      timeCost: data.timeCosts[index],
+      tuples: tupleCost,
+      pipelines: data.pipelineSummaryList[index],
+    };
+  }
+}
+
+function renderOptionalValue(x: Optional<number>, unit?: string) {
+  switch (x) {
+    case AbsentReason.NotSeen:
+      return <AbsentNumberCell>n/a</AbsentNumberCell>;
+    case AbsentReason.CacheHit:
+      return <AbsentNumberCell>cache hit</AbsentNumberCell>;
+    case AbsentReason.Sentinel:
+      return <AbsentNumberCell>sentinel empty</AbsentNumberCell>;
+    default:
+      return (
+        <NumberCell>
+          {formatDecimal(x)}
+          {renderUnit(unit)}
+        </NumberCell>
+      );
+  }
+}
+
+function renderPredicateMetric(
+  x: Optional<PredicateInfo>,
+  metric: Metric,
+  isPerEvaluation: boolean,
+) {
+  return renderOptionalValue(
+    metricGetOptional(metric, x, isPerEvaluation),
+    metric.unit,
+  );
+}
+
+function renderDelta(x: number, unit?: string) {
+  const sign = x > 0 ? "+" : "";
+  return (
+    <NumberCell className={x > 0 ? "bad-value" : x < 0 ? "good-value" : ""}>
+      {sign}
+      {formatDecimal(x)}
+      {renderUnit(unit)}
+    </NumberCell>
+  );
+}
+
+function renderUnit(unit: string | undefined) {
+  return unit == null ? "" : ` ${unit}`;
+}
+
+function orderBy<T>(fn: (x: T) => number | string) {
+  return (x: T, y: T) => {
+    const fx = fn(x);
+    const fy = fn(y);
+    return fx === fy ? 0 : fx < fy ? -1 : 1;
+  };
+}
+
+const ChevronCell = styled.td`
+  width: 1em !important;
+`;
+
+const NameHeader = styled.th`
+  text-align: left;
+`;
+
+const NumberHeader = styled.th`
+  text-align: right;
+  width: 10em !important;
+`;
+
+const NameCell = styled.td``;
+
+const NumberCell = styled.td`
+  text-align: right;
+  width: 10em !important;
+
+  &.bad-value {
+    color: var(--vscode-problemsErrorIcon-foreground);
+    tr.expanded & {
+      color: inherit;
+    }
+  }
+  &.good-value {
+    color: var(--vscode-problemsInfoIcon-foreground);
+    tr.expanded & {
+      color: inherit;
+    }
+  }
+`;
+
+const AbsentNumberCell = styled.td`
+  text-align: right;
+  color: var(--vscode-disabledForeground);
+
+  tr.expanded & {
+    color: inherit;
+  }
+  width: 10em !important;
+`;
+
+const Table = styled.table`
+  border-collapse: collapse;
+  width: 100%;
+  border-spacing: 0;
+  background-color: var(--vscode-background);
+  color: var(--vscode-foreground);
+  & td {
+    padding: 0.5em;
+  }
+  & th {
+    padding: 0.5em;
+  }
+  &.expanded {
+    border: 1px solid var(--vscode-list-activeSelectionBackground);
+    margin-bottom: 1em;
+  }
+  word-break: break-all;
+`;
+
+const PredicateTR = styled.tr`
+  cursor: pointer;
+
+  &.expanded {
+    background-color: var(--vscode-list-activeSelectionBackground);
+    color: var(--vscode-list-activeSelectionForeground);
+    position: sticky;
+    top: 0;
+  }
+
+  & .codicon-chevron-right {
+    visibility: hidden;
+  }
+
+  &:hover:not(.expanded) {
+    background-color: var(--vscode-list-hoverBackground);
+    & .codicon-chevron-right {
+      visibility: visible;
+    }
+  }
+`;
+
+const PipelineStepTR = styled.tr`
+  & td {
+    padding-top: 0.3em;
+    padding-bottom: 0.3em;
+  }
+`;
+
+const Dropdown = styled.select``;
+
+interface PipelineStepProps {
+  before: number | undefined;
+  after: number | undefined;
+  comparison: boolean;
+  step: React.ReactNode;
+}
+
+/**
+ * Row with details of a pipeline step, or one of the high-level stats appearing above the pipelines (evaluation/iteration counts).
+ */
+function PipelineStep(props: PipelineStepProps) {
+  let { before, after, comparison, step } = props;
+  if (before != null && before < 0) {
+    before = undefined;
+  }
+  if (after != null && after < 0) {
+    after = undefined;
+  }
+  const delta = before != null && after != null ? after - before : undefined;
+  return (
+    <PipelineStepTR>
+      <ChevronCell />
+      {comparison && (
+        <NumberCell>{before != null ? formatDecimal(before) : ""}</NumberCell>
+      )}
+      <NumberCell>{after != null ? formatDecimal(after) : ""}</NumberCell>
+      {comparison && (delta != null ? renderDelta(delta) : <td></td>)}
+      <NameCell>{step}</NameCell>
+    </PipelineStepTR>
+  );
+}
+
+const HeaderTR = styled.tr`
+  background-color: var(--vscode-sideBar-background);
+`;
+
+interface HeaderRowProps {
+  hasBefore?: boolean;
+  hasAfter?: boolean;
+  comparison: boolean;
+  title: React.ReactNode;
+}
+
+function HeaderRow(props: HeaderRowProps) {
+  const { comparison, hasBefore, hasAfter, title } = props;
+  return (
+    <HeaderTR>
+      <ChevronCell />
+      {comparison ? (
+        <>
+          <NumberHeader>{hasBefore ? "Before" : ""}</NumberHeader>
+          <NumberHeader>{hasAfter ? "After" : ""}</NumberHeader>
+          <NumberHeader>{hasBefore && hasAfter ? "Delta" : ""}</NumberHeader>
+        </>
+      ) : (
+        <NumberHeader>Value</NumberHeader>
+      )}
+      <NameHeader>{title}</NameHeader>
+    </HeaderTR>
+  );
+}
+
+interface HighLevelStatsProps {
+  before: Optional<PredicateInfo>;
+  after: Optional<PredicateInfo>;
+  comparison: boolean;
+}
+
+function HighLevelStats(props: HighLevelStatsProps) {
+  const { before, after, comparison } = props;
+  const hasBefore = isPresent(before);
+  const hasAfter = isPresent(after);
+  const showEvaluationCount =
+    (hasBefore && before.evaluationCount > 1) ||
+    (hasAfter && after.evaluationCount > 1);
+  return (
+    <>
+      <HeaderRow
+        hasBefore={hasBefore}
+        hasAfter={hasAfter}
+        title="Stats"
+        comparison={comparison}
+      />
+      {showEvaluationCount && (
+        <PipelineStep
+          before={hasBefore ? before.evaluationCount : undefined}
+          after={hasAfter ? after.evaluationCount : undefined}
+          comparison={comparison}
+          step="Number of evaluations"
+        />
+      )}
+      <PipelineStep
+        before={
+          hasBefore ? before.iterationCount / before.evaluationCount : undefined
+        }
+        after={
+          hasAfter ? after.iterationCount / after.evaluationCount : undefined
+        }
+        comparison={comparison}
+        step={
+          showEvaluationCount
+            ? "Number of iterations per evaluation"
+            : "Number of iterations"
+        }
+      />
+    </>
+  );
+}
+
+interface Row {
+  name: string;
+  before: Optional<PredicateInfo>;
+  after: Optional<PredicateInfo>;
+  diff: number;
+}
+
+/**
+ * A set of predicates that have been grouped together because their names have the same fingerprint.
+ */
+interface RowGroup {
+  name: string;
+  rows: Row[];
+  before: Optional<number>;
+  after: Optional<number>;
+  diff: number;
+}
+
+function getSortOrder(sortOrder: "delta" | "absDelta") {
+  if (sortOrder === "absDelta") {
+    return orderBy((row: { diff: number }) => -Math.abs(row.diff));
+  }
+  return orderBy((row: { diff: number }) => row.diff);
+}
+
+interface Metric {
+  title: string;
+  get(info: PredicateInfo): number;
+  unit?: string;
+}
+
+const metrics: Record<string, Metric> = {
+  tuples: {
+    title: "Tuple count",
+    get: (info) => info.tuples,
+  },
+  time: {
+    title: "Time spent",
+    get: (info) => info.timeCost,
+    unit: "ms",
+  },
+  evaluations: {
+    title: "Evaluations",
+    get: (info) => info.evaluationCount,
+  },
+  iterationsTotal: {
+    title: "Iterations",
+    get: (info) => info.iterationCount,
+  },
+};
+
+function metricGetOptional(
+  metric: Metric,
+  info: Optional<PredicateInfo>,
+  isPerEvaluation: boolean,
+): Optional<number> {
+  if (!isPresent(info)) {
+    return info;
+  }
+  const value = metric.get(info);
+  return isPerEvaluation ? (value / info.evaluationCount) | 0 : value;
+}
+
+function addOptionals(a: Optional<number>, b: Optional<number>) {
+  if (isPresent(a) && isPresent(b)) {
+    return a + b;
+  }
+  if (isPresent(a)) {
+    return a;
+  }
+  if (isPresent(b)) {
+    return b;
+  }
+  if (a === b) {
+    return a; // If absent for the same reason, preserve that reason
+  }
+  return 0; // Otherwise collapse to zero
+}
+
+/**
+ * Returns a "fingerprint" from the given name, which is used to group together similar names.
+ */
+function getNameFingerprint(name: string, renamings: Renaming[]) {
+  for (const { patternRegexp, replacement } of renamings) {
+    if (patternRegexp != null) {
+      name = name.replace(patternRegexp, replacement);
+    }
+  }
+  return name;
+}
+
+function Chevron({ expanded }: { expanded: boolean }) {
+  return <Codicon name={expanded ? "chevron-down" : "chevron-right"} />;
+}
+
+function union<T>(a: Set<T> | T[], b: Set<T> | T[]) {
+  const result = new Set(a);
+  for (const x of b) {
+    result.add(x);
+  }
+  return result;
+}
+
+export function ComparePerformance(_: Record<string, never>) {
+  const [data, setData] = useState<
+    SetPerformanceComparisonQueries | undefined
+  >();
+
+  useMessageFromExtension<ToComparePerformanceViewMessage>(
+    (msg) => {
+      setData(msg);
+    },
+    [setData],
+  );
+
+  if (!data) {
+    return <div>Loading performance comparison...</div>;
+  }
+
+  return <ComparePerformanceWithData data={data} />;
+}
+
+function ComparePerformanceWithData(props: {
+  data: SetPerformanceComparisonQueries;
+}) {
+  const { data } = props;
+
+  const { from, to } = useMemo(
+    () => ({
+      from: new ComparisonDataset(data.from),
+      to: new ComparisonDataset(data.to),
+    }),
+    [data],
+  );
+
+  const comparison = data?.comparison;
+
+  const [hideCacheHits, setHideCacheHits] = useState(false);
+
+  const [sortOrder, setSortOrder] = useState<"delta" | "absDelta">("absDelta");
+
+  const [metric, setMetric] = useState<Metric>(metrics.tuples);
+
+  const [isPerEvaluation, setPerEvaluation] = useState(false);
+
+  const nameSet = useMemo(
+    () => union(from.data.names, to.data.names),
+    [from, to],
+  );
+
+  const hasCacheHitMismatch = useRef(false);
+
+  const rows: Row[] = useMemo(() => {
+    hasCacheHitMismatch.current = false;
+    return Array.from(nameSet)
+      .map((name) => {
+        const before = from.getTupleCountInfo(name);
+        const after = to.getTupleCountInfo(name);
+        const beforeValue = metricGetOptional(metric, before, isPerEvaluation);
+        const afterValue = metricGetOptional(metric, after, isPerEvaluation);
+        if (beforeValue === afterValue) {
+          return undefined!;
+        }
+        if (
+          before === AbsentReason.CacheHit ||
+          after === AbsentReason.CacheHit
+        ) {
+          hasCacheHitMismatch.current = true;
+          if (hideCacheHits) {
+            return undefined!;
+          }
+        }
+        const diff =
+          (isPresent(afterValue) ? afterValue : 0) -
+          (isPresent(beforeValue) ? beforeValue : 0);
+        return { name, before, after, diff } satisfies Row;
+      })
+      .filter((x) => !!x)
+      .sort(getSortOrder(sortOrder));
+  }, [nameSet, from, to, metric, hideCacheHits, sortOrder, isPerEvaluation]);
+
+  const { totalBefore, totalAfter, totalDiff } = useMemo(() => {
+    let totalBefore = 0;
+    let totalAfter = 0;
+    let totalDiff = 0;
+    for (const row of rows) {
+      totalBefore += isPresent(row.before) ? metric.get(row.before) : 0;
+      totalAfter += isPresent(row.after) ? metric.get(row.after) : 0;
+      totalDiff += row.diff;
+    }
+    return { totalBefore, totalAfter, totalDiff };
+  }, [rows, metric]);
+
+  const [renamings, setRenamings] = useState<Renaming[]>(() => [
+    new Renaming("#[0-9a-f]{8}(?![0-9a-f])", "#"),
+  ]);
+
+  // Use deferred value to avoid expensive re-rendering for every keypress in the renaming editor
+  const deferredRenamings = useDeferredValue(renamings);
+
+  const rowGroups = useMemo(() => {
+    const groupedRows = new Map<string, Row[]>();
+    for (const row of rows) {
+      const fingerprint = getNameFingerprint(row.name, deferredRenamings);
+      const rows = groupedRows.get(fingerprint);
+      if (rows) {
+        rows.push(row);
+      } else {
+        groupedRows.set(fingerprint, [row]);
+      }
+    }
+    return Array.from(groupedRows.entries())
+      .map(([fingerprint, rows]) => {
+        const before = rows
+          .map((row) => metricGetOptional(metric, row.before, isPerEvaluation))
+          .reduce(addOptionals);
+        const after = rows
+          .map((row) => metricGetOptional(metric, row.after, isPerEvaluation))
+          .reduce(addOptionals);
+        return {
+          name: rows.length === 1 ? rows[0].name : fingerprint,
+          before,
+          after,
+          diff:
+            (isPresent(after) ? after : 0) - (isPresent(before) ? before : 0),
+          rows,
+        } satisfies RowGroup;
+      })
+      .sort(getSortOrder(sortOrder));
+  }, [rows, metric, sortOrder, deferredRenamings, isPerEvaluation]);
+
+  const rowGroupNames = useMemo(
+    () => abbreviateRANames(rowGroups.map((group) => group.name)),
+    [rowGroups],
+  );
+
+  return (
+    <>
+      <ViewTitle>Performance comparison</ViewTitle>
+      {comparison && hasCacheHitMismatch.current && (
+        <WarningBox>
+          <strong>Inconsistent cache hits</strong>
+          <br />
+          Some predicates had a cache hit on one side but not the other. For
+          more accurate results, try running the{" "}
+          <strong>CodeQL: Clear Cache</strong> command before each query.
+          <br />
+          <br />
+          <label>
+            <input
+              type="checkbox"
+              checked={hideCacheHits}
+              onChange={() => setHideCacheHits(!hideCacheHits)}
+            />
+            Hide predicates with cache hits
+          </label>
+        </WarningBox>
+      )}
+      <RenamingInput renamings={renamings} setRenamings={setRenamings} />
+      Compare{" "}
+      <Dropdown
+        onChange={(e: ChangeEvent<HTMLSelectElement>) =>
+          setMetric(metrics[e.target.value])
+        }
+      >
+        {Object.entries(metrics).map(([key, value]) => (
+          <option key={key} value={key}>
+            {value.title}
+          </option>
+        ))}
+      </Dropdown>{" "}
+      <Dropdown
+        onChange={(e: ChangeEvent<HTMLSelectElement>) =>
+          setPerEvaluation(e.target.value === "per-evaluation")
+        }
+      >
+        <option value="total">Overall</option>
+        <option value="per-evaluation">Per evaluation</option>
+      </Dropdown>{" "}
+      {comparison && (
+        <>
+          sorted by{" "}
+          <Dropdown
+            onChange={(e: ChangeEvent<HTMLSelectElement>) =>
+              setSortOrder(e.target.value as "delta" | "absDelta")
+            }
+            value={sortOrder}
+          >
+            <option value="delta">Delta</option>
+            <option value="absDelta">Absolute delta</option>
+          </Dropdown>
+        </>
+      )}
+      <Table>
+        <thead>
+          <HeaderRow comparison={comparison} title="Predicate" />
+        </thead>
+        <tbody>
+          <tr key="total">
+            <ChevronCell />
+            {comparison && renderOptionalValue(totalBefore, metric.unit)}
+            {renderOptionalValue(totalAfter, metric.unit)}
+            {comparison && renderDelta(totalDiff, metric.unit)}
+            <NameCell>
+              <strong>TOTAL</strong>
+            </NameCell>
+          </tr>
+          <tr key="spacing">
+            <td colSpan={5} style={{ height: "1em" }}></td>
+          </tr>
+        </tbody>
+      </Table>
+      <PredicateTable
+        rowGroups={rowGroups}
+        rowGroupNames={rowGroupNames}
+        comparison={comparison}
+        metric={metric}
+        isPerEvaluation={isPerEvaluation}
+      />
+    </>
+  );
+}
+
+interface PredicateTableProps {
+  rowGroups: RowGroup[];
+  rowGroupNames: React.ReactNode[];
+  comparison: boolean;
+  metric: Metric;
+  isPerEvaluation: boolean;
+}
+
+function PredicateTableRaw(props: PredicateTableProps) {
+  const { comparison, metric, rowGroupNames, rowGroups, isPerEvaluation } =
+    props;
+  return rowGroups.map((rowGroup, rowGroupIndex) => (
+    <PredicateRowGroup
+      key={rowGroupIndex}
+      renderedName={rowGroupNames[rowGroupIndex]}
+      rowGroup={rowGroup}
+      comparison={comparison}
+      metric={metric}
+      isPerEvaluation={isPerEvaluation}
+    />
+  ));
+}
+
+const PredicateTable = memo(PredicateTableRaw);
+
+interface PredicateRowGroupProps {
+  renderedName: React.ReactNode;
+  rowGroup: RowGroup;
+  comparison: boolean;
+  metric: Metric;
+  isPerEvaluation: boolean;
+}
+
+function PredicateRowGroup(props: PredicateRowGroupProps) {
+  const { renderedName, rowGroup, comparison, metric, isPerEvaluation } = props;
+  const [isExpanded, setExpanded] = useState(false);
+  const rowNames = useMemo(
+    () => abbreviateRANames(rowGroup.rows.map((row) => row.name)),
+    [rowGroup],
+  );
+  if (rowGroup.rows.length === 1) {
+    return <PredicateRow row={rowGroup.rows[0]} {...props} />;
+  }
+  return (
+    <Table className={isExpanded ? "expanded" : ""}>
+      <tbody>
+        <PredicateTR
+          className={isExpanded ? "expanded" : ""}
+          key={"main"}
+          onClick={() => setExpanded(!isExpanded)}
+        >
+          <ChevronCell>
+            <Chevron expanded={isExpanded} />
+          </ChevronCell>
+          {comparison && renderOptionalValue(rowGroup.before)}
+          {renderOptionalValue(rowGroup.after)}
+          {comparison && renderDelta(rowGroup.diff, metric.unit)}
+          <NameCell>
+            {renderedName} ({rowGroup.rows.length} predicates)
+          </NameCell>
+        </PredicateTR>
+        {isExpanded &&
+          rowGroup.rows.map((row, rowIndex) => (
+            <tr key={rowIndex}>
+              <td colSpan={5}>
+                <PredicateRow
+                  renderedName={rowNames[rowIndex]}
+                  row={row}
+                  comparison={comparison}
+                  metric={metric}
+                  isPerEvaluation={isPerEvaluation}
+                />
+              </td>
+            </tr>
+          ))}
+      </tbody>
+    </Table>
+  );
+}
+
+interface PredicateRowProps {
+  renderedName: React.ReactNode;
+  row: Row;
+  comparison: boolean;
+  metric: Metric;
+  isPerEvaluation: boolean;
+}
+
+function PredicateRow(props: PredicateRowProps) {
+  const [isExpanded, setExpanded] = useState(false);
+  const { renderedName, row, comparison, metric, isPerEvaluation } = props;
+  const evaluationFactorBefore =
+    isPerEvaluation && isPresent(row.before) ? row.before.evaluationCount : 1;
+  const evaluationFactorAfter =
+    isPerEvaluation && isPresent(row.after) ? row.after.evaluationCount : 1;
+  return (
+    <Table className={isExpanded ? "expanded" : ""}>
+      <tbody>
+        <PredicateTR
+          className={isExpanded ? "expanded" : ""}
+          key={"main"}
+          onClick={() => setExpanded(!isExpanded)}
+        >
+          <ChevronCell>
+            <Chevron expanded={isExpanded} />
+          </ChevronCell>
+          {comparison &&
+            renderPredicateMetric(row.before, metric, isPerEvaluation)}
+          {renderPredicateMetric(row.after, metric, isPerEvaluation)}
+          {comparison && renderDelta(row.diff, metric.unit)}
+          <NameCell>{renderedName}</NameCell>
+        </PredicateTR>
+        {isExpanded && (
+          <>
+            <HighLevelStats
+              before={row.before}
+              after={row.after}
+              comparison={comparison}
+            />
+            {collatePipelines(
+              isPresent(row.before) ? row.before.pipelines : {},
+              isPresent(row.after) ? row.after.pipelines : {},
+            ).map(({ name, first, second }, pipelineIndex) => (
+              <Fragment key={pipelineIndex}>
+                <HeaderRow
+                  hasBefore={first != null}
+                  hasAfter={second != null}
+                  comparison={comparison}
+                  title={
+                    <>
+                      Tuple counts for &apos;{name}&apos; pipeline
+                      {comparison &&
+                        (first == null
+                          ? " (after)"
+                          : second == null
+                            ? " (before)"
+                            : "")}
+                    </>
+                  }
+                />
+                {abbreviateRASteps(first?.steps ?? second!.steps).map(
+                  (step, index) => (
+                    <PipelineStep
+                      key={index}
+                      before={
+                        first &&
+                        (first.counts[index] / evaluationFactorBefore) | 0
+                      }
+                      after={
+                        second &&
+                        (second.counts[index] / evaluationFactorAfter) | 0
+                      }
+                      comparison={comparison}
+                      step={step}
+                    />
+                  ),
+                )}
+              </Fragment>
+            ))}
+          </>
+        )}
+      </tbody>
+    </Table>
+  );
+}
+
+interface PipelinePair {
+  name: string;
+  first: PipelineSummary | undefined;
+  second: PipelineSummary | undefined;
+}
+
+function collatePipelines(
+  before: Record<string, PipelineSummary>,
+  after: Record<string, PipelineSummary>,
+): PipelinePair[] {
+  const result: PipelinePair[] = [];
+
+  for (const [name, first] of Object.entries(before)) {
+    const second = after[name];
+    if (second == null) {
+      result.push({ name, first, second: undefined });
+    } else if (samePipeline(first.steps, second.steps)) {
+      result.push({ name, first, second });
+    } else {
+      result.push({ name, first, second: undefined });
+      result.push({ name, first: undefined, second });
+    }
+  }
+
+  for (const [name, second] of Object.entries(after)) {
+    if (before[name] == null) {
+      result.push({ name, first: undefined, second });
+    }
+  }
+
+  return result;
+}
+
+function samePipeline(a: string[], b: string[]) {
+  return a.length === b.length && a.every((x, i) => x === b[i]);
+}

extensions/ql-vscode/src/view/compare-performance/RAPrettyPrinter.tsx

@@ -0,0 +1,282 @@
+import { Fragment, useState } from "react";
+import { styled } from "styled-components";
+
+/**
+ * A set of names, for generating unambiguous abbreviations.
+ */
+class NameSet {
+  private readonly abbreviations = new Map<string, React.ReactNode>();
+
+  constructor(readonly names: string[]) {
+    const qnames = names.map(parseName);
+    const builder = new TrieBuilder();
+    qnames
+      .map((qname) => builder.visitQName(qname))
+      .forEach((r, index) => {
+        this.abbreviations.set(names[index], r.abbreviate(true));
+      });
+  }
+
+  public getAbbreviation(name: string): React.ReactNode {
+    return this.abbreviations.get(name) ?? name;
+  }
+}
+
+/** Name parsed into the form `prefix::name<args>` */
+interface QualifiedName {
+  prefix?: QualifiedName;
+  name: string;
+  args?: QualifiedName[];
+}
+
+function qnameToString(name: QualifiedName): string {
+  const parts: string[] = [];
+  if (name.prefix != null) {
+    parts.push(qnameToString(name.prefix));
+    parts.push("::");
+  }
+  parts.push(name.name);
+  if (name.args != null && name.args.length > 0) {
+    parts.push("<");
+    parts.push(name.args.map(qnameToString).join(","));
+    parts.push(">");
+  }
+  return parts.join("");
+}
+
+function tokeniseName(text: string) {
+  return Array.from(text.matchAll(/:+|<|>|,|"[^"]+"|`[^`]+`|[^:<>,"`]+/g));
+}
+
+function parseName(text: string): QualifiedName {
+  const tokens = tokeniseName(text);
+
+  function next() {
+    return tokens.pop()![0];
+  }
+  function peek() {
+    return tokens[tokens.length - 1][0];
+  }
+  function skipToken(token: string) {
+    if (tokens.length > 0 && peek() === token) {
+      tokens.pop();
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  function parseQName(): QualifiedName {
+    // Note that the tokens stream is parsed in reverse order. This is simpler, but may look confusing initially.
+    let args: QualifiedName[] | undefined;
+    if (skipToken(">")) {
+      args = [];
+      while (tokens.length > 0 && peek() !== "<") {
+        args.push(parseQName());
+        skipToken(",");
+      }
+      args.reverse();
+      skipToken("<");
+    }
+    const name = tokens.length === 0 ? "" : next();
+    const prefix = skipToken("::") ? parseQName() : undefined;
+    return {
+      prefix,
+      name,
+      args,
+    };
+  }
+
+  const result = parseQName();
+  if (tokens.length > 0) {
+    // It's a parse error if we did not consume all tokens.
+    // Just treat the whole text as the 'name'.
+    return { prefix: undefined, name: text, args: undefined };
+  }
+  return result;
+}
+
+class TrieNode {
+  children = new Map<string, TrieNode>();
+  constructor(readonly index: number) {}
+}
+
+interface VisitResult {
+  node: TrieNode;
+  abbreviate: (isRoot?: boolean) => React.ReactNode;
+}
+
+class TrieBuilder {
+  root = new TrieNode(0);
+  nextId = 1;
+
+  getOrCreate(trieNode: TrieNode, child: string) {
+    const { children } = trieNode;
+    let node = children.get(child);
+    if (node == null) {
+      node = new TrieNode(this.nextId++);
+      children.set(child, node);
+    }
+    return node;
+  }
+
+  visitQName(qname: QualifiedName): VisitResult {
+    const prefix =
+      qname.prefix != null ? this.visitQName(qname.prefix) : undefined;
+    const trieNodeBeforeArgs = this.getOrCreate(
+      prefix?.node ?? this.root,
+      qname.name,
+    );
+    let trieNode = trieNodeBeforeArgs;
+    const args = qname.args?.map((arg) => this.visitQName(arg));
+    if (args != null) {
+      const argKey = args.map((arg) => arg.node.index).join(",");
+      trieNode = this.getOrCreate(trieNodeBeforeArgs, argKey);
+    }
+    return {
+      node: trieNode,
+      abbreviate: (isRoot = false) => {
+        const result: React.ReactNode[] = [];
+        if (prefix != null) {
+          result.push(prefix.abbreviate());
+          result.push("::");
+        }
+        const { name } = qname;
+        const hash = name.indexOf("#");
+        if (hash !== -1 && isRoot) {
+          const shortName = name.substring(0, hash);
+          result.push(<IdentifierSpan>{shortName}</IdentifierSpan>);
+          result.push(name.substring(hash));
+        } else {
+          result.push(isRoot ? <IdentifierSpan>{name}</IdentifierSpan> : name);
+        }
+        if (args != null) {
+          result.push("<");
+          if (trieNodeBeforeArgs.children.size === 1) {
+            const argsText = qname
+              .args!.map((arg) => qnameToString(arg))
+              .join(",");
+            result.push(<ExpandableNamePart>{argsText}</ExpandableNamePart>);
+          } else {
+            let first = true;
+            for (const arg of args) {
+              result.push(arg.abbreviate());
+              if (first) {
+                first = false;
+              } else {
+                result.push(",");
+              }
+            }
+          }
+          result.push(">");
+        }
+        return result;
+      },
+    };
+  }
+}
+
+const ExpandableTextButton = styled.button`
+  background: none;
+  border: none;
+  cursor: pointer;
+  padding: 0;
+  color: inherit;
+  &:hover {
+    background-color: rgba(128, 128, 128, 0.2);
+  }
+`;
+
+interface ExpandableNamePartProps {
+  children: React.ReactNode;
+}
+
+function ExpandableNamePart(props: ExpandableNamePartProps) {
+  const [isExpanded, setExpanded] = useState(false);
+  return (
+    <ExpandableTextButton
+      onClick={(event: Event) => {
+        setExpanded(!isExpanded);
+        event.stopPropagation();
+      }}
+    >
+      {isExpanded ? props.children : "..."}
+    </ExpandableTextButton>
+  );
+}
+
+/**
+ * Span enclosing an entire qualified name.
+ *
+ * Can be used to gray out uninteresting parts of the name, though this looks worse than expected.
+ */
+const QNameSpan = styled.span`
+  /* color: var(--vscode-disabledForeground); */
+`;
+
+/** Span enclosing the innermost identifier, e.g. the `foo` in `A::B<X>::foo#abc` */
+const IdentifierSpan = styled.span`
+  font-weight: 600;
+`;
+
+/** Span enclosing keywords such as `JOIN` and `WITH`. */
+const KeywordSpan = styled.span`
+  font-weight: 500;
+`;
+
+const nameTokenRegex = /\b[^ (]+\b/g;
+
+function traverseMatches(
+  text: string,
+  regex: RegExp,
+  callbacks: {
+    onMatch: (match: RegExpMatchArray) => void;
+    onText: (text: string) => void;
+  },
+) {
+  const matches = Array.from(text.matchAll(regex));
+  let lastIndex = 0;
+  for (const match of matches) {
+    const before = text.substring(lastIndex, match.index);
+    if (before !== "") {
+      callbacks.onText(before);
+    }
+    callbacks.onMatch(match);
+    lastIndex = match.index + match[0].length;
+  }
+  const after = text.substring(lastIndex);
+  if (after !== "") {
+    callbacks.onText(after);
+  }
+}
+
+export function abbreviateRASteps(steps: string[]): React.ReactNode[] {
+  const nameTokens = steps.flatMap((step) =>
+    Array.from(step.matchAll(nameTokenRegex)).map((tok) => tok[0]),
+  );
+  const nameSet = new NameSet(nameTokens.filter((name) => name.includes("::")));
+  return steps.map((step, index) => {
+    const result: React.ReactNode[] = [];
+    traverseMatches(step, nameTokenRegex, {
+      onMatch(match) {
+        const text = match[0];
+        if (text.includes("::")) {
+          result.push(<QNameSpan>{nameSet.getAbbreviation(text)}</QNameSpan>);
+        } else if (/[A-Z]+/.test(text)) {
+          result.push(<KeywordSpan>{text}</KeywordSpan>);
+        } else {
+          result.push(match[0]);
+        }
+      },
+      onText(text) {
+        result.push(text);
+      },
+    });
+    return <Fragment key={index}>{result}</Fragment>;
+  });
+}
+
+export function abbreviateRANames(names: string[]): React.ReactNode[] {
+  const nameSet = new NameSet(names);
+  return names.map((name) => nameSet.getAbbreviation(name));
+}

extensions/ql-vscode/src/view/compare-performance/RenamingInput.tsx

@@ -0,0 +1,106 @@
+import type { ChangeEvent } from "react";
+import { styled } from "styled-components";
+import {
+  VSCodeButton,
+  VSCodeTextField,
+} from "@vscode/webview-ui-toolkit/react";
+import { Codicon } from "../common";
+
+export class Renaming {
+  patternRegexp: RegExp | undefined;
+
+  constructor(
+    public pattern: string,
+    public replacement: string,
+  ) {
+    this.patternRegexp = tryCompilePattern(pattern);
+  }
+}
+
+function tryCompilePattern(pattern: string): RegExp | undefined {
+  try {
+    return new RegExp(pattern, "i");
+  } catch {
+    return undefined;
+  }
+}
+
+const Input = styled(VSCodeTextField)`
+  width: 20em;
+`;
+
+const Row = styled.div`
+  display: flex;
+  padding-bottom: 0.25em;
+`;
+
+const Details = styled.details`
+  padding: 1em;
+`;
+
+interface RenamingInputProps {
+  renamings: Renaming[];
+  setRenamings: (renamings: Renaming[]) => void;
+}
+
+export function RenamingInput(props: RenamingInputProps) {
+  const { renamings, setRenamings } = props;
+  return (
+    <Details>
+      <summary>Predicate renaming</summary>
+      <p>
+        The following regexp replacements are applied to every predicate name on
+        both sides. Predicates whose names clash after renaming are grouped
+        together. Can be used to correlate predicates that were renamed between
+        the two runs.
+        <br />
+        Can also be used to group related predicates, for example, renaming{" "}
+        <code>.*ssa.*</code> to <code>SSA</code> will group all SSA-related
+        predicates together.
+      </p>
+      {renamings.map((renaming, index) => (
+        <Row key={index}>
+          <Input
+            value={renaming.pattern}
+            placeholder="Pattern"
+            onInput={(e: ChangeEvent<HTMLInputElement>) => {
+              const newRenamings = [...renamings];
+              newRenamings[index] = new Renaming(
+                e.target.value,
+                renaming.replacement,
+              );
+              setRenamings(newRenamings);
+            }}
+          >
+            <Codicon name="search" slot="start" />
+          </Input>
+          <Input
+            value={renaming.replacement}
+            placeholder="Replacement"
+            onInput={(e: ChangeEvent<HTMLInputElement>) => {
+              const newRenamings = [...renamings];
+              newRenamings[index] = new Renaming(
+                renaming.pattern,
+                e.target.value,
+              );
+              setRenamings(newRenamings);
+            }}
+          ></Input>
+          <VSCodeButton
+            onClick={() =>
+              setRenamings(renamings.filter((_, i) => i !== index))
+            }
+          >
+            <Codicon name="trash" />
+          </VSCodeButton>
+          <br />
+        </Row>
+      ))}
+      <VSCodeButton
+        onClick={() => setRenamings([...renamings, new Renaming("", "")])}
+      >
+        Add renaming rule
+      </VSCodeButton>
+    </Details>
+  );
+}

extensions/ql-vscode/src/view/compare-performance/index.tsx

@@ -0,0 +1,8 @@
+import type { WebviewDefinition } from "../webview-definition";
+import { ComparePerformance } from "./ComparePerformance";
+
+const definition: WebviewDefinition = {
+  component: <ComparePerformance />,
+};
+
+export default definition;

extensions/ql-vscode/src/view/webview.tsx

@@ -6,6 +6,7 @@ import { registerUnhandledErrorListener } from "./common/errors";
 import type { WebviewDefinition } from "./webview-definition";
 
 import compareView from "./compare";
+import comparePerformance from "./compare-performance";
 import dataFlowPathsView from "./data-flow-paths";
 import methodModelingView from "./method-modeling";
 import modelEditorView from "./model-editor";
@@ -18,6 +19,7 @@ import "@vscode/codicons/dist/codicon.css";
 
 const views: Record<string, WebviewDefinition> = {
   compare: compareView,
+  "compare-performance": comparePerformance,
   "data-flow-paths": dataFlowPathsView,
   "method-modeling": methodModelingView,
   "model-editor": modelEditorView,

extensions/ql-vscode/supported_cli_versions.json

@@ -1,9 +1,6 @@
 [
-  "v2.20.0",
+  "v2.20.2",
   "v2.19.4",
-  "v2.19.3",
   "v2.18.4",
-  "v2.17.6",
-  "v2.16.6",
   "nightly"
 ]

extensions/ql-vscode/test/e2e/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM codercom/code-server:4.96.1
+FROM codercom/code-server:4.96.2
 
 USER root
 

extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts

@@ -3,6 +3,7 @@ import { Uri } from "vscode";
 import { remove } from "fs-extra";
 import { join } from "path";
 
+import { isIOError } from "../../../../src/common/files";
 import { QLTestDiscovery } from "../../../../src/query-testing/qltest-discovery";
 import type { DirectoryResult } from "tmp-promise";
 import { dir } from "tmp-promise";
@@ -49,7 +50,15 @@ describe("qltest-discovery", () => {
     });
 
     afterEach(async () => {
-      await directory.cleanup();
+      try {
+        await directory.cleanup();
+      } catch (e) {
+        if (isIOError(e) && e.code === "ENOENT") {
+          // This is fine, the directory was already removed
+        } else {
+          throw e;
+        }
+      }
     });
 
     it("should run discovery", async () => {

extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/history-tree-data-provider.test.ts

@@ -38,6 +38,7 @@ describe("HistoryTreeDataProvider", () => {
   let app: App;
   let configListener: QueryHistoryConfigListener;
   const doCompareCallback = jest.fn();
+  const doComparePerformanceCallback = jest.fn();
 
   let queryHistoryManager: QueryHistoryManager;
 
@@ -506,6 +507,7 @@ describe("HistoryTreeDataProvider", () => {
       }),
       languageContext,
       doCompareCallback,
+      doComparePerformanceCallback,
     );
     (qhm.treeDataProvider as any).history = [...allHistory];
     await workspace.saveAll();

extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/query-history-manager.test.ts

@@ -40,6 +40,7 @@ describe("QueryHistoryManager", () => {
     typeof variantAnalysisManagerStub.cancelVariantAnalysis
   >;
   const doCompareCallback = jest.fn();
+  const doComparePerformanceCallback = jest.fn();
 
   let executeCommand: jest.MockedFn<
     (commandName: string, ...args: any[]) => Promise<any>
@@ -939,6 +940,7 @@ describe("QueryHistoryManager", () => {
       }),
       new LanguageContextStore(mockApp),
       doCompareCallback,
+      doComparePerformanceCallback,
     );
     (qhm.treeDataProvider as any).history = [...allHistory];
     await workspace.saveAll();

extensions/ql-vscode/test/vscode-tests/no-workspace/query-history/variant-analysis-history.test.ts

@@ -105,6 +105,7 @@ describe("Variant Analyses and QueryHistoryManager", () => {
       }),
       new LanguageContextStore(app),
       asyncNoop,
+      asyncNoop,
     );
     disposables.push(qhm);
 

extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts

@@ -291,7 +291,7 @@ describe("query-results", () => {
         });
 
         const finished = new Promise((res, rej) => {
-          validSarifStream.addListener("close", res);
+          validSarifStream.addListener("close", () => res(undefined));
           validSarifStream.addListener("error", rej);
         });
 
@@ -357,7 +357,7 @@ describe("query-results", () => {
         });
 
         const finished = new Promise((res, rej) => {
-          invalidSarifStream.addListener("close", res);
+          invalidSarifStream.addListener("close", () => res(undefined));
           invalidSarifStream.addListener("error", rej);
         });