From e9552df395c8dc344ede707ec76c27356dafff04 Mon Sep 17 00:00:00 2001
From: Shati Patel <42641846+shati-patel@users.noreply.github.com>
Date: Mon, 15 May 2023 12:10:51 +0100
Subject: [PATCH 001/119] Create a basic tree view for the queries panel
 (#2418)

---
 extensions/ql-vscode/src/extension.ts         |  3 ++
 .../src/queries-panel/queries-module.ts       | 33 ++++++++++++
 .../src/queries-panel/queries-panel.ts        | 21 ++++++++
 .../queries-panel/query-tree-data-provider.ts | 54 +++++++++++++++++++
 .../src/queries-panel/query-tree-view-item.ts | 11 ++++
 5 files changed, 122 insertions(+)
 create mode 100644 extensions/ql-vscode/src/queries-panel/queries-module.ts
 create mode 100644 extensions/ql-vscode/src/queries-panel/queries-panel.ts
 create mode 100644 extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
 create mode 100644 extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts

diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts
index c6d4597e5..aad6940f5 100644
--- a/extensions/ql-vscode/src/extension.ts
+++ b/extensions/ql-vscode/src/extension.ts
@@ -125,6 +125,7 @@ import { TestManager } from "./query-testing/test-manager";
 import { TestRunner } from "./query-testing/test-runner";
 import { TestManagerBase } from "./query-testing/test-manager-base";
 import { NewQueryRunner, QueryRunner, QueryServerClient } from "./query-server";
+import { QueriesModule } from "./queries-panel/queries-module";
 
 /**
  * extension.ts
@@ -732,6 +733,8 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(databaseUI);
 
+  QueriesModule.initialize(app);
+
   void extLogger.log("Initializing evaluator log viewer.");
   const evalLogViewer = new EvalLogViewer();
   ctx.subscriptions.push(evalLogViewer);
diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts
new file mode 100644
index 000000000..8946eef55
--- /dev/null
+++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts
@@ -0,0 +1,33 @@
+import { extLogger } from "../common";
+import { App, AppMode } from "../common/app";
+import { isCanary, showQueriesPanel } from "../config";
+import { DisposableObject } from "../pure/disposable-object";
+import { QueriesPanel } from "./queries-panel";
+
+export class QueriesModule extends DisposableObject {
+  private queriesPanel: QueriesPanel | undefined;
+
+  private constructor(readonly app: App) {
+    super();
+  }
+
+  private initialize(app: App): void {
+    if (app.mode === AppMode.Production || !isCanary() || !showQueriesPanel()) {
+      // Currently, we only want to expose the new panel when we are in development and canary mode
+      // and the developer has enabled the "Show queries panel" flag.
+      return;
+    }
+    void extLogger.log("Initializing queries panel.");
+
+    this.queriesPanel = new QueriesPanel();
+    this.push(this.queriesPanel);
+  }
+
+  public static initialize(app: App): QueriesModule {
+    const queriesModule = new QueriesModule(app);
+    app.subscriptions.push(queriesModule);
+
+    queriesModule.initialize(app);
+    return queriesModule;
+  }
+}
diff --git a/extensions/ql-vscode/src/queries-panel/queries-panel.ts b/extensions/ql-vscode/src/queries-panel/queries-panel.ts
new file mode 100644
index 000000000..04f30b30d
--- /dev/null
+++ b/extensions/ql-vscode/src/queries-panel/queries-panel.ts
@@ -0,0 +1,21 @@
+import * as vscode from "vscode";
+import { DisposableObject } from "../pure/disposable-object";
+import { QueryTreeDataProvider } from "./query-tree-data-provider";
+import { QueryTreeViewItem } from "./query-tree-view-item";
+
+export class QueriesPanel extends DisposableObject {
+  private readonly dataProvider: QueryTreeDataProvider;
+  private readonly treeView: vscode.TreeView<QueryTreeViewItem>;
+
+  public constructor() {
+    super();
+
+    this.dataProvider = new QueryTreeDataProvider();
+
+    this.treeView = vscode.window.createTreeView("codeQLQueries", {
+      treeDataProvider: this.dataProvider,
+    });
+
+    this.push(this.treeView);
+  }
+}
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
new file mode 100644
index 000000000..690d8997d
--- /dev/null
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -0,0 +1,54 @@
+import * as vscode from "vscode";
+import { QueryTreeViewItem } from "./query-tree-view-item";
+import { DisposableObject } from "../pure/disposable-object";
+
+export class QueryTreeDataProvider
+  extends DisposableObject
+  implements vscode.TreeDataProvider<QueryTreeViewItem>
+{
+  private queryTreeItems: QueryTreeViewItem[];
+
+  public constructor() {
+    super();
+
+    this.queryTreeItems = this.createTree();
+  }
+
+  private createTree(): QueryTreeViewItem[] {
+    // Temporary mock data, just to populate the tree view.
+    return [
+      {
+        label: "name1",
+        tooltip: "path1",
+        children: [],
+      },
+    ];
+  }
+
+  /**
+   * Returns the UI presentation of the element that gets displayed in the view.
+   * @param item The item to represent.
+   * @returns The UI presentation of the item.
+   */
+  public getTreeItem(
+    item: QueryTreeViewItem,
+  ): vscode.TreeItem | Thenable<vscode.TreeItem> {
+    return item;
+  }
+
+  /**
+   * Called when expanding an item (including the root item).
+   * @param item The item to expand.
+   * @returns The children of the item.
+   */
+  public getChildren(
+    item?: QueryTreeViewItem,
+  ): vscode.ProviderResult<QueryTreeViewItem[]> {
+    if (!item) {
+      // We're at the root.
+      return Promise.resolve(this.queryTreeItems);
+    } else {
+      return Promise.resolve(item.children);
+    }
+  }
+}
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
new file mode 100644
index 000000000..59b93b204
--- /dev/null
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
@@ -0,0 +1,11 @@
+import * as vscode from "vscode";
+
+export class QueryTreeViewItem extends vscode.TreeItem {
+  constructor(
+    public readonly label: string,
+    public readonly tooltip: string | undefined,
+    public readonly children: QueryTreeViewItem[],
+  ) {
+    super(label);
+  }
+}

From 927a0f0691bfdf2b6440e604062d4c261664e7fb Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Mon, 15 May 2023 14:50:13 +0200
Subject: [PATCH 002/119] Track currently monitoring variant analyses

---
 .../variant-analysis-monitor.ts               | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
index c3b48a9db..6254b738a 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
@@ -27,6 +27,8 @@ export class VariantAnalysisMonitor extends DisposableObject {
   );
   readonly onVariantAnalysisChange = this._onVariantAnalysisChange.event;
 
+  private readonly monitoringVariantAnalyses = new Set<number>();
+
   constructor(
     private readonly app: App,
     private readonly shouldCancelMonitor: (
@@ -38,6 +40,24 @@ export class VariantAnalysisMonitor extends DisposableObject {
 
   public async monitorVariantAnalysis(
     variantAnalysis: VariantAnalysis,
+  ): Promise<void> {
+    if (this.monitoringVariantAnalyses.has(variantAnalysis.id)) {
+      void extLogger.log(
+        `Already monitoring variant analysis ${variantAnalysis.id}`,
+      );
+      return;
+    }
+
+    this.monitoringVariantAnalyses.add(variantAnalysis.id);
+    try {
+      await this._monitorVariantAnalysis(variantAnalysis);
+    } finally {
+      this.monitoringVariantAnalyses.delete(variantAnalysis.id);
+    }
+  }
+
+  private async _monitorVariantAnalysis(
+    variantAnalysis: VariantAnalysis,
   ): Promise<void> {
     let attemptCount = 0;
     const scannedReposDownloaded: number[] = [];

From 9e74ae0bbfb080feabbbd2a5e585126b78964e48 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Mon, 15 May 2023 14:55:59 +0200
Subject: [PATCH 003/119] Cancel monitoring variant analysis on 404

---
 .../variant-analysis-monitor.ts               | 26 +++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
index 6254b738a..00cdd67d5 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
@@ -1,5 +1,6 @@
 import { env, EventEmitter } from "vscode";
 import { getVariantAnalysis } from "./gh-api/gh-api-client";
+import { RequestError } from "@octokit/request-error";
 
 import {
   isFinalVariantAnalysisStatus,
@@ -59,6 +60,12 @@ export class VariantAnalysisMonitor extends DisposableObject {
   private async _monitorVariantAnalysis(
     variantAnalysis: VariantAnalysis,
   ): Promise<void> {
+    const variantAnalysisLabel = `${variantAnalysis.query.name} (${
+      variantAnalysis.query.language
+    }) [${new Date(variantAnalysis.executionStartTime).toLocaleString(
+      env.language,
+    )}]`;
+
     let attemptCount = 0;
     const scannedReposDownloaded: number[] = [];
 
@@ -81,11 +88,7 @@ export class VariantAnalysisMonitor extends DisposableObject {
       } catch (e) {
         const errorMessage = getErrorMessage(e);
 
-        const message = `Error while monitoring variant analysis ${
-          variantAnalysis.query.name
-        } (${variantAnalysis.query.language}) [${new Date(
-          variantAnalysis.executionStartTime,
-        ).toLocaleString(env.language)}]: ${errorMessage}`;
+        const message = `Error while monitoring variant analysis ${variantAnalysisLabel}: ${errorMessage}`;
 
         // If we have already shown this error to the user, don't show it again.
         if (lastErrorShown === errorMessage) {
@@ -95,6 +98,19 @@ export class VariantAnalysisMonitor extends DisposableObject {
           lastErrorShown = errorMessage;
         }
 
+        if (e instanceof RequestError && e.status === 404) {
+          // We want to show the error message to the user, but we don't want to
+          // keep polling for the variant analysis if it no longer exists.
+          // Therefore, this block is down here rather than at the top of the
+          // catch block.
+          void extLogger.log(
+            `Variant analysis ${variantAnalysisLabel} no longer exists or is no longer accessible, stopping monitoring.`,
+          );
+          // Cancel monitoring on 404, as this probably means the user does not have access to it anymore
+          // e.g. lost access to repo, or repo was deleted
+          return;
+        }
+
         continue;
       }
 

From 5b9ed39f4f56ea731d54dfd31903d1104138291f Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Mon, 15 May 2023 15:06:34 +0200
Subject: [PATCH 004/119] Restart variant analysis monitor when
 reauthenticating

---
 extensions/ql-vscode/src/common/commands.ts   |  3 ++
 .../src/common/vscode/authentication.ts       |  2 +-
 .../variant-analysis-manager.ts               | 41 +++++++++++++++++++
 .../variant-analysis-monitor.ts               |  4 ++
 4 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/common/commands.ts b/extensions/ql-vscode/src/common/commands.ts
index bbd8c7ece..1b0a677a2 100644
--- a/extensions/ql-vscode/src/common/commands.ts
+++ b/extensions/ql-vscode/src/common/commands.ts
@@ -251,6 +251,9 @@ export type VariantAnalysisCommands = {
   "codeQL.monitorRehydratedVariantAnalysis": (
     variantAnalysis: VariantAnalysis,
   ) => Promise<void>;
+  "codeQL.monitorReauthenticatedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
   "codeQL.openVariantAnalysisLogs": (
     variantAnalysisId: number,
   ) => Promise<void>;
diff --git a/extensions/ql-vscode/src/common/vscode/authentication.ts b/extensions/ql-vscode/src/common/vscode/authentication.ts
index 625a31d36..1a7190ec4 100644
--- a/extensions/ql-vscode/src/common/vscode/authentication.ts
+++ b/extensions/ql-vscode/src/common/vscode/authentication.ts
@@ -3,7 +3,7 @@ import * as Octokit from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
 import { Credentials } from "../authentication";
 
-const GITHUB_AUTH_PROVIDER_ID = "github";
+export const GITHUB_AUTH_PROVIDER_ID = "github";
 
 // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist,
 // and 'read:packages' for reading private CodeQL packages.
diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
index f639d5b7d..35e197b9b 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
@@ -5,6 +5,8 @@ import {
   getVariantAnalysisRepo,
 } from "./gh-api/gh-api-client";
 import {
+  authentication,
+  AuthenticationSessionsChangeEvent,
   CancellationToken,
   env,
   EventEmitter,
@@ -72,6 +74,7 @@ import {
   REPO_STATES_FILENAME,
   writeRepoStates,
 } from "./repo-states-store";
+import { GITHUB_AUTH_PROVIDER_ID } from "../common/vscode/authentication";
 
 export class VariantAnalysisManager
   extends DisposableObject
@@ -131,6 +134,10 @@ export class VariantAnalysisManager
     this.variantAnalysisResultsManager.onResultLoaded(
       this.onRepoResultLoaded.bind(this),
     );
+
+    this.push(
+      authentication.onDidChangeSessions(this.onDidChangeSessions.bind(this)),
+    );
   }
 
   getCommands(): VariantAnalysisCommands {
@@ -144,6 +151,8 @@ export class VariantAnalysisManager
         this.monitorVariantAnalysis.bind(this),
       "codeQL.monitorRehydratedVariantAnalysis":
         this.monitorVariantAnalysis.bind(this),
+      "codeQL.monitorReauthenticatedVariantAnalysis":
+        this.monitorVariantAnalysis.bind(this),
       "codeQL.openVariantAnalysisLogs": this.openVariantAnalysisLogs.bind(this),
       "codeQL.openVariantAnalysisView": this.showView.bind(this),
       "codeQL.runVariantAnalysis":
@@ -504,6 +513,38 @@ export class VariantAnalysisManager
     repoStates[repoState.repositoryId] = repoState;
   }
 
+  private async onDidChangeSessions(
+    event: AuthenticationSessionsChangeEvent,
+  ): Promise<void> {
+    if (event.provider.id !== GITHUB_AUTH_PROVIDER_ID) {
+      return;
+    }
+
+    for (const variantAnalysis of this.variantAnalyses.values()) {
+      if (
+        this.variantAnalysisMonitor.isMonitoringVariantAnalysis(
+          variantAnalysis.id,
+        )
+      ) {
+        continue;
+      }
+
+      if (
+        await isVariantAnalysisComplete(
+          variantAnalysis,
+          this.makeResultDownloadChecker(variantAnalysis),
+        )
+      ) {
+        continue;
+      }
+
+      void this.app.commands.execute(
+        "codeQL.monitorReauthenticatedVariantAnalysis",
+        variantAnalysis,
+      );
+    }
+  }
+
   public async monitorVariantAnalysis(
     variantAnalysis: VariantAnalysis,
   ): Promise<void> {
diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
index 00cdd67d5..32ae083d4 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts
@@ -39,6 +39,10 @@ export class VariantAnalysisMonitor extends DisposableObject {
     super();
   }
 
+  public isMonitoringVariantAnalysis(variantAnalysisId: number): boolean {
+    return this.monitoringVariantAnalyses.has(variantAnalysisId);
+  }
+
   public async monitorVariantAnalysis(
     variantAnalysis: VariantAnalysis,
   ): Promise<void> {

From 2d2a1fb2d135ee47249ac938863c75b068972c42 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Mon, 15 May 2023 15:41:15 +0200
Subject: [PATCH 005/119] Add tests for variant analysis monitor

---
 .../variant-analysis-monitor.test.ts          | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts
index b42e059df..9ffb9ee3a 100644
--- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-monitor.test.ts
@@ -1,4 +1,5 @@
 import * as ghApiClient from "../../../../src/variant-analysis/gh-api/gh-api-client";
+import { RequestError } from "@octokit/request-error";
 import { VariantAnalysisMonitor } from "../../../../src/variant-analysis/variant-analysis-monitor";
 import {
   VariantAnalysis as VariantAnalysisApiResponse,
@@ -297,6 +298,55 @@ describe("Variant Analysis Monitor", () => {
         expect(mockEecuteCommand).not.toBeCalled();
       });
     });
+
+    describe("when a 404 is returned", () => {
+      let showAndLogWarningMessageSpy: jest.SpiedFunction<
+        typeof helpers.showAndLogWarningMessage
+      >;
+
+      beforeEach(async () => {
+        showAndLogWarningMessageSpy = jest
+          .spyOn(helpers, "showAndLogWarningMessage")
+          .mockResolvedValue(undefined);
+
+        const scannedRepos = createMockScannedRepos([
+          "pending",
+          "in_progress",
+          "in_progress",
+          "in_progress",
+          "pending",
+          "pending",
+        ]);
+        mockApiResponse = createMockApiResponse("in_progress", scannedRepos);
+        mockGetVariantAnalysis.mockResolvedValueOnce(mockApiResponse);
+
+        mockGetVariantAnalysis.mockRejectedValueOnce(
+          new RequestError("Not Found", 404, {
+            request: {
+              method: "GET",
+              url: "",
+              headers: {},
+            },
+            response: {
+              status: 404,
+              headers: {},
+              url: "",
+              data: {},
+            },
+          }),
+        );
+      });
+
+      it("should stop requesting the variant analysis", async () => {
+        await variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis);
+
+        expect(mockGetVariantAnalysis).toHaveBeenCalledTimes(2);
+        expect(showAndLogWarningMessageSpy).toHaveBeenCalledTimes(1);
+        expect(showAndLogWarningMessageSpy).toHaveBeenCalledWith(
+          expect.stringMatching(/not found/i),
+        );
+      });
+    });
   });
 
   function limitNumberOfAttemptsToMonitor() {

From 4c33c06d55fcb47a2d22e67058572829dfc8b90f Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Mon, 15 May 2023 15:51:48 -0700
Subject: [PATCH 006/119] Allow compare view to work with quick-eval

The compare view typically works by matching the results sets of
queries. It only allows the results sets of queries with identical
names to be compared.

Manually run queries use `#select` as the default result set. However,
quick eval queries use a different, generated, name. Therefore, these
two kinds of queries cannot be compared.

This commit changes the heuristics so that if there are no identical;y
named results sets, the first result set of each query that is prefixed
with `#` is used (this is the default result set).

It also makes a slightly nicer error message if there are no comparable
results sets.
---
 .../ql-vscode/src/compare/compare-view.ts     | 25 ++++++++++++++++---
 .../ql-vscode/src/view/compare/Compare.tsx    |  4 +--
 .../src/view/compare/CompareSelector.tsx      |  6 ++++-
 3 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/extensions/ql-vscode/src/compare/compare-view.ts b/extensions/ql-vscode/src/compare/compare-view.ts
index df684d19e..c9c033f34 100644
--- a/extensions/ql-vscode/src/compare/compare-view.ts
+++ b/extensions/ql-vscode/src/compare/compare-view.ts
@@ -175,21 +175,40 @@ export class CompareView extends AbstractWebview<
     const commonResultSetNames = fromSchemaNames.filter((name) =>
       toSchemaNames.includes(name),
     );
+
+    // Fall back on the default result set names if there are no common ones.
+    const defaultFromResultSetName = fromSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+    const defaultToResultSetName = toSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+
+    if (
+      commonResultSetNames.length === 0 &&
+      !(defaultFromResultSetName || defaultToResultSetName)
+    ) {
+      throw new Error(
+        "No common result sets found between the two queries. Please check that the queries are compatible.",
+      );
+    }
+
     const currentResultSetName =
       selectedResultSetName || commonResultSetNames[0];
     const fromResultSet = await this.getResultSet(
       fromSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultFromResultSetName!,
       from.completedQuery.query.resultsPaths.resultsPath,
     );
     const toResultSet = await this.getResultSet(
       toSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultToResultSetName!,
       to.completedQuery.query.resultsPaths.resultsPath,
     );
     return [
       commonResultSetNames,
-      currentResultSetName,
+      currentResultSetName ||
+        `${defaultFromResultSetName} <-> ${defaultToResultSetName}`,
       fromResultSet,
       toResultSet,
     ];
diff --git a/extensions/ql-vscode/src/view/compare/Compare.tsx b/extensions/ql-vscode/src/view/compare/Compare.tsx
index 00aef5449..a98626e42 100644
--- a/extensions/ql-vscode/src/view/compare/Compare.tsx
+++ b/extensions/ql-vscode/src/view/compare/Compare.tsx
@@ -59,9 +59,7 @@ export function Compare(_: Record<string, never>): JSX.Element {
     return (
       <>
         <div className="vscode-codeql__compare-header">
-          <div className="vscode-codeql__compare-header-item">
-            Table to compare:
-          </div>
+          <div className="vscode-codeql__compare-header-item">Comparing:</div>
           <CompareSelector
             availableResultSets={comparison.commonResultSetNames}
             currentResultSetName={comparison.currentResultSetName}
diff --git a/extensions/ql-vscode/src/view/compare/CompareSelector.tsx b/extensions/ql-vscode/src/view/compare/CompareSelector.tsx
index 627def9b3..af7866bdc 100644
--- a/extensions/ql-vscode/src/view/compare/CompareSelector.tsx
+++ b/extensions/ql-vscode/src/view/compare/CompareSelector.tsx
@@ -7,7 +7,8 @@ interface Props {
 }
 
 export default function CompareSelector(props: Props) {
-  return (
+  return props.availableResultSets.length ? (
+    // Handle case where there are shared result sets
     <select
       value={props.currentResultSetName}
       onChange={(e) => props.updateResultSet(e.target.value)}
@@ -18,5 +19,8 @@ export default function CompareSelector(props: Props) {
         </option>
       ))}
     </select>
+  ) : (
+    // Handle case where there are no shared result sets
+    <div>{props.currentResultSetName}</div>
   );
 }

From 9ce95a3554b46096eae8993279113c406235c8b8 Mon Sep 17 00:00:00 2001
From: Shati Patel <42641846+shati-patel@users.noreply.github.com>
Date: Tue, 16 May 2023 16:45:59 +0100
Subject: [PATCH 007/119] Queries panel: Expand/collapse child nodes (#2419)

---
 .../queries-panel/query-tree-data-provider.ts    | 16 +++++++++-------
 .../src/queries-panel/query-tree-view-item.ts    |  4 ++++
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
index 690d8997d..0999306bb 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -17,11 +17,13 @@ export class QueryTreeDataProvider
   private createTree(): QueryTreeViewItem[] {
     // Temporary mock data, just to populate the tree view.
     return [
-      {
-        label: "name1",
-        tooltip: "path1",
-        children: [],
-      },
+      new QueryTreeViewItem("name1", "path1", []),
+      new QueryTreeViewItem("name2", "path2", [
+        new QueryTreeViewItem("name3", "path3", []),
+        new QueryTreeViewItem("name4", "path4", [
+          new QueryTreeViewItem("name5", "path5", []),
+        ]),
+      ]),
     ];
   }
 
@@ -46,9 +48,9 @@ export class QueryTreeDataProvider
   ): vscode.ProviderResult<QueryTreeViewItem[]> {
     if (!item) {
       // We're at the root.
-      return Promise.resolve(this.queryTreeItems);
+      return this.queryTreeItems;
     } else {
-      return Promise.resolve(item.children);
+      return item.children;
     }
   }
 }
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
index 59b93b204..c277b5466 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
@@ -1,11 +1,15 @@
 import * as vscode from "vscode";
 
 export class QueryTreeViewItem extends vscode.TreeItem {
+  public collapsibleState: vscode.TreeItemCollapsibleState;
   constructor(
     public readonly label: string,
     public readonly tooltip: string | undefined,
     public readonly children: QueryTreeViewItem[],
   ) {
     super(label);
+    this.collapsibleState = this.children.length
+      ? vscode.TreeItemCollapsibleState.Collapsed
+      : vscode.TreeItemCollapsibleState.None;
   }
 }

From b0371b5075d2d79bc31b645c9b46210c66a186cf Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 16 May 2023 16:55:52 +0100
Subject: [PATCH 008/119] Simplify types on TreeDataProvider methods

---
 .../src/queries-panel/query-tree-data-provider.ts         | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
index 0999306bb..ae9492fee 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -32,9 +32,7 @@ export class QueryTreeDataProvider
    * @param item The item to represent.
    * @returns The UI presentation of the item.
    */
-  public getTreeItem(
-    item: QueryTreeViewItem,
-  ): vscode.TreeItem | Thenable<vscode.TreeItem> {
+  public getTreeItem(item: QueryTreeViewItem): vscode.TreeItem {
     return item;
   }
 
@@ -43,9 +41,7 @@ export class QueryTreeDataProvider
    * @param item The item to expand.
    * @returns The children of the item.
    */
-  public getChildren(
-    item?: QueryTreeViewItem,
-  ): vscode.ProviderResult<QueryTreeViewItem[]> {
+  public getChildren(item?: QueryTreeViewItem): QueryTreeViewItem[] {
     if (!item) {
       // We're at the root.
       return this.queryTreeItems;

From 1ceccc8b4dcf2540a46334c88b422f9a8e1d478c Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 16 May 2023 16:57:02 +0100
Subject: [PATCH 009/119] Avoid double-declaring fields that are inherited from
 TreeItem

---
 .../ql-vscode/src/queries-panel/query-tree-view-item.ts     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
index c277b5466..0efb36692 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
@@ -1,13 +1,13 @@
 import * as vscode from "vscode";
 
 export class QueryTreeViewItem extends vscode.TreeItem {
-  public collapsibleState: vscode.TreeItemCollapsibleState;
   constructor(
-    public readonly label: string,
-    public readonly tooltip: string | undefined,
+    label: string,
+    tooltip: string | undefined,
     public readonly children: QueryTreeViewItem[],
   ) {
     super(label);
+    this.tooltip = tooltip;
     this.collapsibleState = this.children.length
       ? vscode.TreeItemCollapsibleState.Collapsed
       : vscode.TreeItemCollapsibleState.None;

From 91c58f3618bc2e7d76318819c1f835e2c975a91c Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Tue, 16 May 2023 12:32:29 -0700
Subject: [PATCH 010/119] Update changelog

For #2422.
---
 extensions/ql-vscode/CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md
index 767313d68..58e63b143 100644
--- a/extensions/ql-vscode/CHANGELOG.md
+++ b/extensions/ql-vscode/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## [UNRELEASED]
 
 - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
 
 ## 1.8.4 - 3 May 2023
 

From 276e2cfdcf27d1bc99fc838a47ead0ffe7a16800 Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Tue, 16 May 2023 14:48:19 -0700
Subject: [PATCH 011/119] Ensure full stack traces are included in log messages

Pass in the `fullMessage` to `internalShowAndLog` so that stack traces
aren't truncated.
---
 extensions/ql-vscode/src/common/vscode/commands.ts | 2 +-
 extensions/ql-vscode/src/helpers.ts                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/common/vscode/commands.ts b/extensions/ql-vscode/src/common/vscode/commands.ts
index dfd765e17..27f8455e5 100644
--- a/extensions/ql-vscode/src/common/vscode/commands.ts
+++ b/extensions/ql-vscode/src/common/vscode/commands.ts
@@ -49,7 +49,6 @@ export function registerCommandWithErrorHandling(
       const errorMessage = redactableError(error)`${
         getErrorMessage(e) || e
       } (${commandId})`;
-      const errorStack = getErrorStack(e);
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
@@ -61,6 +60,7 @@ export function registerCommandWithErrorHandling(
         }
       } else {
         // Include the full stack in the error log only.
+        const errorStack = getErrorStack(e);
         const fullMessage = errorStack
           ? `${errorMessage.fullMessage}\n${errorStack}`
           : errorMessage.fullMessage;
diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts
index e7e33f206..d4e8a9a8c 100644
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@@ -98,7 +98,7 @@ export async function showAndLogErrorMessage(
   return internalShowAndLog(
     dropLinesExceptInitial(message),
     Window.showErrorMessage,
-    options,
+    { fullMessage: message, ...options },
   );
 }
 

From 55b65e33ad2a997d8c63fd2691138657bcee8027 Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Tue, 16 May 2023 14:51:03 -0700
Subject: [PATCH 012/119] Update changelog

---
 extensions/ql-vscode/CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md
index 767313d68..1d07f268c 100644
--- a/extensions/ql-vscode/CHANGELOG.md
+++ b/extensions/ql-vscode/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## [UNRELEASED]
 
 - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Fix bug to ensure error messages have complete stack trace in message logs. [#2425](https://github.com/github/vscode-codeql/pull/2425)
 
 ## 1.8.4 - 3 May 2023
 

From ec1fda21d031b6ec2babc3915c38ad2629366b66 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 16 May 2023 17:25:01 +0100
Subject: [PATCH 013/119] Open query file on click

---
 .../queries-panel/query-tree-data-provider.ts   | 17 ++++++++++++-----
 .../src/queries-panel/query-tree-view-item.ts   | 16 +++++++++-------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
index ae9492fee..85c446465 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -17,11 +17,18 @@ export class QueryTreeDataProvider
   private createTree(): QueryTreeViewItem[] {
     // Temporary mock data, just to populate the tree view.
     return [
-      new QueryTreeViewItem("name1", "path1", []),
-      new QueryTreeViewItem("name2", "path2", [
-        new QueryTreeViewItem("name3", "path3", []),
-        new QueryTreeViewItem("name4", "path4", [
-          new QueryTreeViewItem("name5", "path5", []),
+      new QueryTreeViewItem("custom-pack", [
+        new QueryTreeViewItem("custom-pack/example.ql", []),
+      ]),
+      new QueryTreeViewItem("ql", [
+        new QueryTreeViewItem("ql/javascript", [
+          new QueryTreeViewItem("ql/javascript/example.ql", []),
+        ]),
+        new QueryTreeViewItem("ql/go", [
+          new QueryTreeViewItem("ql/go/security", [
+            new QueryTreeViewItem("ql/go/security/query1.ql", []),
+            new QueryTreeViewItem("ql/go/security/query2.ql", []),
+          ]),
         ]),
       ]),
     ];
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
index 0efb36692..a6f82030a 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
@@ -1,15 +1,17 @@
 import * as vscode from "vscode";
+import { basename } from "path";
 
 export class QueryTreeViewItem extends vscode.TreeItem {
-  constructor(
-    label: string,
-    tooltip: string | undefined,
-    public readonly children: QueryTreeViewItem[],
-  ) {
-    super(label);
-    this.tooltip = tooltip;
+  constructor(path: string, public readonly children: QueryTreeViewItem[]) {
+    super(basename(path));
+    this.tooltip = path;
     this.collapsibleState = this.children.length
       ? vscode.TreeItemCollapsibleState.Collapsed
       : vscode.TreeItemCollapsibleState.None;
+    this.command = {
+      title: "Open",
+      command: "vscode.open",
+      arguments: [vscode.Uri.file(path)],
+    };
   }
 }

From 1955086cb6db4ba704f88c0efe5aba0e55dbea53 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 17 May 2023 12:11:45 +0000
Subject: [PATCH 014/119] Add new command and add it to db panel

---
 extensions/ql-vscode/package.json                   | 13 +++++++++++++
 extensions/ql-vscode/src/common/commands.ts         |  1 +
 extensions/ql-vscode/src/databases/ui/db-panel.ts   |  8 ++++++++
 .../src/databases/ui/db-tree-view-item-action.ts    | 11 +++++++++--
 4 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json
index 87788fe8d..b6ba2b393 100644
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@@ -516,6 +516,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+        "title": "Import Repos from GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -961,6 +965,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1297,6 +1306,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
diff --git a/extensions/ql-vscode/src/common/commands.ts b/extensions/ql-vscode/src/common/commands.ts
index 1b0a677a2..15d566bfc 100644
--- a/extensions/ql-vscode/src/common/commands.ts
+++ b/extensions/ql-vscode/src/common/commands.ts
@@ -275,6 +275,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {
diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index 16abf9f50..203074222 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -93,6 +93,8 @@ export class DbPanel extends DisposableObject {
         this.renameItem.bind(this),
       "codeQLVariantAnalysisRepositories.removeItemContextMenu":
         this.removeItem.bind(this),
+      "codeQLVariantAnalysisRepositories.importCodeSearch":
+        this.importCodeSearch.bind(this),
     };
   }
 
@@ -323,6 +325,12 @@ export class DbPanel extends DisposableObject {
     await this.dbManager.removeDbItem(treeViewItem.dbItem);
   }
 
+  private async importCodeSearch(treeViewItem: DbTreeViewItem): Promise<void> {
+    if (treeViewItem.dbItem === undefined) {
+      throw new Error("Please select a valid list to add code search results.");
+    }
+  }
+
   private async onDidCollapseElement(
     event: TreeViewExpansionEvent<DbTreeViewItem>,
   ): Promise<void> {
diff --git a/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts b/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts
index 2755934f7..315b12c8b 100644
--- a/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts
@@ -4,7 +4,8 @@ export type DbTreeViewItemAction =
   | "canBeSelected"
   | "canBeRemoved"
   | "canBeRenamed"
-  | "canBeOpenedOnGitHub";
+  | "canBeOpenedOnGitHub"
+  | "canImportCodeSearch";
 
 export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   const actions: DbTreeViewItemAction[] = [];
@@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   if (canBeOpenedOnGitHub(dbItem)) {
     actions.push("canBeOpenedOnGitHub");
   }
-
+  if (canImportCodeSearch(dbItem)) {
+    actions.push("canImportCodeSearch");
+  }
   return actions;
 }
 
@@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean {
   return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind);
 }
 
+function canImportCodeSearch(dbItem: DbItem): boolean {
+  return DbItemKind.RemoteUserDefinedList === dbItem.kind;
+}
+
 export function getGitHubUrl(dbItem: DbItem): string | undefined {
   switch (dbItem.kind) {
     case DbItemKind.RemoteOwner:

From fa39bd1c2ca811b0564c5655e1f1ebf8f04b865d Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 17 May 2023 15:03:28 +0200
Subject: [PATCH 015/119] Update export/copy buttons copy when repositories are
 selected

This changes the text of the export/copy buttons on a variant analysis
when at least one repository is selected. This makes it more clear that
the user is only exporting/copying the results of the selected
repositories.
---
 .../variant-analysis/VariantAnalysisActions.tsx     | 11 +++++++++--
 .../view/variant-analysis/VariantAnalysisHeader.tsx |  3 +++
 .../__tests__/VariantAnalysisActions.spec.tsx       | 13 +++++++++++++
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx
index 4df2a9157..5b403d629 100644
--- a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx
+++ b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx
@@ -14,6 +14,8 @@ export type VariantAnalysisActionsProps = {
   onExportResultsClick: () => void;
   copyRepositoryListDisabled?: boolean;
   exportResultsDisabled?: boolean;
+
+  hasSelectedRepositories?: boolean;
 };
 
 const Container = styled.div`
@@ -35,6 +37,7 @@ export const VariantAnalysisActions = ({
   onExportResultsClick,
   copyRepositoryListDisabled,
   exportResultsDisabled,
+  hasSelectedRepositories,
 }: VariantAnalysisActionsProps) => {
   return (
     <Container>
@@ -45,14 +48,18 @@ export const VariantAnalysisActions = ({
             onClick={onCopyRepositoryListClick}
             disabled={copyRepositoryListDisabled}
           >
-            Copy repository list
+            {hasSelectedRepositories
+              ? "Copy selected repositories as a list"
+              : "Copy repository list"}
           </Button>
           <Button
             appearance="primary"
             onClick={onExportResultsClick}
             disabled={exportResultsDisabled}
           >
-            Export results
+            {hasSelectedRepositories
+              ? "Export selected results"
+              : "Export results"}
           </Button>
         </>
       )}
diff --git a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx
index a579a4268..befb15708 100644
--- a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx
+++ b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx
@@ -131,6 +131,9 @@ export const VariantAnalysisHeader = ({
           stopQueryDisabled={!variantAnalysis.actionsWorkflowRunId}
           exportResultsDisabled={!hasDownloadedRepos}
           copyRepositoryListDisabled={!hasReposWithResults}
+          hasSelectedRepositories={
+            selectedRepositoryIds && selectedRepositoryIds.length > 0
+          }
         />
       </Row>
       <VariantAnalysisStats
diff --git a/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx b/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx
index a655f7d1c..ce5ddd522 100644
--- a/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx
+++ b/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx
@@ -93,4 +93,17 @@ describe(VariantAnalysisActions.name, () => {
 
     expect(container.querySelectorAll("vscode-button").length).toEqual(0);
   });
+
+  it("changes the text on the buttons when repositories are selected", async () => {
+    render({
+      variantAnalysisStatus: VariantAnalysisStatus.Succeeded,
+      showResultActions: true,
+      hasSelectedRepositories: true,
+    });
+
+    expect(screen.getByText("Export selected results")).toBeInTheDocument();
+    expect(
+      screen.getByText("Copy selected repositories as a list"),
+    ).toBeInTheDocument();
+  });
 });

From 0125d10ffb85c0692a1252b86030ad992670080b Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 17 May 2023 15:08:55 +0200
Subject: [PATCH 016/119] Take filtered state into account for buttons text

When the user filters the repositories, the buttons should reflect that
the results are filtered and that the user is not exporting or copying
all the results. If the user has selected repositories, the buttons
should still say that they are exporting selected results.
---
 .../VariantAnalysisActions.tsx                | 44 ++++++++++++++++---
 .../VariantAnalysisHeader.tsx                 |  4 ++
 .../__tests__/VariantAnalysisActions.spec.tsx | 15 +++++++
 3 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx
index 5b403d629..26ce0b3e7 100644
--- a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx
+++ b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx
@@ -16,6 +16,7 @@ export type VariantAnalysisActionsProps = {
   exportResultsDisabled?: boolean;
 
   hasSelectedRepositories?: boolean;
+  hasFilteredRepositories?: boolean;
 };
 
 const Container = styled.div`
@@ -28,6 +29,28 @@ const Button = styled(VSCodeButton)`
   white-space: nowrap;
 `;
 
+const chooseText = ({
+  hasSelectedRepositories,
+  hasFilteredRepositories,
+  normalText,
+  selectedText,
+  filteredText,
+}: {
+  hasSelectedRepositories?: boolean;
+  hasFilteredRepositories?: boolean;
+  normalText: string;
+  selectedText: string;
+  filteredText: string;
+}) => {
+  if (hasSelectedRepositories) {
+    return selectedText;
+  }
+  if (hasFilteredRepositories) {
+    return filteredText;
+  }
+  return normalText;
+};
+
 export const VariantAnalysisActions = ({
   variantAnalysisStatus,
   onStopQueryClick,
@@ -38,6 +61,7 @@ export const VariantAnalysisActions = ({
   copyRepositoryListDisabled,
   exportResultsDisabled,
   hasSelectedRepositories,
+  hasFilteredRepositories,
 }: VariantAnalysisActionsProps) => {
   return (
     <Container>
@@ -48,18 +72,26 @@ export const VariantAnalysisActions = ({
             onClick={onCopyRepositoryListClick}
             disabled={copyRepositoryListDisabled}
           >
-            {hasSelectedRepositories
-              ? "Copy selected repositories as a list"
-              : "Copy repository list"}
+            {chooseText({
+              hasSelectedRepositories,
+              hasFilteredRepositories,
+              normalText: "Copy repository list",
+              selectedText: "Copy selected repositories as a list",
+              filteredText: "Copy filtered repositories as a list",
+            })}
           </Button>
           <Button
             appearance="primary"
             onClick={onExportResultsClick}
             disabled={exportResultsDisabled}
           >
-            {hasSelectedRepositories
-              ? "Export selected results"
-              : "Export results"}
+            {chooseText({
+              hasSelectedRepositories,
+              hasFilteredRepositories,
+              normalText: "Export results",
+              selectedText: "Export selected results",
+              filteredText: "Export filtered results",
+            })}
           </Button>
         </>
       )}
diff --git a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx
index befb15708..1c146f19f 100644
--- a/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx
+++ b/extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisHeader.tsx
@@ -131,6 +131,10 @@ export const VariantAnalysisHeader = ({
           stopQueryDisabled={!variantAnalysis.actionsWorkflowRunId}
           exportResultsDisabled={!hasDownloadedRepos}
           copyRepositoryListDisabled={!hasReposWithResults}
+          hasFilteredRepositories={
+            variantAnalysis.scannedRepos?.length !==
+            filteredRepositories?.length
+          }
           hasSelectedRepositories={
             selectedRepositoryIds && selectedRepositoryIds.length > 0
           }
diff --git a/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx b/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx
index ce5ddd522..39a4d4332 100644
--- a/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx
+++ b/extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx
@@ -99,6 +99,7 @@ describe(VariantAnalysisActions.name, () => {
       variantAnalysisStatus: VariantAnalysisStatus.Succeeded,
       showResultActions: true,
       hasSelectedRepositories: true,
+      hasFilteredRepositories: true,
     });
 
     expect(screen.getByText("Export selected results")).toBeInTheDocument();
@@ -106,4 +107,18 @@ describe(VariantAnalysisActions.name, () => {
       screen.getByText("Copy selected repositories as a list"),
     ).toBeInTheDocument();
   });
+
+  it("changes the text on the buttons when repositories are filtered", async () => {
+    render({
+      variantAnalysisStatus: VariantAnalysisStatus.Succeeded,
+      showResultActions: true,
+      hasSelectedRepositories: false,
+      hasFilteredRepositories: true,
+    });
+
+    expect(screen.getByText("Export filtered results")).toBeInTheDocument();
+    expect(
+      screen.getByText("Copy filtered repositories as a list"),
+    ).toBeInTheDocument();
+  });
 });

From 565a7a53e012e7466f2a7ce8b885ecdc91f0ea16 Mon Sep 17 00:00:00 2001
From: shati-patel <42641846+shati-patel@users.noreply.github.com>
Date: Wed, 17 May 2023 14:43:43 +0100
Subject: [PATCH 017/119] Only implement "open file" when you click a file node
 (not a folder)

---
 .../src/queries-panel/query-tree-view-item.ts        | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
index a6f82030a..f9adc5c00 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
@@ -8,10 +8,12 @@ export class QueryTreeViewItem extends vscode.TreeItem {
     this.collapsibleState = this.children.length
       ? vscode.TreeItemCollapsibleState.Collapsed
       : vscode.TreeItemCollapsibleState.None;
-    this.command = {
-      title: "Open",
-      command: "vscode.open",
-      arguments: [vscode.Uri.file(path)],
-    };
+    if (this.children.length === 0) {
+      this.command = {
+        title: "Open",
+        command: "vscode.open",
+        arguments: [vscode.Uri.file(path)],
+      };
+    }
   }
 }

From ee2d78fbfc8ec4f5f271dbae02ada56e8d0d9b31 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 17 May 2023 16:25:06 +0200
Subject: [PATCH 018/119] Add warning when using unsupported CLI version

This adds a warning when the user is using an unsupported version of the
CodeQL CLI. The warning is shown once per session, and only if the
version is older than the oldest supported version.
---
 extensions/ql-vscode/src/codeql-cli/cli.ts |  4 ++++
 extensions/ql-vscode/src/extension.ts      | 24 +++++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts
index d5ef116d4..969ba374b 100644
--- a/extensions/ql-vscode/src/codeql-cli/cli.ts
+++ b/extensions/ql-vscode/src/codeql-cli/cli.ts
@@ -1737,6 +1737,10 @@ export function shouldDebugCliServer() {
 }
 
 export class CliVersionConstraint {
+  // The oldest version of the CLI that we support. This is used to determine
+  // whether to show a warning about the CLI being too old on startup.
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.7.6");
+
   /**
    * CLI version where building QLX packs for remote queries is supported.
    * (The options were _accepted_ by a few earlier versions, but only from
diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts
index aad6940f5..b3625f50a 100644
--- a/extensions/ql-vscode/src/extension.ts
+++ b/extensions/ql-vscode/src/extension.ts
@@ -24,7 +24,7 @@ import {
   activate as archiveFilesystemProvider_activate,
   zipArchiveScheme,
 } from "./common/vscode/archive-filesystem-provider";
-import { CodeQLCliServer } from "./codeql-cli/cli";
+import { CliVersionConstraint, CodeQLCliServer } from "./codeql-cli/cli";
 import {
   CliConfigListener,
   DistributionConfigListener,
@@ -408,6 +408,28 @@ export async function activate(
     codeQlExtension.cliServer.addVersionChangedListener((ver) => {
       telemetryListener.cliVersion = ver;
     });
+
+    let unsupportedWarningShown = false;
+    codeQlExtension.cliServer.addVersionChangedListener((ver) => {
+      if (!ver) {
+        return;
+      }
+
+      if (unsupportedWarningShown) {
+        return;
+      }
+
+      if (CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION.compare(ver) < 0) {
+        return;
+      }
+
+      void showAndLogWarningMessage(
+        `You are using an unsupported version of the CodeQL CLI (${ver}). ` +
+          `The minimum supported version is ${CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION}. ` +
+          `Please upgrade to the latest version of the CodeQL CLI.`,
+      );
+      unsupportedWarningShown = true;
+    });
   }
 
   return codeQlExtension;

From 5a7f5c2ff1eca43076ae31fa47edb90ddf193235 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 17 May 2023 16:46:35 +0200
Subject: [PATCH 019/119] Update CHANGELOG

---
 extensions/ql-vscode/CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md
index 767313d68..cf06f04cb 100644
--- a/extensions/ql-vscode/CHANGELOG.md
+++ b/extensions/ql-vscode/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## [UNRELEASED]
 
 - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
 
 ## 1.8.4 - 3 May 2023
 

From 3e9b4273c34c04bb9642e2006775690b63031b03 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 17 May 2023 16:55:11 +0200
Subject: [PATCH 020/119] Update CHANGELOG

---
 extensions/ql-vscode/CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md
index 767313d68..aba7da7b4 100644
--- a/extensions/ql-vscode/CHANGELOG.md
+++ b/extensions/ql-vscode/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## [UNRELEASED]
 
 - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
 
 ## 1.8.4 - 3 May 2023
 

From a5af2f2e4bf44d89cae9b4bfe0da4a14b59d5c1e Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 19 May 2023 12:16:48 +0100
Subject: [PATCH 021/119] Pull out node classes to shared location

---
 .../ql-vscode/src/common/file-tree-nodes.ts   | 103 +++++++++++++++
 .../src/query-testing/qltest-discovery.ts     | 119 ++----------------
 .../src/query-testing/test-adapter.ts         |  24 ++--
 .../src/query-testing/test-manager.ts         |  20 +--
 4 files changed, 135 insertions(+), 131 deletions(-)
 create mode 100644 extensions/ql-vscode/src/common/file-tree-nodes.ts

diff --git a/extensions/ql-vscode/src/common/file-tree-nodes.ts b/extensions/ql-vscode/src/common/file-tree-nodes.ts
new file mode 100644
index 000000000..32144c885
--- /dev/null
+++ b/extensions/ql-vscode/src/common/file-tree-nodes.ts
@@ -0,0 +1,103 @@
+import { basename, dirname, join } from "path";
+import { env } from "vscode";
+
+/**
+ * A node in the tree of files. This will be either a `FileTreeDirectory` or a `FileTreeLeaf`.
+ */
+export abstract class FileTreeNode {
+  constructor(private _path: string, private _name: string) {}
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public abstract get children(): readonly FileTreeNode[];
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more files or other directories.
+ */
+export class FileTreeDirectory extends FileTreeNode {
+  constructor(
+    _path: string,
+    _name: string,
+    private _children: FileTreeNode[] = [],
+  ) {
+    super(_path, _name);
+  }
+
+  public get children(): readonly FileTreeNode[] {
+    return this._children;
+  }
+
+  public addChild(child: FileTreeNode): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): FileTreeDirectory {
+    const dirName = dirname(relativePath);
+    if (dirName === ".") {
+      return this.createChildDirectory(relativePath);
+    } else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    // remove empty directories
+    this._children.filter(
+      (child) => child instanceof FileTreeLeaf || child.children.length > 0,
+    );
+    this._children.sort((a, b) => a.name.localeCompare(b.name, env.language));
+    this._children.forEach((child, i) => {
+      child.finish();
+      if (
+        child.children?.length === 1 &&
+        child.children[0] instanceof FileTreeDirectory
+      ) {
+        // collapse children
+        const replacement = new FileTreeDirectory(
+          child.children[0].path,
+          `${child.name} / ${child.children[0].name}`,
+          Array.from(child.children[0].children),
+        );
+        this._children[i] = replacement;
+      }
+    });
+  }
+
+  private createChildDirectory(name: string): FileTreeDirectory {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return existingChild as FileTreeDirectory;
+    } else {
+      const newChild = new FileTreeDirectory(join(this.path, name), name);
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single file.
+ */
+export class FileTreeLeaf extends FileTreeNode {
+  constructor(_path: string, _name: string) {
+    super(_path, _name);
+  }
+
+  public get children(): readonly FileTreeNode[] {
+    return [];
+  }
+
+  public finish(): void {
+    /**/
+  }
+}
diff --git a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
index 891d2c965..b7d333ff6 100644
--- a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
+++ b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
@@ -1,4 +1,4 @@
-import { dirname, basename, join, normalize, relative, extname } from "path";
+import { dirname, basename, normalize, relative, extname } from "path";
 import { Discovery } from "../common/discovery";
 import {
   EventEmitter,
@@ -6,112 +6,11 @@ import {
   Uri,
   RelativePattern,
   WorkspaceFolder,
-  env,
 } from "vscode";
 import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { pathExists } from "fs-extra";
-
-/**
- * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`.
- */
-export abstract class QLTestNode {
-  constructor(private _path: string, private _name: string) {}
-
-  public get path(): string {
-    return this._path;
-  }
-
-  public get name(): string {
-    return this._name;
-  }
-
-  public abstract get children(): readonly QLTestNode[];
-
-  public abstract finish(): void;
-}
-
-/**
- * A directory containing one or more QL tests or other test directories.
- */
-export class QLTestDirectory extends QLTestNode {
-  constructor(
-    _path: string,
-    _name: string,
-    private _children: QLTestNode[] = [],
-  ) {
-    super(_path, _name);
-  }
-
-  public get children(): readonly QLTestNode[] {
-    return this._children;
-  }
-
-  public addChild(child: QLTestNode): void {
-    this._children.push(child);
-  }
-
-  public createDirectory(relativePath: string): QLTestDirectory {
-    const dirName = dirname(relativePath);
-    if (dirName === ".") {
-      return this.createChildDirectory(relativePath);
-    } else {
-      const parent = this.createDirectory(dirName);
-      return parent.createDirectory(basename(relativePath));
-    }
-  }
-
-  public finish(): void {
-    // remove empty directories
-    this._children.filter(
-      (child) => child instanceof QLTestFile || child.children.length > 0,
-    );
-    this._children.sort((a, b) => a.name.localeCompare(b.name, env.language));
-    this._children.forEach((child, i) => {
-      child.finish();
-      if (
-        child.children?.length === 1 &&
-        child.children[0] instanceof QLTestDirectory
-      ) {
-        // collapse children
-        const replacement = new QLTestDirectory(
-          child.children[0].path,
-          `${child.name} / ${child.children[0].name}`,
-          Array.from(child.children[0].children),
-        );
-        this._children[i] = replacement;
-      }
-    });
-  }
-
-  private createChildDirectory(name: string): QLTestDirectory {
-    const existingChild = this._children.find((child) => child.name === name);
-    if (existingChild !== undefined) {
-      return existingChild as QLTestDirectory;
-    } else {
-      const newChild = new QLTestDirectory(join(this.path, name), name);
-      this.addChild(newChild);
-      return newChild;
-    }
-  }
-}
-
-/**
- * A single QL test. This will be either a `.ql` file or a `.qlref` file.
- */
-export class QLTestFile extends QLTestNode {
-  constructor(_path: string, _name: string) {
-    super(_path, _name);
-  }
-
-  public get children(): readonly QLTestNode[] {
-    return [];
-  }
-
-  public finish(): void {
-    /**/
-  }
-}
+import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
 
 /**
  * The results of discovering QL tests.
@@ -120,7 +19,7 @@ interface QLTestDiscoveryResults {
   /**
    * A directory that contains one or more QL Tests, or other QLTestDirectories.
    */
-  testDirectory: QLTestDirectory | undefined;
+  testDirectory: FileTreeDirectory | undefined;
 
   /**
    * The file system path to a directory to watch. If any ql or qlref file changes in
@@ -137,7 +36,7 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
   private readonly watcher: MultiFileSystemWatcher = this.push(
     new MultiFileSystemWatcher(),
   );
-  private _testDirectory: QLTestDirectory | undefined;
+  private _testDirectory: FileTreeDirectory | undefined;
 
   constructor(
     private readonly workspaceFolder: WorkspaceFolder,
@@ -159,7 +58,7 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
    * The root directory. There is at least one test in this directory, or
    * in a subdirectory of this.
    */
-  public get testDirectory(): QLTestDirectory | undefined {
+  public get testDirectory(): FileTreeDirectory | undefined {
     return this._testDirectory;
   }
 
@@ -194,10 +93,10 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
    * @returns A `QLTestDirectory` object describing the contents of the directory, or `undefined` if
    *   no tests were found.
    */
-  private async discoverTests(): Promise<QLTestDirectory> {
+  private async discoverTests(): Promise<FileTreeDirectory> {
     const fullPath = this.workspaceFolder.uri.fsPath;
     const name = this.workspaceFolder.name;
-    const rootDirectory = new QLTestDirectory(fullPath, name);
+    const rootDirectory = new FileTreeDirectory(fullPath, name);
 
     // Don't try discovery on workspace folders that don't exist on the filesystem
     if (await pathExists(fullPath)) {
@@ -208,7 +107,9 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
         const relativePath = normalize(relative(fullPath, testPath));
         const dirName = dirname(relativePath);
         const parentDirectory = rootDirectory.createDirectory(dirName);
-        parentDirectory.addChild(new QLTestFile(testPath, basename(testPath)));
+        parentDirectory.addChild(
+          new FileTreeLeaf(testPath, basename(testPath)),
+        );
       }
 
       rootDirectory.finish();
diff --git a/extensions/ql-vscode/src/query-testing/test-adapter.ts b/extensions/ql-vscode/src/query-testing/test-adapter.ts
index dae8f9780..86c37b845 100644
--- a/extensions/ql-vscode/src/query-testing/test-adapter.ts
+++ b/extensions/ql-vscode/src/query-testing/test-adapter.ts
@@ -13,17 +13,17 @@ import {
   TestHub,
 } from "vscode-test-adapter-api";
 import { TestAdapterRegistrar } from "vscode-test-adapter-util";
-import {
-  QLTestFile,
-  QLTestNode,
-  QLTestDirectory,
-  QLTestDiscovery,
-} from "./qltest-discovery";
+import { QLTestDiscovery } from "./qltest-discovery";
 import { Event, EventEmitter, CancellationTokenSource } from "vscode";
 import { DisposableObject } from "../pure/disposable-object";
 import { CodeQLCliServer, TestCompleted } from "../codeql-cli/cli";
 import { testLogger } from "../common";
 import { TestRunner } from "./test-runner";
+import {
+  FileTreeDirectory,
+  FileTreeLeaf,
+  FileTreeNode,
+} from "../common/file-tree-nodes";
 
 /**
  * Get the full path of the `.expected` file for the specified QL test.
@@ -135,7 +135,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
   }
 
   private static createTestOrSuiteInfos(
-    testNodes: readonly QLTestNode[],
+    testNodes: readonly FileTreeNode[],
   ): Array<TestSuiteInfo | TestInfo> {
     return testNodes.map((childNode) => {
       return QLTestAdapter.createTestOrSuiteInfo(childNode);
@@ -143,18 +143,18 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
   }
 
   private static createTestOrSuiteInfo(
-    testNode: QLTestNode,
+    testNode: FileTreeNode,
   ): TestSuiteInfo | TestInfo {
-    if (testNode instanceof QLTestFile) {
+    if (testNode instanceof FileTreeLeaf) {
       return QLTestAdapter.createTestInfo(testNode);
-    } else if (testNode instanceof QLTestDirectory) {
+    } else if (testNode instanceof FileTreeDirectory) {
       return QLTestAdapter.createTestSuiteInfo(testNode, testNode.name);
     } else {
       throw new Error("Unexpected test type.");
     }
   }
 
-  private static createTestInfo(testFile: QLTestFile): TestInfo {
+  private static createTestInfo(testFile: FileTreeLeaf): TestInfo {
     return {
       type: "test",
       id: testFile.path,
@@ -165,7 +165,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
   }
 
   private static createTestSuiteInfo(
-    testDirectory: QLTestDirectory,
+    testDirectory: FileTreeDirectory,
     label: string,
   ): TestSuiteInfo {
     return {
diff --git a/extensions/ql-vscode/src/query-testing/test-manager.ts b/extensions/ql-vscode/src/query-testing/test-manager.ts
index fc5789b8e..94b79432b 100644
--- a/extensions/ql-vscode/src/query-testing/test-manager.ts
+++ b/extensions/ql-vscode/src/query-testing/test-manager.ts
@@ -16,12 +16,7 @@ import {
   workspace,
 } from "vscode";
 import { DisposableObject } from "../pure/disposable-object";
-import {
-  QLTestDirectory,
-  QLTestDiscovery,
-  QLTestFile,
-  QLTestNode,
-} from "./qltest-discovery";
+import { QLTestDiscovery } from "./qltest-discovery";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { getErrorMessage } from "../pure/helpers-pure";
 import { BaseLogger, LogOptions } from "../common";
@@ -29,6 +24,11 @@ import { TestRunner } from "./test-runner";
 import { TestManagerBase } from "./test-manager-base";
 import { App } from "../common/app";
 import { isWorkspaceFolderOnDisk } from "../helpers";
+import {
+  FileTreeDirectory,
+  FileTreeLeaf,
+  FileTreeNode,
+} from "../common/file-tree-nodes";
 
 /**
  * Returns the complete text content of the specified file. If there is an error reading the file,
@@ -209,7 +209,7 @@ export class TestManager extends TestManagerBase {
    */
   public updateTestsForWorkspaceFolder(
     workspaceFolder: WorkspaceFolder,
-    testDirectory: QLTestDirectory | undefined,
+    testDirectory: FileTreeDirectory | undefined,
   ): void {
     if (testDirectory !== undefined) {
       // Adding an item with the same ID as an existing item will replace it, which is exactly what
@@ -229,9 +229,9 @@ export class TestManager extends TestManagerBase {
   /**
    * Creates a tree of `TestItem`s from the root `QlTestNode` provided by test discovery.
    */
-  private createTestItemTree(node: QLTestNode, isRoot: boolean): TestItem {
+  private createTestItemTree(node: FileTreeNode, isRoot: boolean): TestItem {
     // Prefix the ID to identify it as a directory or a test
-    const itemType = node instanceof QLTestDirectory ? "dir" : "test";
+    const itemType = node instanceof FileTreeDirectory ? "dir" : "test";
     const testItem = this.testController.createTestItem(
       // For the root of a workspace folder, use the full path as the ID. Otherwise, use the node's
       // name as the ID, since it's shorter but still unique.
@@ -242,7 +242,7 @@ export class TestManager extends TestManagerBase {
 
     for (const childNode of node.children) {
       const childItem = this.createTestItemTree(childNode, false);
-      if (childNode instanceof QLTestFile) {
+      if (childNode instanceof FileTreeLeaf) {
         childItem.range = new Range(0, 0, 0, 0);
       }
       testItem.children.add(childItem);

From a7f6401be7ed85146a0955a1582790955f5b3dad Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 19 May 2023 12:17:34 +0100
Subject: [PATCH 022/119] Add workspace folders to App interface

---
 extensions/ql-vscode/src/common/app.ts               |  7 +++++++
 extensions/ql-vscode/src/common/vscode/vscode-app.ts |  8 ++++++++
 extensions/ql-vscode/test/__mocks__/appMock.ts       | 11 +++++++++++
 3 files changed, 26 insertions(+)

diff --git a/extensions/ql-vscode/src/common/app.ts b/extensions/ql-vscode/src/common/app.ts
index 2e749c018..f50eb729d 100644
--- a/extensions/ql-vscode/src/common/app.ts
+++ b/extensions/ql-vscode/src/common/app.ts
@@ -4,6 +4,11 @@ import { AppEventEmitter } from "./events";
 import { Logger } from "./logging";
 import { Memento } from "./memento";
 import { AppCommandManager } from "./commands";
+import type {
+  WorkspaceFolder,
+  Event,
+  WorkspaceFoldersChangeEvent,
+} from "vscode";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
@@ -14,6 +19,8 @@ export interface App {
   readonly globalStoragePath: string;
   readonly workspaceStoragePath?: string;
   readonly workspaceState: Memento;
+  readonly workspaceFolders: readonly WorkspaceFolder[] | undefined;
+  readonly onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>;
   readonly credentials: Credentials;
   readonly commands: AppCommandManager;
 }
diff --git a/extensions/ql-vscode/src/common/vscode/vscode-app.ts b/extensions/ql-vscode/src/common/vscode/vscode-app.ts
index 986d74c9a..9bc12afef 100644
--- a/extensions/ql-vscode/src/common/vscode/vscode-app.ts
+++ b/extensions/ql-vscode/src/common/vscode/vscode-app.ts
@@ -39,6 +39,14 @@ export class ExtensionApp implements App {
     return this.extensionContext.workspaceState;
   }
 
+  public get workspaceFolders(): readonly vscode.WorkspaceFolder[] | undefined {
+    return vscode.workspace.workspaceFolders;
+  }
+
+  public get onDidChangeWorkspaceFolders(): vscode.Event<vscode.WorkspaceFoldersChangeEvent> {
+    return vscode.workspace.onDidChangeWorkspaceFolders;
+  }
+
   public get subscriptions(): Disposable[] {
     return this.extensionContext.subscriptions;
   }
diff --git a/extensions/ql-vscode/test/__mocks__/appMock.ts b/extensions/ql-vscode/test/__mocks__/appMock.ts
index 1b6d9da9d..d53fa58b9 100644
--- a/extensions/ql-vscode/test/__mocks__/appMock.ts
+++ b/extensions/ql-vscode/test/__mocks__/appMock.ts
@@ -8,6 +8,11 @@ import { testCredentialsWithStub } from "../factories/authentication";
 import { Credentials } from "../../src/common/authentication";
 import { AppCommandManager } from "../../src/common/commands";
 import { createMockCommandManager } from "./commandsMock";
+import type {
+  Event,
+  WorkspaceFolder,
+  WorkspaceFoldersChangeEvent,
+} from "vscode";
 
 export function createMockApp({
   extensionPath = "/mock/extension/path",
@@ -15,6 +20,8 @@ export function createMockApp({
   globalStoragePath = "/mock/global/storage/path",
   createEventEmitter = <T>() => new MockAppEventEmitter<T>(),
   workspaceState = createMockMemento(),
+  workspaceFolders = [],
+  onDidChangeWorkspaceFolders = jest.fn(),
   credentials = testCredentialsWithStub(),
   commands = createMockCommandManager(),
 }: {
@@ -23,6 +30,8 @@ export function createMockApp({
   globalStoragePath?: string;
   createEventEmitter?: <T>() => AppEventEmitter<T>;
   workspaceState?: Memento;
+  workspaceFolders?: readonly WorkspaceFolder[] | undefined;
+  onDidChangeWorkspaceFolders?: Event<WorkspaceFoldersChangeEvent>;
   credentials?: Credentials;
   commands?: AppCommandManager;
 }): App {
@@ -34,6 +43,8 @@ export function createMockApp({
     workspaceStoragePath,
     globalStoragePath,
     workspaceState,
+    workspaceFolders,
+    onDidChangeWorkspaceFolders,
     createEventEmitter,
     credentials,
     commands,

From 13f8f19339634d9f00244b1d692adc633d654607 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 19 May 2023 12:18:14 +0100
Subject: [PATCH 023/119] Add resolveQueries to CodeQL CLI

---
 extensions/ql-vscode/src/codeql-cli/cli.ts | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts
index d5ef116d4..5c21cb457 100644
--- a/extensions/ql-vscode/src/codeql-cli/cli.ts
+++ b/extensions/ql-vscode/src/codeql-cli/cli.ts
@@ -134,6 +134,11 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries`.
+ */
+export type ResolvedQueries = string[];
+
 /**
  * The expected output of `codeql resolve tests`.
  */
@@ -731,6 +736,20 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Finds all available queries in a given directory.
+   * @param queryDir Root of directory tree to search for queries.
+   * @returns The list of queries that were found.
+   */
+  public async resolveQueries(queryDir: string): Promise<ResolvedQueries> {
+    const subcommandArgs = [queryDir];
+    return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
+      ["resolve", "queries"],
+      subcommandArgs,
+      "Resolving queries",
+    );
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.

From 0a534ae3609930174b52933049bd7679fed3d983 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 19 May 2023 12:32:17 +0100
Subject: [PATCH 024/119] Add QueryDiscovery class

---
 extensions/ql-vscode/src/extension.ts         |   2 +-
 .../src/queries-panel/queries-module.ts       |  16 +-
 .../src/queries-panel/query-discovery.ts      | 137 ++++++++++++++++++
 3 files changed, 151 insertions(+), 4 deletions(-)
 create mode 100644 extensions/ql-vscode/src/queries-panel/query-discovery.ts

diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts
index aad6940f5..a3771790e 100644
--- a/extensions/ql-vscode/src/extension.ts
+++ b/extensions/ql-vscode/src/extension.ts
@@ -733,7 +733,7 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(databaseUI);
 
-  QueriesModule.initialize(app);
+  QueriesModule.initialize(app, cliServer);
 
   void extLogger.log("Initializing evaluator log viewer.");
   const evalLogViewer = new EvalLogViewer();
diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts
index 8946eef55..4066c5b0a 100644
--- a/extensions/ql-vscode/src/queries-panel/queries-module.ts
+++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts
@@ -1,17 +1,20 @@
+import { CodeQLCliServer } from "../codeql-cli/cli";
 import { extLogger } from "../common";
 import { App, AppMode } from "../common/app";
 import { isCanary, showQueriesPanel } from "../config";
 import { DisposableObject } from "../pure/disposable-object";
 import { QueriesPanel } from "./queries-panel";
+import { QueryDiscovery } from "./query-discovery";
 
 export class QueriesModule extends DisposableObject {
   private queriesPanel: QueriesPanel | undefined;
+  private queryDiscovery: QueryDiscovery | undefined;
 
   private constructor(readonly app: App) {
     super();
   }
 
-  private initialize(app: App): void {
+  private initialize(app: App, cliServer: CodeQLCliServer): void {
     if (app.mode === AppMode.Production || !isCanary() || !showQueriesPanel()) {
       // Currently, we only want to expose the new panel when we are in development and canary mode
       // and the developer has enabled the "Show queries panel" flag.
@@ -19,15 +22,22 @@ export class QueriesModule extends DisposableObject {
     }
     void extLogger.log("Initializing queries panel.");
 
+    this.queryDiscovery = new QueryDiscovery(app, cliServer);
+    this.push(this.queryDiscovery);
+    this.queryDiscovery.refresh();
+
     this.queriesPanel = new QueriesPanel();
     this.push(this.queriesPanel);
   }
 
-  public static initialize(app: App): QueriesModule {
+  public static initialize(
+    app: App,
+    cliServer: CodeQLCliServer,
+  ): QueriesModule {
     const queriesModule = new QueriesModule(app);
     app.subscriptions.push(queriesModule);
 
-    queriesModule.initialize(app);
+    queriesModule.initialize(app, cliServer);
     return queriesModule;
   }
 }
diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
new file mode 100644
index 000000000..e42720c3c
--- /dev/null
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -0,0 +1,137 @@
+import { dirname, basename, normalize, relative } from "path";
+import { Discovery } from "../common/discovery";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { pathExists } from "fs-extra";
+import {
+  Event,
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFolder,
+} from "vscode";
+import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
+import { App } from "../common/app";
+import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
+
+/**
+ * The results of discovering queries.
+ */
+interface QueryDiscoveryResults {
+  /**
+   * A tree of directories and query files.
+   * May have multiple roots because of multiple workspaces.
+   */
+  queries: FileTreeDirectory[];
+
+  /**
+   * File system paths to watch. If any ql file changes in these directories
+   * or any subdirectories, then this could signify a change in queries.
+   */
+  watchPaths: Uri[];
+}
+
+/**
+ * Discovers all query files contained in the QL packs in a given workspace folder.
+ */
+export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
+  private results: QueryDiscoveryResults | undefined;
+
+  private readonly onDidChangeQueriesEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  constructor(
+    private readonly app: App,
+    private readonly cliServer: CodeQLCliServer,
+  ) {
+    super("Query Discovery");
+
+    this.push(app.onDidChangeWorkspaceFolders(this.refresh.bind(this)));
+    this.push(this.watcher.onDidChange(this.refresh.bind(this)));
+  }
+
+  public get queries(): FileTreeDirectory[] | undefined {
+    return this.results?.queries;
+  }
+
+  /**
+   * Event to be fired when the set of discovered queries may have changed.
+   */
+  public get onDidChangeQueries(): Event<void> {
+    return this.onDidChangeQueriesEmitter.event;
+  }
+
+  protected async discover(): Promise<QueryDiscoveryResults> {
+    const workspaceFolders = this.app.workspaceFolders;
+    if (workspaceFolders === undefined || workspaceFolders.length === 0) {
+      return {
+        queries: [],
+        watchPaths: [],
+      };
+    }
+
+    const queries = await this.discoverQueries(workspaceFolders);
+
+    return {
+      queries,
+      watchPaths: workspaceFolders.map((f) => f.uri),
+    };
+  }
+
+  protected update(results: QueryDiscoveryResults): void {
+    this.results = results;
+
+    this.watcher.clear();
+    for (const watchPath of results.watchPaths) {
+      // Watch for changes to any `.ql` file
+      this.watcher.addWatch(new RelativePattern(watchPath, "**/*.{ql}"));
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(watchPath, "**/"));
+    }
+    this.onDidChangeQueriesEmitter.fire();
+  }
+
+  /**
+   * Discover all queries in the specified directory and its subdirectories.
+   * @returns A `QueryDirectory` object describing the contents of the directory, or `undefined` if
+   *   no queries were found.
+   */
+  private async discoverQueries(
+    workspaceFolders: readonly WorkspaceFolder[],
+  ): Promise<FileTreeDirectory[]> {
+    const rootDirectories = [];
+    for (const workspaceFolder of workspaceFolders) {
+      rootDirectories.push(
+        await this.discoverQueriesInWorkspace(workspaceFolder),
+      );
+    }
+    return rootDirectories;
+  }
+
+  private async discoverQueriesInWorkspace(
+    workspaceFolder: WorkspaceFolder,
+  ): Promise<FileTreeDirectory> {
+    const fullPath = workspaceFolder.uri.fsPath;
+    const name = workspaceFolder.name;
+    const rootDirectory = new FileTreeDirectory(fullPath, name);
+
+    // Don't try discovery on workspace folders that don't exist on the filesystem
+    if (await pathExists(fullPath)) {
+      const resolvedQueries = await this.cliServer.resolveQueries(fullPath);
+      for (const queryPath of resolvedQueries) {
+        const relativePath = normalize(relative(fullPath, queryPath));
+        const dirName = dirname(relativePath);
+        const parentDirectory = rootDirectory.createDirectory(dirName);
+        parentDirectory.addChild(
+          new FileTreeLeaf(queryPath, basename(queryPath)),
+        );
+      }
+
+      rootDirectory.finish();
+    }
+    return rootDirectory;
+  }
+}

From a8f8990793aa7aa72bf9eef99360c3ee7370998c Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 19 May 2023 12:45:20 +0100
Subject: [PATCH 025/119] Hook query discovery results up to query panel

---
 .../src/queries-panel/queries-module.ts       |  2 +-
 .../src/queries-panel/queries-panel.ts        |  5 +-
 .../queries-panel/query-tree-data-provider.ts | 52 +++++++++++--------
 3 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts
index 4066c5b0a..f681e1c81 100644
--- a/extensions/ql-vscode/src/queries-panel/queries-module.ts
+++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts
@@ -26,7 +26,7 @@ export class QueriesModule extends DisposableObject {
     this.push(this.queryDiscovery);
     this.queryDiscovery.refresh();
 
-    this.queriesPanel = new QueriesPanel();
+    this.queriesPanel = new QueriesPanel(this.queryDiscovery);
     this.push(this.queriesPanel);
   }
 
diff --git a/extensions/ql-vscode/src/queries-panel/queries-panel.ts b/extensions/ql-vscode/src/queries-panel/queries-panel.ts
index 04f30b30d..31b95bd19 100644
--- a/extensions/ql-vscode/src/queries-panel/queries-panel.ts
+++ b/extensions/ql-vscode/src/queries-panel/queries-panel.ts
@@ -2,15 +2,16 @@ import * as vscode from "vscode";
 import { DisposableObject } from "../pure/disposable-object";
 import { QueryTreeDataProvider } from "./query-tree-data-provider";
 import { QueryTreeViewItem } from "./query-tree-view-item";
+import { QueryDiscovery } from "./query-discovery";
 
 export class QueriesPanel extends DisposableObject {
   private readonly dataProvider: QueryTreeDataProvider;
   private readonly treeView: vscode.TreeView<QueryTreeViewItem>;
 
-  public constructor() {
+  public constructor(queryDiscovery: QueryDiscovery) {
     super();
 
-    this.dataProvider = new QueryTreeDataProvider();
+    this.dataProvider = new QueryTreeDataProvider(queryDiscovery);
 
     this.treeView = vscode.window.createTreeView("codeQLQueries", {
       treeDataProvider: this.dataProvider,
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
index 85c446465..dd90e56e2 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -1,37 +1,47 @@
-import * as vscode from "vscode";
+import { Event, EventEmitter, TreeDataProvider, TreeItem } from "vscode";
 import { QueryTreeViewItem } from "./query-tree-view-item";
 import { DisposableObject } from "../pure/disposable-object";
+import { QueryDiscovery } from "./query-discovery";
+import { FileTreeNode } from "../common/file-tree-nodes";
 
 export class QueryTreeDataProvider
   extends DisposableObject
-  implements vscode.TreeDataProvider<QueryTreeViewItem>
+  implements TreeDataProvider<QueryTreeViewItem>
 {
   private queryTreeItems: QueryTreeViewItem[];
 
-  public constructor() {
+  private readonly onDidChangeTreeDataEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+
+  public constructor(private readonly queryDiscovery: QueryDiscovery) {
     super();
 
+    queryDiscovery.onDidChangeQueries(() => {
+      this.queryTreeItems = this.createTree();
+      this.onDidChangeTreeDataEmitter.fire();
+    });
+
     this.queryTreeItems = this.createTree();
   }
 
+  public get onDidChangeTreeData(): Event<void> {
+    return this.onDidChangeTreeDataEmitter.event;
+  }
+
   private createTree(): QueryTreeViewItem[] {
-    // Temporary mock data, just to populate the tree view.
-    return [
-      new QueryTreeViewItem("custom-pack", [
-        new QueryTreeViewItem("custom-pack/example.ql", []),
-      ]),
-      new QueryTreeViewItem("ql", [
-        new QueryTreeViewItem("ql/javascript", [
-          new QueryTreeViewItem("ql/javascript/example.ql", []),
-        ]),
-        new QueryTreeViewItem("ql/go", [
-          new QueryTreeViewItem("ql/go/security", [
-            new QueryTreeViewItem("ql/go/security/query1.ql", []),
-            new QueryTreeViewItem("ql/go/security/query2.ql", []),
-          ]),
-        ]),
-      ]),
-    ];
+    return (this.queryDiscovery.queries || []).map(
+      this.convertFileTreeNode.bind(this),
+    );
+  }
+
+  private convertFileTreeNode(
+    fileTreeDirectory: FileTreeNode,
+  ): QueryTreeViewItem {
+    return new QueryTreeViewItem(
+      fileTreeDirectory.path,
+      fileTreeDirectory.children.map(this.convertFileTreeNode.bind(this)),
+    );
   }
 
   /**
@@ -39,7 +49,7 @@ export class QueryTreeDataProvider
    * @param item The item to represent.
    * @returns The UI presentation of the item.
    */
-  public getTreeItem(item: QueryTreeViewItem): vscode.TreeItem {
+  public getTreeItem(item: QueryTreeViewItem): TreeItem {
     return item;
   }
 

From 4e46d87a77a8d7c690f79355202d3e731a81f368 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 19 May 2023 16:40:29 +0100
Subject: [PATCH 026/119] Use name from file tree instead of calculating from
 path

---
 .../src/queries-panel/query-tree-data-provider.ts        | 1 +
 .../ql-vscode/src/queries-panel/query-tree-view-item.ts  | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
index dd90e56e2..0b92b4338 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -39,6 +39,7 @@ export class QueryTreeDataProvider
     fileTreeDirectory: FileTreeNode,
   ): QueryTreeViewItem {
     return new QueryTreeViewItem(
+      fileTreeDirectory.name,
       fileTreeDirectory.path,
       fileTreeDirectory.children.map(this.convertFileTreeNode.bind(this)),
     );
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
index f9adc5c00..17caea32d 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts
@@ -1,9 +1,12 @@
 import * as vscode from "vscode";
-import { basename } from "path";
 
 export class QueryTreeViewItem extends vscode.TreeItem {
-  constructor(path: string, public readonly children: QueryTreeViewItem[]) {
-    super(basename(path));
+  constructor(
+    name: string,
+    path: string,
+    public readonly children: QueryTreeViewItem[],
+  ) {
+    super(name);
     this.tooltip = path;
     this.collapsibleState = this.children.length
       ? vscode.TreeItemCollapsibleState.Collapsed

From 791a4453421b55e07b8e7d4c8fdca205d2bc5a6c Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@users.noreply.github.com>
Date: Mon, 22 May 2023 11:37:42 +0200
Subject: [PATCH 027/119] Update copy of deprecation warning message

Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
---
 extensions/ql-vscode/src/extension.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts
index b3625f50a..888884bdd 100644
--- a/extensions/ql-vscode/src/extension.ts
+++ b/extensions/ql-vscode/src/extension.ts
@@ -426,7 +426,7 @@ export async function activate(
       void showAndLogWarningMessage(
         `You are using an unsupported version of the CodeQL CLI (${ver}). ` +
           `The minimum supported version is ${CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION}. ` +
-          `Please upgrade to the latest version of the CodeQL CLI.`,
+          `Please upgrade to a newer version of the CodeQL CLI.`,
       );
       unsupportedWarningShown = true;
     });

From 935c9b996f50e4a4500f6ffd86a29684875941dd Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Mon, 22 May 2023 10:41:49 +0100
Subject: [PATCH 028/119] Don't include bad workspace roots in the list

---
 .../src/queries-panel/query-discovery.ts      | 38 +++++++++++--------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index e42720c3c..8abca9d16 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -104,34 +104,40 @@ export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
   ): Promise<FileTreeDirectory[]> {
     const rootDirectories = [];
     for (const workspaceFolder of workspaceFolders) {
-      rootDirectories.push(
-        await this.discoverQueriesInWorkspace(workspaceFolder),
+      const rootDirectory = await this.discoverQueriesInWorkspace(
+        workspaceFolder,
       );
+      if (rootDirectory !== undefined) {
+        rootDirectories.push(rootDirectory);
+      }
     }
     return rootDirectories;
   }
 
   private async discoverQueriesInWorkspace(
     workspaceFolder: WorkspaceFolder,
-  ): Promise<FileTreeDirectory> {
+  ): Promise<FileTreeDirectory | undefined> {
     const fullPath = workspaceFolder.uri.fsPath;
     const name = workspaceFolder.name;
-    const rootDirectory = new FileTreeDirectory(fullPath, name);
 
     // Don't try discovery on workspace folders that don't exist on the filesystem
-    if (await pathExists(fullPath)) {
-      const resolvedQueries = await this.cliServer.resolveQueries(fullPath);
-      for (const queryPath of resolvedQueries) {
-        const relativePath = normalize(relative(fullPath, queryPath));
-        const dirName = dirname(relativePath);
-        const parentDirectory = rootDirectory.createDirectory(dirName);
-        parentDirectory.addChild(
-          new FileTreeLeaf(queryPath, basename(queryPath)),
-        );
-      }
-
-      rootDirectory.finish();
+    if (!(await pathExists(fullPath))) {
+      return undefined;
     }
+
+    const rootDirectory = new FileTreeDirectory(fullPath, name);
+
+    const resolvedQueries = await this.cliServer.resolveQueries(fullPath);
+    for (const queryPath of resolvedQueries) {
+      const relativePath = normalize(relative(fullPath, queryPath));
+      const dirName = dirname(relativePath);
+      const parentDirectory = rootDirectory.createDirectory(dirName);
+      parentDirectory.addChild(
+        new FileTreeLeaf(queryPath, basename(queryPath)),
+      );
+    }
+
+    rootDirectory.finish();
     return rootDirectory;
   }
 }

From 78f11397e2f63566f9c1aa1b33c0cf55106ab653 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Mon, 22 May 2023 10:42:47 +0100
Subject: [PATCH 029/119] Exclude src archives and other non-normal workspace
 folders

---
 extensions/ql-vscode/src/queries-panel/query-discovery.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index 8abca9d16..21563c928 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -12,6 +12,7 @@ import {
 import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
 import { App } from "../common/app";
 import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
+import { getOnDiskWorkspaceFolders } from "../helpers";
 
 /**
  * The results of discovering queries.
@@ -121,7 +122,10 @@ export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
     const name = workspaceFolder.name;
 
     // Don't try discovery on workspace folders that don't exist on the filesystem
-    if (!(await pathExists(fullPath))) {
+    if (
+      !(await pathExists(fullPath)) ||
+      !getOnDiskWorkspaceFolders().includes(fullPath)
+    ) {
       return undefined;
     }
 

From cda3483b01d377245fd5960adcc3d0251bf28301 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Mon, 22 May 2023 10:51:32 +0100
Subject: [PATCH 030/119] Use getOnDiskWorkspaceFoldersObjects early instead of
 getting all folders and filtering later

---
 .../src/queries-panel/query-discovery.ts      | 29 +++++--------------
 1 file changed, 7 insertions(+), 22 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index 21563c928..408537706 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -1,7 +1,6 @@
 import { dirname, basename, normalize, relative } from "path";
 import { Discovery } from "../common/discovery";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { pathExists } from "fs-extra";
 import {
   Event,
   EventEmitter,
@@ -12,7 +11,7 @@ import {
 import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
 import { App } from "../common/app";
 import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
-import { getOnDiskWorkspaceFolders } from "../helpers";
+import { getOnDiskWorkspaceFoldersObjects } from "../helpers";
 
 /**
  * The results of discovering queries.
@@ -44,10 +43,7 @@ export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
     new MultiFileSystemWatcher(),
   );
 
-  constructor(
-    private readonly app: App,
-    private readonly cliServer: CodeQLCliServer,
-  ) {
+  constructor(app: App, private readonly cliServer: CodeQLCliServer) {
     super("Query Discovery");
 
     this.push(app.onDidChangeWorkspaceFolders(this.refresh.bind(this)));
@@ -66,8 +62,8 @@ export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
   }
 
   protected async discover(): Promise<QueryDiscoveryResults> {
-    const workspaceFolders = this.app.workspaceFolders;
-    if (workspaceFolders === undefined || workspaceFolders.length === 0) {
+    const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+    if (workspaceFolders.length === 0) {
       return {
         queries: [],
         watchPaths: [],
@@ -105,30 +101,19 @@ export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
   ): Promise<FileTreeDirectory[]> {
     const rootDirectories = [];
     for (const workspaceFolder of workspaceFolders) {
-      const rootDirectory = await this.discoverQueriesInWorkspace(
-        workspaceFolder,
+      rootDirectories.push(
+        await this.discoverQueriesInWorkspace(workspaceFolder),
       );
-      if (rootDirectory !== undefined) {
-        rootDirectories.push(rootDirectory);
-      }
     }
     return rootDirectories;
   }
 
   private async discoverQueriesInWorkspace(
     workspaceFolder: WorkspaceFolder,
-  ): Promise<FileTreeDirectory | undefined> {
+  ): Promise<FileTreeDirectory> {
     const fullPath = workspaceFolder.uri.fsPath;
     const name = workspaceFolder.name;
 
-    // Don't try discovery on workspace folders that don't exist on the filesystem
-    if (
-      !(await pathExists(fullPath)) ||
-      !getOnDiskWorkspaceFolders().includes(fullPath)
-    ) {
-      return undefined;
-    }
-
     const rootDirectory = new FileTreeDirectory(fullPath, name);
 
     const resolvedQueries = await this.cliServer.resolveQueries(fullPath);

From 73e41f8d610a841fc766a0f67ad1e26a862776df Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Mon, 22 May 2023 12:19:53 +0100
Subject: [PATCH 031/119] Mark AppEventEmitter as disposable

---
 extensions/ql-vscode/src/common/events.ts                | 2 +-
 .../ql-vscode/src/databases/config/db-config-store.ts    | 4 +++-
 extensions/ql-vscode/src/databases/db-manager.ts         | 9 +++++++--
 extensions/ql-vscode/test/__mocks__/appMock.ts           | 4 ++++
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/extensions/ql-vscode/src/common/events.ts b/extensions/ql-vscode/src/common/events.ts
index 67e457405..16ba4821c 100644
--- a/extensions/ql-vscode/src/common/events.ts
+++ b/extensions/ql-vscode/src/common/events.ts
@@ -4,7 +4,7 @@ export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;
 }
 
-export interface AppEventEmitter<T> {
+export interface AppEventEmitter<T> extends Disposable {
   event: AppEvent<T>;
   fire(data: T): void;
 }
diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts
index 3013d164f..54b216929 100644
--- a/extensions/ql-vscode/src/databases/config/db-config-store.ts
+++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts
@@ -61,7 +61,9 @@ export class DbConfigStore extends DisposableObject {
     this.configErrors = [];
     this.configWatcher = undefined;
     this.configValidator = new DbConfigValidator(app.extensionPath);
-    this.onDidChangeConfigEventEmitter = app.createEventEmitter<void>();
+    this.onDidChangeConfigEventEmitter = this.push(
+      app.createEventEmitter<void>(),
+    );
     this.onDidChangeConfig = this.onDidChangeConfigEventEmitter.event;
   }
 
diff --git a/extensions/ql-vscode/src/databases/db-manager.ts b/extensions/ql-vscode/src/databases/db-manager.ts
index cc85bccc0..74c45056b 100644
--- a/extensions/ql-vscode/src/databases/db-manager.ts
+++ b/extensions/ql-vscode/src/databases/db-manager.ts
@@ -1,6 +1,7 @@
 import { App } from "../common/app";
 import { AppEvent, AppEventEmitter } from "../common/events";
 import { ValueResult } from "../common/value-result";
+import { DisposableObject } from "../pure/disposable-object";
 import { DbConfigStore } from "./config/db-config-store";
 import {
   DbItem,
@@ -23,7 +24,7 @@ import {
 import { createRemoteTree } from "./db-tree-creator";
 import { DbConfigValidationError } from "./db-validation-errors";
 
-export class DbManager {
+export class DbManager extends DisposableObject {
   public readonly onDbItemsChanged: AppEvent<void>;
   public static readonly DB_EXPANDED_STATE_KEY = "db_expanded";
   private readonly onDbItemsChangesEventEmitter: AppEventEmitter<void>;
@@ -32,7 +33,11 @@ export class DbManager {
     private readonly app: App,
     private readonly dbConfigStore: DbConfigStore,
   ) {
-    this.onDbItemsChangesEventEmitter = app.createEventEmitter<void>();
+    super();
+
+    this.onDbItemsChangesEventEmitter = this.push(
+      app.createEventEmitter<void>(),
+    );
     this.onDbItemsChanged = this.onDbItemsChangesEventEmitter.event;
 
     this.dbConfigStore.onDidChangeConfig(() => {
diff --git a/extensions/ql-vscode/test/__mocks__/appMock.ts b/extensions/ql-vscode/test/__mocks__/appMock.ts
index d53fa58b9..418310d5b 100644
--- a/extensions/ql-vscode/test/__mocks__/appMock.ts
+++ b/extensions/ql-vscode/test/__mocks__/appMock.ts
@@ -63,4 +63,8 @@ export class MockAppEventEmitter<T> implements AppEventEmitter<T> {
   public fire(): void {
     // no-op
   }
+
+  public dispose() {
+    // no-op
+  }
 }

From f855d81526c5744ad93eeec91c2549d414162b8c Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Mon, 22 May 2023 13:34:48 +0100
Subject: [PATCH 032/119] Make sure DbManager is also disposed

---
 extensions/ql-vscode/src/databases/db-module.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/databases/db-module.ts b/extensions/ql-vscode/src/databases/db-module.ts
index 8d47f75a8..0ca2898db 100644
--- a/extensions/ql-vscode/src/databases/db-module.ts
+++ b/extensions/ql-vscode/src/databases/db-module.ts
@@ -17,7 +17,7 @@ export class DbModule extends DisposableObject {
     super();
 
     this.dbConfigStore = new DbConfigStore(app);
-    this.dbManager = new DbManager(app, this.dbConfigStore);
+    this.dbManager = this.push(new DbManager(app, this.dbConfigStore));
   }
 
   public static async initialize(app: App): Promise<DbModule> {

From da5d0d2a8455e7df5de2e08d3b58a0b04294fa87 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Mon, 22 May 2023 12:23:19 +0100
Subject: [PATCH 033/119] Use app.createEventEmitter in QueryDiscovery

---
 .../ql-vscode/src/queries-panel/query-discovery.ts  | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index 408537706..c0b3cb4e8 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -1,13 +1,7 @@
 import { dirname, basename, normalize, relative } from "path";
 import { Discovery } from "../common/discovery";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import {
-  Event,
-  EventEmitter,
-  RelativePattern,
-  Uri,
-  WorkspaceFolder,
-} from "vscode";
+import { Event, RelativePattern, Uri, WorkspaceFolder } from "vscode";
 import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watcher";
 import { App } from "../common/app";
 import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
@@ -36,9 +30,7 @@ interface QueryDiscoveryResults {
 export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
   private results: QueryDiscoveryResults | undefined;
 
-  private readonly onDidChangeQueriesEmitter = this.push(
-    new EventEmitter<void>(),
-  );
+  private readonly onDidChangeQueriesEmitter;
   private readonly watcher: MultiFileSystemWatcher = this.push(
     new MultiFileSystemWatcher(),
   );
@@ -46,6 +38,7 @@ export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
   constructor(app: App, private readonly cliServer: CodeQLCliServer) {
     super("Query Discovery");
 
+    this.onDidChangeQueriesEmitter = this.push(app.createEventEmitter<void>());
     this.push(app.onDidChangeWorkspaceFolders(this.refresh.bind(this)));
     this.push(this.watcher.onDidChange(this.refresh.bind(this)));
   }

From fed61586150bc67eae49831a0a1c83eacbd41eb4 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Fri, 19 May 2023 12:32:48 +0000
Subject: [PATCH 034/119] Add GH client method to query search endpoint

---
 .../variant-analysis/gh-api/gh-api-client.ts  | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
index 00bbafde9..3ef305c71 100644
--- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
+++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
@@ -8,6 +8,26 @@ import {
 } from "./variant-analysis";
 import { Repository } from "./repository";
 
+export async function getCodeSearchRepositories(
+  credentials: Credentials,
+  query: string,
+): Promise<string[]> {
+  const octokit = await credentials.getOctokit();
+
+  const response = await octokit.rest.search.repos({
+    q: query,
+    per_page: 100,
+  });
+
+  if (response.status === 200) {
+    const nwos = response.data.items.map((item) => item.full_name);
+
+    return [...new Set(nwos)];
+  }
+
+  return [];
+}
+
 export async function submitVariantAnalysis(
   credentials: Credentials,
   submissionDetails: VariantAnalysisSubmission,

From 4d9e8d98b418fa71e7bc4e3b15420c389b734beb Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Mon, 22 May 2023 15:49:25 +0000
Subject: [PATCH 035/119] Ask for user input and add response to list

---
 .../src/databases/config/db-config-store.ts   | 25 ++++++++++
 .../ql-vscode/src/databases/db-manager.ts     |  7 +++
 .../ql-vscode/src/databases/ui/db-panel.ts    | 50 ++++++++++++++++++-
 .../ui/db-tree-view-item-action.test.ts       |  9 +++-
 .../databases/db-panel-rendering.test.ts      |  7 ++-
 5 files changed, 94 insertions(+), 4 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts
index 3013d164f..f1e7e4637 100644
--- a/extensions/ql-vscode/src/databases/config/db-config-store.ts
+++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts
@@ -145,6 +145,31 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
+  public async addRemoteReposToList(
+    repoNwoList: string[],
+    parentList: string,
+  ): Promise<void> {
+    if (!this.config) {
+      throw Error("Cannot add variant analysis repos if config is not loaded");
+    }
+
+    const config = cloneDbConfig(this.config);
+    const parent = config.databases.variantAnalysis.repositoryLists.find(
+      (list) => list.name === parentList,
+    );
+    if (!parent) {
+      throw Error(`Cannot find parent list '${parentList}'`);
+    }
+
+    const newRepositoriesList = new Set([
+      ...new Set(parent.repositories),
+      ...new Set(repoNwoList),
+    ]);
+    parent.repositories = [...newRepositoriesList];
+
+    await this.writeConfig(config);
+  }
+
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
diff --git a/extensions/ql-vscode/src/databases/db-manager.ts b/extensions/ql-vscode/src/databases/db-manager.ts
index cc85bccc0..b66d5d5de 100644
--- a/extensions/ql-vscode/src/databases/db-manager.ts
+++ b/extensions/ql-vscode/src/databases/db-manager.ts
@@ -100,6 +100,13 @@ export class DbManager {
     await this.dbConfigStore.addRemoteRepo(nwo, parentList);
   }
 
+  public async addNewRemoteReposToList(
+    nwoList: string[],
+    parentList: string,
+  ): Promise<void> {
+    await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
+  }
+
   public async addNewRemoteOwner(owner: string): Promise<void> {
     await this.dbConfigStore.addRemoteOwner(owner);
   }
diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index 203074222..0d77e13c8 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -32,6 +32,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query";
 import { getErrorMessage } from "../../pure/helpers-pure";
 import { DatabasePanelCommands } from "../../common/commands";
 import { App } from "../../common/app";
+import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client";
+import { QueryLanguage } from "../../common/query-language";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
@@ -41,6 +43,10 @@ export interface AddListQuickPickItem extends QuickPickItem {
   databaseKind: DbListKind;
 }
 
+export interface CodeSearchQuickPickItem extends QuickPickItem {
+  language: string;
+}
+
 export class DbPanel extends DisposableObject {
   private readonly dataProvider: DbTreeDataProvider;
   private readonly treeView: TreeView<DbTreeViewItem>;
@@ -326,9 +332,51 @@ export class DbPanel extends DisposableObject {
   }
 
   private async importCodeSearch(treeViewItem: DbTreeViewItem): Promise<void> {
-    if (treeViewItem.dbItem === undefined) {
+    if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) {
       throw new Error("Please select a valid list to add code search results.");
     }
+
+    const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values(
+      QueryLanguage,
+    ).map((language) => ({
+      label: language.toString(),
+      alwaysShow: true,
+      language: language.toString(),
+    }));
+
+    const codeSearchLanguage =
+      await window.showQuickPick<CodeSearchQuickPickItem>(
+        languageQuickPickItems,
+        {
+          title: "Select the language you want to query",
+          placeHolder: "Select an option",
+          ignoreFocusOut: true,
+        },
+      );
+    if (!codeSearchLanguage) {
+      // We don't need to display a warning pop-up in this case, since the user just escaped out of the operation.
+      // We set 'true' to make this a silent exception.
+      throw new UserCancellationException("No language selected", true);
+    }
+
+    const codeSearchQuery = await window.showInputBox({
+      title: "Code search query",
+      prompt: "Insert code search query",
+      placeHolder: "org:github",
+    });
+    if (codeSearchQuery === undefined || codeSearchQuery === "") {
+      return;
+    }
+
+    const repositories = await getCodeSearchRepositories(
+      this.app.credentials,
+      `${codeSearchQuery} language:${codeSearchLanguage.language}`,
+    );
+
+    await this.dbManager.addNewRemoteReposToList(
+      repositories,
+      treeViewItem.dbItem.listName,
+    );
   }
 
   private async onDidCollapseElement(
diff --git a/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts b/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts
index 036be67d8..48f8cfc83 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/ui/db-tree-view-item-action.test.ts
@@ -62,12 +62,17 @@ describe("getDbItemActions", () => {
     expect(actions.length).toEqual(0);
   });
 
-  it("should set canBeSelected, canBeRemoved and canBeRenamed for remote user defined db list", () => {
+  it("should set canBeSelected, canBeRemoved, canBeRenamed and canImportCodeSearch for remote user defined db list", () => {
     const dbItem = createRemoteUserDefinedListDbItem();
 
     const actions = getDbItemActions(dbItem);
 
-    expect(actions).toEqual(["canBeSelected", "canBeRemoved", "canBeRenamed"]);
+    expect(actions).toEqual([
+      "canBeSelected",
+      "canBeRemoved",
+      "canBeRenamed",
+      "canImportCodeSearch",
+    ]);
   });
 
   it("should not set canBeSelected for remote user defined db list that is already selected", () => {
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts
index d3e08fdac..8f8c8689f 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/databases/db-panel-rendering.test.ts
@@ -349,7 +349,12 @@ describe("db panel rendering nodes", () => {
     expect(item.tooltip).toBeUndefined();
     expect(item.iconPath).toBeUndefined();
     expect(item.collapsibleState).toBe(TreeItemCollapsibleState.Collapsed);
-    checkDbItemActions(item, ["canBeSelected", "canBeRenamed", "canBeRemoved"]);
+    checkDbItemActions(item, [
+      "canBeSelected",
+      "canBeRenamed",
+      "canBeRemoved",
+      "canImportCodeSearch",
+    ]);
     expect(item.children).toBeTruthy();
     expect(item.children.length).toBe(repos.length);
 

From 57f04fcae5282f48b610b591b8514cfa80389f27 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 23 May 2023 09:46:04 +0100
Subject: [PATCH 036/119] Add type to onDidChangeQueriesEmitter

---
 extensions/ql-vscode/src/queries-panel/query-discovery.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index c0b3cb4e8..b05866bfc 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -6,6 +6,7 @@ import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watch
 import { App } from "../common/app";
 import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
 import { getOnDiskWorkspaceFoldersObjects } from "../helpers";
+import { AppEventEmitter } from "../common/events";
 
 /**
  * The results of discovering queries.
@@ -30,7 +31,7 @@ interface QueryDiscoveryResults {
 export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
   private results: QueryDiscoveryResults | undefined;
 
-  private readonly onDidChangeQueriesEmitter;
+  private readonly onDidChangeQueriesEmitter: AppEventEmitter<void>;
   private readonly watcher: MultiFileSystemWatcher = this.push(
     new MultiFileSystemWatcher(),
   );

From e964ce67137b8fa7773fd5504b8567ad25c48ac4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaroslav=20Loba=C4=8Devski?= <jarlob@github.com>
Date: Tue, 23 May 2023 10:51:46 +0200
Subject: [PATCH 037/119] Retry results download if connection times out

---
 .../variant-analysis-manager.ts               | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
index 35e197b9b..0ca195789 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
@@ -75,6 +75,7 @@ import {
   writeRepoStates,
 } from "./repo-states-store";
 import { GITHUB_AUTH_PROVIDER_ID } from "../common/vscode/authentication";
+import { FetchError } from "node-fetch";
 
 export class VariantAnalysisManager
   extends DisposableObject
@@ -613,12 +614,27 @@ export class VariantAnalysisManager
             });
           }
         };
-        await this.variantAnalysisResultsManager.download(
-          variantAnalysis.id,
-          repoTask,
-          this.getVariantAnalysisStorageLocation(variantAnalysis.id),
-          updateRepoStateCallback,
-        );
+        let retry = 0;
+        for (;;) {
+          try {
+            await this.variantAnalysisResultsManager.download(
+              variantAnalysis.id,
+              repoTask,
+              this.getVariantAnalysisStorageLocation(variantAnalysis.id),
+              updateRepoStateCallback,
+            );
+            break;
+          } catch (e) {
+            if (
+              retry++ < 3 &&
+              e instanceof FetchError &&
+              (e.code === "ETIMEDOUT" || e.code === "ECONNRESET")
+            ) {
+              continue;
+            }
+            throw e;
+          }
+        }
       } catch (e) {
         repoState.downloadStatus =
           VariantAnalysisScannedRepositoryDownloadStatus.Failed;

From 32fb4e5db6eab9bbe0d01809a58dbaf5e110ad90 Mon Sep 17 00:00:00 2001
From: jarlob <novaisas@gmail.com>
Date: Tue, 23 May 2023 22:47:13 +0200
Subject: [PATCH 038/119] Move max retry to a constant

---
 .../src/variant-analysis/variant-analysis-manager.ts          | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
index 0ca195789..e84e83e01 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
@@ -77,6 +77,8 @@ import {
 import { GITHUB_AUTH_PROVIDER_ID } from "../common/vscode/authentication";
 import { FetchError } from "node-fetch";
 
+const maxRetryCount = 3;
+
 export class VariantAnalysisManager
   extends DisposableObject
   implements VariantAnalysisViewManager<VariantAnalysisView>
@@ -626,7 +628,7 @@ export class VariantAnalysisManager
             break;
           } catch (e) {
             if (
-              retry++ < 3 &&
+              retry++ < maxRetryCount &&
               e instanceof FetchError &&
               (e.code === "ETIMEDOUT" || e.code === "ECONNRESET")
             ) {

From 65cfd851f23ebc81281575326a366b5a61a65976 Mon Sep 17 00:00:00 2001
From: jarlob <novaisas@gmail.com>
Date: Tue, 23 May 2023 23:21:51 +0200
Subject: [PATCH 039/119] Add logging

---
 .../src/variant-analysis/variant-analysis-manager.ts       | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
index e84e83e01..b3896884e 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
@@ -76,6 +76,7 @@ import {
 } from "./repo-states-store";
 import { GITHUB_AUTH_PROVIDER_ID } from "../common/vscode/authentication";
 import { FetchError } from "node-fetch";
+import { extLogger } from "../common";
 
 const maxRetryCount = 3;
 
@@ -632,8 +633,14 @@ export class VariantAnalysisManager
               e instanceof FetchError &&
               (e.code === "ETIMEDOUT" || e.code === "ECONNRESET")
             ) {
+              void extLogger.log(
+                `Timeout while trying to download variant analysis with id: ${variantAnalysis.id}. Retrying...`,
+              );
               continue;
             }
+            void extLogger.log(
+              `Failed to download variable analysis after ${retry} attempts.`,
+            );
             throw e;
           }
         }

From 6f6c229ca3c038bb64d036ae95a66dcf4fec255f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaroslav=20Loba=C4=8Devski?= <jarlob@github.com>
Date: Tue, 23 May 2023 23:24:18 +0200
Subject: [PATCH 040/119] Update
 extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts

---
 .../ql-vscode/src/variant-analysis/variant-analysis-manager.ts  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
index b3896884e..51628ef52 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
@@ -639,7 +639,7 @@ export class VariantAnalysisManager
               continue;
             }
             void extLogger.log(
-              `Failed to download variable analysis after ${retry} attempts.`,
+              `Failed to download variant analysis after ${retry} attempts.`,
             );
             throw e;
           }

From f76d7bfd9615f658a6f0228693803e8c7e9a1060 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 24 May 2023 08:05:30 +0000
Subject: [PATCH 041/119] Use pagination for code search api call

---
 .../variant-analysis/gh-api/gh-api-client.ts  | 20 +++++++++----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
index 3ef305c71..8c348d207 100644
--- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
+++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
@@ -14,18 +14,16 @@ export async function getCodeSearchRepositories(
 ): Promise<string[]> {
   const octokit = await credentials.getOctokit();
 
-  const response = await octokit.rest.search.repos({
-    q: query,
-    per_page: 100,
-  });
+  const nwos = await octokit.paginate(
+    octokit.rest.search.repos,
+    {
+      q: query,
+      per_page: 100,
+    },
+    (response) => response.data.map((item) => item.full_name),
+  );
 
-  if (response.status === 200) {
-    const nwos = response.data.items.map((item) => item.full_name);
-
-    return [...new Set(nwos)];
-  }
-
-  return [];
+  return [...new Set(nwos)];
 }
 
 export async function submitVariantAnalysis(

From dd01832ebe870dfcb633dcba52bb197613714f0a Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 24 May 2023 08:06:31 +0000
Subject: [PATCH 042/119] Add no more than 1000 items to a list plus tests

---
 .../src/databases/config/db-config-store.ts   |  6 +-
 .../databases/config/db-config-store.test.ts  | 74 +++++++++++++++++++
 2 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts
index f1e7e4637..f25bf81f8 100644
--- a/extensions/ql-vscode/src/databases/config/db-config-store.ts
+++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts
@@ -165,8 +165,12 @@ export class DbConfigStore extends DisposableObject {
       ...new Set(parent.repositories),
       ...new Set(repoNwoList),
     ]);
-    parent.repositories = [...newRepositoriesList];
 
+    if (newRepositoriesList.size > 1000) {
+      parent.repositories = [...Array.from(newRepositoriesList).slice(0, 1000)];
+    } else {
+      parent.repositories = [...newRepositoriesList];
+    }
     await this.writeConfig(config);
   }
 
diff --git a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
index ca4b7ead6..3bb918734 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
@@ -241,6 +241,80 @@ describe("db config store", () => {
       configStore.dispose();
     });
 
+    it("should add unique remote repositories to the correct list", async () => {
+      // Initial set up
+      const dbConfig = createDbConfig({
+        remoteLists: [
+          {
+            name: "list1",
+            repositories: ["owner/repo1"],
+          },
+        ],
+      });
+
+      const configStore = await initializeConfig(dbConfig, configPath, app);
+      expect(
+        configStore.getConfig().value.databases.variantAnalysis
+          .repositoryLists[0],
+      ).toEqual({
+        name: "list1",
+        repositories: ["owner/repo1"],
+      });
+
+      // Add
+      await configStore.addRemoteReposToList(
+        ["owner/repo1", "owner/repo2"],
+        "list1",
+      );
+
+      // Read the config file
+      const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
+
+      // Check that the config file has been updated
+      const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
+      expect(updatedRemoteDbs.repositories).toHaveLength(0);
+      expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
+      expect(updatedRemoteDbs.repositoryLists[0]).toEqual({
+        name: "list1",
+        repositories: ["owner/repo1", "owner/repo2"],
+      });
+
+      configStore.dispose();
+    });
+
+    it("should add no more than 1000 repositories to a list", async () => {
+      // Initial set up
+      const dbConfig = createDbConfig({
+        remoteLists: [
+          {
+            name: "list1",
+            repositories: [],
+          },
+        ],
+      });
+
+      const configStore = await initializeConfig(dbConfig, configPath, app);
+
+      // Add
+      await configStore.addRemoteReposToList(
+        [...Array(1001).keys()].map((i) => `owner/db${i}`),
+        "list1",
+      );
+
+      // Read the config file
+      const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
+
+      // Check that the config file has been updated
+      const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
+      expect(updatedRemoteDbs.repositories).toHaveLength(0);
+      expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
+      expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
+        1000,
+      );
+
+      configStore.dispose();
+    });
+
     it("should add a remote owner", async () => {
       // Initial set up
       const dbConfig = createDbConfig();

From 7ef35af68a192f2658d525727bdc5876d675cc07 Mon Sep 17 00:00:00 2001
From: Shati Patel <42641846+shati-patel@users.noreply.github.com>
Date: Wed, 24 May 2023 10:15:34 +0100
Subject: [PATCH 043/119] Don't create `"."` directory in queries/tests tree
 view (#2442)

---
 extensions/ql-vscode/src/common/file-tree-nodes.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/extensions/ql-vscode/src/common/file-tree-nodes.ts b/extensions/ql-vscode/src/common/file-tree-nodes.ts
index 32144c885..3544265ca 100644
--- a/extensions/ql-vscode/src/common/file-tree-nodes.ts
+++ b/extensions/ql-vscode/src/common/file-tree-nodes.ts
@@ -41,6 +41,9 @@ export class FileTreeDirectory extends FileTreeNode {
   }
 
   public createDirectory(relativePath: string): FileTreeDirectory {
+    if (relativePath === ".") {
+      return this;
+    }
     const dirName = dirname(relativePath);
     if (dirName === ".") {
       return this.createChildDirectory(relativePath);

From 9bd852a71b6c20f6b62363b61349e677fc38d5c7 Mon Sep 17 00:00:00 2001
From: jarlob <novaisas@gmail.com>
Date: Wed, 24 May 2023 11:20:52 +0200
Subject: [PATCH 044/119] Add error message to logging.

---
 .../src/variant-analysis/variant-analysis-manager.ts          | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
index 51628ef52..6c84b9915 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts
@@ -634,7 +634,9 @@ export class VariantAnalysisManager
               (e.code === "ETIMEDOUT" || e.code === "ECONNRESET")
             ) {
               void extLogger.log(
-                `Timeout while trying to download variant analysis with id: ${variantAnalysis.id}. Retrying...`,
+                `Timeout while trying to download variant analysis with id: ${
+                  variantAnalysis.id
+                }. Error: ${getErrorMessage(e)}. Retrying...`,
               );
               continue;
             }

From 69f9ecc2f459a520c54e5a066aaf7cd13c966175 Mon Sep 17 00:00:00 2001
From: jarlob <novaisas@gmail.com>
Date: Wed, 24 May 2023 11:25:50 +0200
Subject: [PATCH 045/119] Delete unfinished downloads in case of retry

---
 .../src/variant-analysis/variant-analysis-results-manager.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts
index 5c538e621..2ccbee55d 100644
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts
@@ -1,4 +1,4 @@
-import { appendFile, pathExists } from "fs-extra";
+import { appendFile, pathExists, rm } from "fs-extra";
 import fetch from "node-fetch";
 import { EOL } from "os";
 import { join } from "path";
@@ -82,6 +82,9 @@ export class VariantAnalysisResultsManager extends DisposableObject {
 
     const zipFilePath = join(resultDirectory, "results.zip");
 
+    // in case of restarted download delete possible artifact from previous download
+    await rm(zipFilePath, { force: true });
+
     const response = await fetch(repoTask.artifactUrl);
 
     let responseSize = parseInt(response.headers.get("content-length") || "0");

From 9dee2a132e348d2f6d6ff07baa81792655fda3b3 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 24 May 2023 10:37:22 +0100
Subject: [PATCH 046/119] Remove member variables that could be local variables

---
 .../ql-vscode/src/queries-panel/queries-module.ts   | 13 +++++--------
 .../ql-vscode/src/queries-panel/queries-panel.ts    | 13 ++++---------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts
index f681e1c81..d62812f46 100644
--- a/extensions/ql-vscode/src/queries-panel/queries-module.ts
+++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts
@@ -7,9 +7,6 @@ import { QueriesPanel } from "./queries-panel";
 import { QueryDiscovery } from "./query-discovery";
 
 export class QueriesModule extends DisposableObject {
-  private queriesPanel: QueriesPanel | undefined;
-  private queryDiscovery: QueryDiscovery | undefined;
-
   private constructor(readonly app: App) {
     super();
   }
@@ -22,12 +19,12 @@ export class QueriesModule extends DisposableObject {
     }
     void extLogger.log("Initializing queries panel.");
 
-    this.queryDiscovery = new QueryDiscovery(app, cliServer);
-    this.push(this.queryDiscovery);
-    this.queryDiscovery.refresh();
+    const queryDiscovery = new QueryDiscovery(app, cliServer);
+    this.push(queryDiscovery);
+    queryDiscovery.refresh();
 
-    this.queriesPanel = new QueriesPanel(this.queryDiscovery);
-    this.push(this.queriesPanel);
+    const queriesPanel = new QueriesPanel(queryDiscovery);
+    this.push(queriesPanel);
   }
 
   public static initialize(
diff --git a/extensions/ql-vscode/src/queries-panel/queries-panel.ts b/extensions/ql-vscode/src/queries-panel/queries-panel.ts
index 31b95bd19..5988509f8 100644
--- a/extensions/ql-vscode/src/queries-panel/queries-panel.ts
+++ b/extensions/ql-vscode/src/queries-panel/queries-panel.ts
@@ -1,22 +1,17 @@
 import * as vscode from "vscode";
 import { DisposableObject } from "../pure/disposable-object";
 import { QueryTreeDataProvider } from "./query-tree-data-provider";
-import { QueryTreeViewItem } from "./query-tree-view-item";
 import { QueryDiscovery } from "./query-discovery";
 
 export class QueriesPanel extends DisposableObject {
-  private readonly dataProvider: QueryTreeDataProvider;
-  private readonly treeView: vscode.TreeView<QueryTreeViewItem>;
-
   public constructor(queryDiscovery: QueryDiscovery) {
     super();
 
-    this.dataProvider = new QueryTreeDataProvider(queryDiscovery);
+    const dataProvider = new QueryTreeDataProvider(queryDiscovery);
 
-    this.treeView = vscode.window.createTreeView("codeQLQueries", {
-      treeDataProvider: this.dataProvider,
+    const treeView = vscode.window.createTreeView("codeQLQueries", {
+      treeDataProvider: dataProvider,
     });
-
-    this.push(this.treeView);
+    this.push(treeView);
   }
 }

From b9ce91cff9ccdc74d0968360454219ab98f23405 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 24 May 2023 11:22:31 +0100
Subject: [PATCH 047/119] Add a QueryDiscoverer interface to make testing
 QueryTreeDataProvider easier

---
 .../ql-vscode/src/queries-panel/query-discovery.ts   |  6 +++++-
 .../src/queries-panel/query-tree-data-provider.ts    | 12 ++++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index b05866bfc..00322d634 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -7,6 +7,7 @@ import { App } from "../common/app";
 import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
 import { getOnDiskWorkspaceFoldersObjects } from "../helpers";
 import { AppEventEmitter } from "../common/events";
+import { QueryDiscoverer } from "./query-tree-data-provider";
 
 /**
  * The results of discovering queries.
@@ -28,7 +29,10 @@ interface QueryDiscoveryResults {
 /**
  * Discovers all query files contained in the QL packs in a given workspace folder.
  */
-export class QueryDiscovery extends Discovery<QueryDiscoveryResults> {
+export class QueryDiscovery
+  extends Discovery<QueryDiscoveryResults>
+  implements QueryDiscoverer
+{
   private results: QueryDiscoveryResults | undefined;
 
   private readonly onDidChangeQueriesEmitter: AppEventEmitter<void>;
diff --git a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
index 0b92b4338..0810ded8a 100644
--- a/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts
@@ -1,9 +1,13 @@
 import { Event, EventEmitter, TreeDataProvider, TreeItem } from "vscode";
 import { QueryTreeViewItem } from "./query-tree-view-item";
 import { DisposableObject } from "../pure/disposable-object";
-import { QueryDiscovery } from "./query-discovery";
 import { FileTreeNode } from "../common/file-tree-nodes";
 
+export interface QueryDiscoverer {
+  readonly queries: FileTreeNode[] | undefined;
+  readonly onDidChangeQueries: Event<void>;
+}
+
 export class QueryTreeDataProvider
   extends DisposableObject
   implements TreeDataProvider<QueryTreeViewItem>
@@ -14,10 +18,10 @@ export class QueryTreeDataProvider
     new EventEmitter<void>(),
   );
 
-  public constructor(private readonly queryDiscovery: QueryDiscovery) {
+  public constructor(private readonly queryDiscoverer: QueryDiscoverer) {
     super();
 
-    queryDiscovery.onDidChangeQueries(() => {
+    queryDiscoverer.onDidChangeQueries(() => {
       this.queryTreeItems = this.createTree();
       this.onDidChangeTreeDataEmitter.fire();
     });
@@ -30,7 +34,7 @@ export class QueryTreeDataProvider
   }
 
   private createTree(): QueryTreeViewItem[] {
-    return (this.queryDiscovery.queries || []).map(
+    return (this.queryDiscoverer.queries || []).map(
       this.convertFileTreeNode.bind(this),
     );
   }

From 27a88032f79aa67c9377643c2241f5180d7cbb11 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 24 May 2023 11:41:41 +0100
Subject: [PATCH 048/119] Add QueryTreeDataProvider tests

---
 .../query-tree-data-provider.test.ts          | 93 +++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts

diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts
new file mode 100644
index 000000000..080528689
--- /dev/null
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-tree-data-provider.test.ts
@@ -0,0 +1,93 @@
+import { EventEmitter } from "vscode";
+import {
+  FileTreeDirectory,
+  FileTreeLeaf,
+} from "../../../../src/common/file-tree-nodes";
+import {
+  QueryDiscoverer,
+  QueryTreeDataProvider,
+} from "../../../../src/queries-panel/query-tree-data-provider";
+
+describe("QueryTreeDataProvider", () => {
+  describe("getChildren", () => {
+    it("returns no children when queries is undefined", async () => {
+      const dataProvider = new QueryTreeDataProvider({
+        queries: undefined,
+        onDidChangeQueries: jest.fn(),
+      });
+
+      expect(dataProvider.getChildren()).toEqual([]);
+    });
+
+    it("returns no children when there are no queries", async () => {
+      const dataProvider = new QueryTreeDataProvider({
+        queries: [],
+        onDidChangeQueries: jest.fn(),
+      });
+
+      expect(dataProvider.getChildren()).toEqual([]);
+    });
+
+    it("converts FileTreeNode to QueryTreeViewItem", async () => {
+      const dataProvider = new QueryTreeDataProvider({
+        queries: [
+          new FileTreeDirectory("dir1", "dir1", [
+            new FileTreeDirectory("dir1/dir2", "dir2", [
+              new FileTreeLeaf("dir1/dir2/file1", "file1"),
+              new FileTreeLeaf("dir1/dir2/file1", "file2"),
+            ]),
+          ]),
+          new FileTreeDirectory("dir3", "dir3", [
+            new FileTreeLeaf("dir3/file3", "file3"),
+          ]),
+        ],
+        onDidChangeQueries: jest.fn(),
+      });
+
+      expect(dataProvider.getChildren().length).toEqual(2);
+
+      expect(dataProvider.getChildren()[0].label).toEqual("dir1");
+      expect(dataProvider.getChildren()[0].children.length).toEqual(1);
+      expect(dataProvider.getChildren()[0].children[0].label).toEqual("dir2");
+      expect(dataProvider.getChildren()[0].children[0].children.length).toEqual(
+        2,
+      );
+      expect(
+        dataProvider.getChildren()[0].children[0].children[0].label,
+      ).toEqual("file1");
+      expect(
+        dataProvider.getChildren()[0].children[0].children[1].label,
+      ).toEqual("file2");
+
+      expect(dataProvider.getChildren()[1].label).toEqual("dir3");
+      expect(dataProvider.getChildren()[1].children.length).toEqual(1);
+      expect(dataProvider.getChildren()[1].children[0].label).toEqual("file3");
+    });
+  });
+
+  describe("onDidChangeQueries", () => {
+    it("should update tree when the queries change", async () => {
+      const onDidChangeQueriesEmitter = new EventEmitter<void>();
+      const queryDiscoverer: QueryDiscoverer = {
+        queries: [
+          new FileTreeDirectory("dir1", "dir1", [
+            new FileTreeLeaf("dir1/file1", "file1"),
+          ]),
+        ],
+        onDidChangeQueries: onDidChangeQueriesEmitter.event,
+      };
+
+      const dataProvider = new QueryTreeDataProvider(queryDiscoverer);
+      expect(dataProvider.getChildren().length).toEqual(1);
+
+      queryDiscoverer.queries?.push(
+        new FileTreeDirectory("dir2", "dir2", [
+          new FileTreeLeaf("dir2/file2", "file2"),
+        ]),
+      );
+      onDidChangeQueriesEmitter.fire();
+
+      expect(dataProvider.getChildren().length).toEqual(2);
+    });
+  });
+});

From d07b7c8c05ee8fa7508a7af705d755abd7dcb9eb Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 24 May 2023 10:57:13 +0000
Subject: [PATCH 049/119] Add db manager test

---
 .../unit-tests/databases/db-manager.test.ts   | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
index fa49ce2f9..92733aed8 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
@@ -88,6 +88,33 @@ describe("db manager", () => {
         ).toEqual("owner2/repo2");
       });
 
+      it("should add new remote repos to a user defined list", async () => {
+        const dbConfig: DbConfig = createDbConfig({
+          remoteLists: [
+            {
+              name: "my-list-1",
+              repositories: ["owner1/repo1"],
+            },
+          ],
+        });
+
+        await saveDbConfig(dbConfig);
+
+        await dbManager.addNewRemoteReposToList(["owner2/repo2"], "my-list-1");
+
+        const dbConfigFileContents = await readDbConfigDirectly();
+        expect(
+          dbConfigFileContents.databases.variantAnalysis.repositoryLists.length,
+        ).toBe(1);
+
+        expect(
+          dbConfigFileContents.databases.variantAnalysis.repositoryLists[0],
+        ).toEqual({
+          name: "my-list-1",
+          repositories: ["owner1/repo1", "owner2/repo2"],
+        });
+      });
+
       it("should add a new remote repo to a user defined list", async () => {
         const dbConfig: DbConfig = createDbConfig({
           remoteLists: [

From 2b4e302e2916cbc3002eadae3aba16c39a834460 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 24 May 2023 10:58:13 +0000
Subject: [PATCH 050/119] Surface truncated repos when a list hits 1000 items

---
 .../src/databases/config/db-config-store.ts   | 27 +++++++------
 .../ql-vscode/src/databases/db-manager.ts     |  8 ++--
 .../ql-vscode/src/databases/ui/db-panel.ts    | 28 ++++++++++++-
 .../databases/config/db-config-store.test.ts  | 36 ++++++++++++++++-
 .../unit-tests/databases/db-manager.test.ts   | 40 +++++++++++++++++++
 5 files changed, 119 insertions(+), 20 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts
index f25bf81f8..ef62c61b3 100644
--- a/extensions/ql-vscode/src/databases/config/db-config-store.ts
+++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts
@@ -148,7 +148,7 @@ export class DbConfigStore extends DisposableObject {
   public async addRemoteReposToList(
     repoNwoList: string[],
     parentList: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repos if config is not loaded");
     }
@@ -161,23 +161,22 @@ export class DbConfigStore extends DisposableObject {
       throw Error(`Cannot find parent list '${parentList}'`);
     }
 
-    const newRepositoriesList = new Set([
-      ...new Set(parent.repositories),
-      ...new Set(repoNwoList),
-    ]);
+    // Remove duplicates from the list of repositories.
+    const newRepositoriesList = [
+      ...new Set([...new Set(parent.repositories), ...new Set(repoNwoList)]),
+    ];
+
+    parent.repositories = newRepositoriesList.slice(0, 1000);
+    const truncatedRepositories = newRepositoriesList.slice(1000);
 
-    if (newRepositoriesList.size > 1000) {
-      parent.repositories = [...Array.from(newRepositoriesList).slice(0, 1000)];
-    } else {
-      parent.repositories = [...newRepositoriesList];
-    }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repo if config is not loaded");
     }
@@ -192,6 +191,7 @@ export class DbConfigStore extends DisposableObject {
       );
     }
 
+    const truncatedRepositories = [];
     const config = cloneDbConfig(this.config);
     if (parentList) {
       const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -200,12 +200,15 @@ export class DbConfigStore extends DisposableObject {
       if (!parent) {
         throw Error(`Cannot find parent list '${parentList}'`);
       } else {
-        parent.repositories.push(repoNwo);
+        const newRepositories = [...parent.repositories, repoNwo];
+        parent.repositories = newRepositories.slice(0, 1000);
+        truncatedRepositories.push(...newRepositories.slice(1000));
       }
     } else {
       config.databases.variantAnalysis.repositories.push(repoNwo);
     }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteOwner(owner: string): Promise<void> {
diff --git a/extensions/ql-vscode/src/databases/db-manager.ts b/extensions/ql-vscode/src/databases/db-manager.ts
index b66d5d5de..5c1a156d9 100644
--- a/extensions/ql-vscode/src/databases/db-manager.ts
+++ b/extensions/ql-vscode/src/databases/db-manager.ts
@@ -96,15 +96,15 @@ export class DbManager {
   public async addNewRemoteRepo(
     nwo: string,
     parentList?: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
   }
 
   public async addNewRemoteReposToList(
     nwoList: string[],
     parentList: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
   }
 
   public async addNewRemoteOwner(owner: string): Promise<void> {
diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index 0d77e13c8..61daf987c 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -179,7 +179,14 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await this.dbManager.addNewRemoteRepo(nwo, parentList);
+    const truncatedRepositories = await this.dbManager.addNewRemoteRepo(
+      nwo,
+      parentList,
+    );
+
+    if (parentList) {
+      this.truncatedReposNote(truncatedRepositories, parentList);
+    }
   }
 
   private async addNewRemoteOwner(): Promise<void> {
@@ -373,10 +380,27 @@ export class DbPanel extends DisposableObject {
       `${codeSearchQuery} language:${codeSearchLanguage.language}`,
     );
 
-    await this.dbManager.addNewRemoteReposToList(
+    const truncatedRepositories = await this.dbManager.addNewRemoteReposToList(
       repositories,
       treeViewItem.dbItem.listName,
     );
+    this.truncatedReposNote(
+      truncatedRepositories,
+      treeViewItem.dbItem.listName,
+    );
+  }
+
+  private truncatedReposNote(
+    truncatedRepositories: string[],
+    listName: string,
+  ) {
+    if (truncatedRepositories.length > 0) {
+      void showAndLogErrorMessage(
+        `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join(
+          ", ",
+        )}`,
+      );
+    }
   }
 
   private async onDidCollapseElement(
diff --git a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
index 3bb918734..6673fb07a 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
@@ -282,7 +282,7 @@ describe("db config store", () => {
       configStore.dispose();
     });
 
-    it("should add no more than 1000 repositories to a list", async () => {
+    it("should add no more than 1000 repositories to a remote list using #addRemoteReposToList", async () => {
       // Initial set up
       const dbConfig = createDbConfig({
         remoteLists: [
@@ -296,7 +296,7 @@ describe("db config store", () => {
       const configStore = await initializeConfig(dbConfig, configPath, app);
 
       // Add
-      await configStore.addRemoteReposToList(
+      const reponse = await configStore.addRemoteReposToList(
         [...Array(1001).keys()].map((i) => `owner/db${i}`),
         "list1",
       );
@@ -311,6 +311,38 @@ describe("db config store", () => {
       expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
         1000,
       );
+      expect(reponse).toEqual(["owner/db1000"]);
+
+      configStore.dispose();
+    });
+
+    it("should add no more than 1000 repositories to a remote list using #addRemoteRepo", async () => {
+      // Initial set up
+      const dbConfig = createDbConfig({
+        remoteLists: [
+          {
+            name: "list1",
+            repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
+          },
+        ],
+      });
+
+      const configStore = await initializeConfig(dbConfig, configPath, app);
+
+      // Add
+      const reponse = await configStore.addRemoteRepo("owner/db1000", "list1");
+
+      // Read the config file
+      const updatedDbConfig = (await readJSON(configPath)) as DbConfig;
+
+      // Check that the config file has been updated
+      const updatedRemoteDbs = updatedDbConfig.databases.variantAnalysis;
+      expect(updatedRemoteDbs.repositories).toHaveLength(0);
+      expect(updatedRemoteDbs.repositoryLists).toHaveLength(1);
+      expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
+        1000,
+      );
+      expect(reponse).toEqual(["owner/db1000"]);
 
       configStore.dispose();
     });
diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
index 92733aed8..ff0bc8ba6 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
@@ -115,6 +115,46 @@ describe("db manager", () => {
         });
       });
 
+      it("should return truncated repos when adding to a user defined list using #addNewRemoteReposToList", async () => {
+        const dbConfig: DbConfig = createDbConfig({
+          remoteLists: [
+            {
+              name: "my-list-1",
+              repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
+            },
+          ],
+        });
+
+        await saveDbConfig(dbConfig);
+
+        const response = await dbManager.addNewRemoteReposToList(
+          ["owner2/repo2"],
+          "my-list-1",
+        );
+
+        expect(response).toEqual(["owner2/repo2"]);
+      });
+
+      it("should return truncated repos when adding to a user defined list using #addNewRemoteRepo", async () => {
+        const dbConfig: DbConfig = createDbConfig({
+          remoteLists: [
+            {
+              name: "my-list-1",
+              repositories: [...Array(1000).keys()].map((i) => `owner/db${i}`),
+            },
+          ],
+        });
+
+        await saveDbConfig(dbConfig);
+
+        const response = await dbManager.addNewRemoteRepo(
+          "owner2/repo2",
+          "my-list-1",
+        );
+
+        expect(response).toEqual(["owner2/repo2"]);
+      });
+
       it("should add a new remote repo to a user defined list", async () => {
         const dbConfig: DbConfig = createDbConfig({
           remoteLists: [

From 6331cddbfd4608c9e3d4831e5d0dbb9574be694d Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 16:51:38 +0200
Subject: [PATCH 051/119] Move DatabaseOptions to separate file

---
 .../ql-vscode/src/databases/local-databases.ts  | 17 ++++-------------
 .../local-databases/database-options.ts         | 12 ++++++++++++
 2 files changed, 16 insertions(+), 13 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-options.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index 8bc50c70f..400772b2a 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -35,6 +35,10 @@ import { QlPackGenerator } from "../qlpack-generator";
 import { QueryLanguage } from "../common/query-language";
 import { App } from "../common/app";
 import { existsSync } from "fs";
+import {
+  DatabaseOptions,
+  FullDatabaseOptions,
+} from "./local-databases/database-options";
 
 /**
  * databases.ts
@@ -58,19 +62,6 @@ const CURRENT_DB = "currentDatabase";
  */
 const DB_LIST = "databaseList";
 
-export interface DatabaseOptions {
-  displayName?: string;
-  ignoreSourceArchive?: boolean;
-  dateAdded?: number | undefined;
-  language?: string;
-}
-
-export interface FullDatabaseOptions extends DatabaseOptions {
-  ignoreSourceArchive: boolean;
-  dateAdded: number | undefined;
-  language: string | undefined;
-}
-
 interface PersistedDatabaseItem {
   uri: string;
   options?: DatabaseOptions;
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-options.ts b/extensions/ql-vscode/src/databases/local-databases/database-options.ts
new file mode 100644
index 000000000..b8990e759
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-options.ts
@@ -0,0 +1,12 @@
+export interface DatabaseOptions {
+  displayName?: string;
+  ignoreSourceArchive?: boolean;
+  dateAdded?: number | undefined;
+  language?: string;
+}
+
+export interface FullDatabaseOptions extends DatabaseOptions {
+  ignoreSourceArchive: boolean;
+  dateAdded: number | undefined;
+  language: string | undefined;
+}

From 4bb4627d30f09dcfe9fccaf118277a627207d132 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 16:52:53 +0200
Subject: [PATCH 052/119] Move DatabaseItem to separate file

---
 .../src/databases/local-databases.ts          | 96 ++-----------------
 .../local-databases/database-item.ts          | 91 ++++++++++++++++++
 2 files changed, 97 insertions(+), 90 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-item.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index 400772b2a..97ae3dd3e 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -35,10 +35,13 @@ import { QlPackGenerator } from "../qlpack-generator";
 import { QueryLanguage } from "../common/query-language";
 import { App } from "../common/app";
 import { existsSync } from "fs";
+import { FullDatabaseOptions } from "./local-databases/database-options";
 import {
-  DatabaseOptions,
-  FullDatabaseOptions,
-} from "./local-databases/database-options";
+  DatabaseItem,
+  PersistedDatabaseItem,
+} from "./local-databases/database-item";
+
+export { DatabaseItem } from "./local-databases/database-item";
 
 /**
  * databases.ts
@@ -62,11 +65,6 @@ const CURRENT_DB = "currentDatabase";
  */
 const DB_LIST = "databaseList";
 
-interface PersistedDatabaseItem {
-  uri: string;
-  options?: DatabaseOptions;
-}
-
 /**
  * The layout of the database.
  */
@@ -226,88 +224,6 @@ export class DatabaseResolver {
   }
 }
 
-/** An item in the list of available databases */
-export interface DatabaseItem {
-  /** The URI of the database */
-  readonly databaseUri: vscode.Uri;
-  /** The name of the database to be displayed in the UI */
-  name: string;
-
-  /** The primary language of the database or empty string if unknown */
-  readonly language: string;
-  /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
-  readonly sourceArchive: vscode.Uri | undefined;
-  /**
-   * The contents of the database.
-   * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`.
-   */
-  readonly contents: DatabaseContents | undefined;
-
-  /**
-   * The date this database was added as a unix timestamp. Or undefined if we don't know.
-   */
-  readonly dateAdded: number | undefined;
-
-  /** If the database is invalid, describes why. */
-  readonly error: Error | undefined;
-  /**
-   * Resolves the contents of the database.
-   *
-   * @remarks
-   * The contents include the database directory, source archive, and metadata about the database.
-   * If the database is invalid, `this.error` is updated with the error object that describes why
-   * the database is invalid. This error is also thrown.
-   */
-  refresh(): Promise<void>;
-  /**
-   * Resolves a filename to its URI in the source archive.
-   *
-   * @param file Filename within the source archive. May be `undefined` to return a dummy file path.
-   */
-  resolveSourceFile(file: string | undefined): vscode.Uri;
-
-  /**
-   * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file.
-   */
-  hasMetadataFile(): Promise<boolean>;
-
-  /**
-   * Returns `sourceLocationPrefix` of exported database.
-   */
-  getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
-
-  /**
-   * Returns dataset folder of exported database.
-   */
-  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
-
-  /**
-   * Returns the root uri of the virtual filesystem for this database's source archive,
-   * as displayed in the filesystem explorer.
-   */
-  getSourceArchiveExplorerUri(): vscode.Uri;
-
-  /**
-   * Holds if `uri` belongs to this database's source archive.
-   */
-  belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
-
-  /**
-   * Whether the database may be affected by test execution for the given path.
-   */
-  isAffectedByTest(testPath: string): Promise<boolean>;
-
-  /**
-   * Gets the state of this database, to be persisted in the workspace state.
-   */
-  getPersistedState(): PersistedDatabaseItem;
-
-  /**
-   * Verifies that this database item has a zipped source folder. Returns an error message if it does not.
-   */
-  verifyZippedSources(): string | undefined;
-}
-
 export enum DatabaseEventKind {
   Add = "Add",
   Remove = "Remove",
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
new file mode 100644
index 000000000..494ae3744
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
@@ -0,0 +1,91 @@
+import vscode from "vscode";
+import * as cli from "../../codeql-cli/cli";
+import { DatabaseContents } from "../local-databases";
+import { DatabaseOptions } from "./database-options";
+
+/** An item in the list of available databases */
+export interface DatabaseItem {
+  /** The URI of the database */
+  readonly databaseUri: vscode.Uri;
+  /** The name of the database to be displayed in the UI */
+  name: string;
+
+  /** The primary language of the database or empty string if unknown */
+  readonly language: string;
+  /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
+  readonly sourceArchive: vscode.Uri | undefined;
+  /**
+   * The contents of the database.
+   * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`.
+   */
+  readonly contents: DatabaseContents | undefined;
+
+  /**
+   * The date this database was added as a unix timestamp. Or undefined if we don't know.
+   */
+  readonly dateAdded: number | undefined;
+
+  /** If the database is invalid, describes why. */
+  readonly error: Error | undefined;
+  /**
+   * Resolves the contents of the database.
+   *
+   * @remarks
+   * The contents include the database directory, source archive, and metadata about the database.
+   * If the database is invalid, `this.error` is updated with the error object that describes why
+   * the database is invalid. This error is also thrown.
+   */
+  refresh(): Promise<void>;
+  /**
+   * Resolves a filename to its URI in the source archive.
+   *
+   * @param file Filename within the source archive. May be `undefined` to return a dummy file path.
+   */
+  resolveSourceFile(file: string | undefined): vscode.Uri;
+
+  /**
+   * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file.
+   */
+  hasMetadataFile(): Promise<boolean>;
+
+  /**
+   * Returns `sourceLocationPrefix` of exported database.
+   */
+  getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
+
+  /**
+   * Returns dataset folder of exported database.
+   */
+  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
+
+  /**
+   * Returns the root uri of the virtual filesystem for this database's source archive,
+   * as displayed in the filesystem explorer.
+   */
+  getSourceArchiveExplorerUri(): vscode.Uri;
+
+  /**
+   * Holds if `uri` belongs to this database's source archive.
+   */
+  belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
+
+  /**
+   * Whether the database may be affected by test execution for the given path.
+   */
+  isAffectedByTest(testPath: string): Promise<boolean>;
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  getPersistedState(): PersistedDatabaseItem;
+
+  /**
+   * Verifies that this database item has a zipped source folder. Returns an error message if it does not.
+   */
+  verifyZippedSources(): string | undefined;
+}
+
+export interface PersistedDatabaseItem {
+  uri: string;
+  options?: DatabaseOptions;
+}

From d02e53fbd20a367a386e37d8320abfa0c6c1abef Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 16:53:59 +0200
Subject: [PATCH 053/119] Move DatabaseItemImpl to separate file

---
 .../src/databases/local-databases.ts          | 241 +----------------
 .../local-databases/database-item-impl.ts     | 247 ++++++++++++++++++
 2 files changed, 252 insertions(+), 236 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index 97ae3dd3e..c52f5ae0d 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -1,30 +1,24 @@
-import { pathExists, stat, remove } from "fs-extra";
+import { pathExists, remove } from "fs-extra";
 import { glob } from "glob";
-import { join, basename, resolve, relative, dirname, extname } from "path";
+import { join, basename, resolve, dirname, extname } from "path";
 import * as vscode from "vscode";
 import * as cli from "../codeql-cli/cli";
 import { ExtensionContext } from "vscode";
 import {
   showAndLogWarningMessage,
   showAndLogInformationMessage,
-  isLikelyDatabaseRoot,
   showAndLogExceptionWithTelemetry,
   isFolderAlreadyInWorkspace,
   getFirstWorkspaceFolder,
   showNeverAskAgainDialog,
 } from "../helpers";
 import { ProgressCallback, withProgress } from "../common/vscode/progress";
-import {
-  zipArchiveScheme,
-  encodeArchiveBasePath,
-  decodeSourceArchiveUri,
-  encodeSourceArchiveUri,
-} from "../common/vscode/archive-filesystem-provider";
+import { encodeArchiveBasePath } from "../common/vscode/archive-filesystem-provider";
 import { DisposableObject } from "../pure/disposable-object";
 import { Logger, extLogger } from "../common";
 import { asError, getErrorMessage } from "../pure/helpers-pure";
 import { QueryRunner } from "../query-server";
-import { containsPath, pathsEqual } from "../pure/files";
+import { containsPath } from "../pure/files";
 import { redactableError } from "../pure/errors";
 import {
   getAutogenerateQlPacks,
@@ -40,6 +34,7 @@ import {
   DatabaseItem,
   PersistedDatabaseItem,
 } from "./local-databases/database-item";
+import { DatabaseItemImpl } from "./local-databases/database-item-impl";
 
 export { DatabaseItem } from "./local-databases/database-item";
 
@@ -242,232 +237,6 @@ export interface DatabaseChangedEvent {
   item: DatabaseItem | undefined;
 }
 
-// Exported for testing
-export class DatabaseItemImpl implements DatabaseItem {
-  private _error: Error | undefined = undefined;
-  private _contents: DatabaseContents | undefined;
-  /** A cache of database info */
-  private _dbinfo: cli.DbInfo | undefined;
-
-  public constructor(
-    public readonly databaseUri: vscode.Uri,
-    contents: DatabaseContents | undefined,
-    private options: FullDatabaseOptions,
-    private readonly onChanged: (event: DatabaseChangedEvent) => void,
-  ) {
-    this._contents = contents;
-  }
-
-  public get name(): string {
-    if (this.options.displayName) {
-      return this.options.displayName;
-    } else if (this._contents) {
-      return this._contents.name;
-    } else {
-      return basename(this.databaseUri.fsPath);
-    }
-  }
-
-  public set name(newName: string) {
-    this.options.displayName = newName;
-  }
-
-  public get sourceArchive(): vscode.Uri | undefined {
-    if (this.options.ignoreSourceArchive || this._contents === undefined) {
-      return undefined;
-    } else {
-      return this._contents.sourceArchiveUri;
-    }
-  }
-
-  public get contents(): DatabaseContents | undefined {
-    return this._contents;
-  }
-
-  public get dateAdded(): number | undefined {
-    return this.options.dateAdded;
-  }
-
-  public get error(): Error | undefined {
-    return this._error;
-  }
-
-  public async refresh(): Promise<void> {
-    try {
-      try {
-        this._contents = await DatabaseResolver.resolveDatabaseContents(
-          this.databaseUri,
-        );
-        this._error = undefined;
-      } catch (e) {
-        this._contents = undefined;
-        this._error = asError(e);
-        throw e;
-      }
-    } finally {
-      this.onChanged({
-        kind: DatabaseEventKind.Refresh,
-        item: this,
-      });
-    }
-  }
-
-  public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
-    const sourceArchive = this.sourceArchive;
-    const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
-    if (uri && uri.scheme !== "file") {
-      throw new Error(
-        `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`,
-      );
-    }
-    if (!sourceArchive) {
-      if (uri) {
-        return uri;
-      } else {
-        return this.databaseUri;
-      }
-    }
-
-    if (uri) {
-      const relativeFilePath = decodeURI(uri.path)
-        .replace(":", "_")
-        .replace(/^\/*/, "");
-      if (sourceArchive.scheme === zipArchiveScheme) {
-        const zipRef = decodeSourceArchiveUri(sourceArchive);
-        const pathWithinSourceArchive =
-          zipRef.pathWithinSourceArchive === "/"
-            ? relativeFilePath
-            : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`;
-        return encodeSourceArchiveUri({
-          pathWithinSourceArchive,
-          sourceArchiveZipPath: zipRef.sourceArchiveZipPath,
-        });
-      } else {
-        let newPath = sourceArchive.path;
-        if (!newPath.endsWith("/")) {
-          // Ensure a trailing slash.
-          newPath += "/";
-        }
-        newPath += relativeFilePath;
-
-        return sourceArchive.with({ path: newPath });
-      }
-    } else {
-      return sourceArchive;
-    }
-  }
-
-  /**
-   * Gets the state of this database, to be persisted in the workspace state.
-   */
-  public getPersistedState(): PersistedDatabaseItem {
-    return {
-      uri: this.databaseUri.toString(true),
-      options: this.options,
-    };
-  }
-
-  /**
-   * Holds if the database item refers to an exported snapshot
-   */
-  public async hasMetadataFile(): Promise<boolean> {
-    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
-  }
-
-  /**
-   * Returns information about a database.
-   */
-  private async getDbInfo(server: cli.CodeQLCliServer): Promise<cli.DbInfo> {
-    if (this._dbinfo === undefined) {
-      this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath);
-    }
-    return this._dbinfo;
-  }
-
-  /**
-   * Returns `sourceLocationPrefix` of database. Requires that the database
-   * has a `.dbinfo` file, which is the source of the prefix.
-   */
-  public async getSourceLocationPrefix(
-    server: cli.CodeQLCliServer,
-  ): Promise<string> {
-    const dbInfo = await this.getDbInfo(server);
-    return dbInfo.sourceLocationPrefix;
-  }
-
-  /**
-   * Returns path to dataset folder of database.
-   */
-  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
-    const dbInfo = await this.getDbInfo(server);
-    return dbInfo.datasetFolder;
-  }
-
-  public get language() {
-    return this.options.language || "";
-  }
-
-  /**
-   * Returns the root uri of the virtual filesystem for this database's source archive.
-   */
-  public getSourceArchiveExplorerUri(): vscode.Uri {
-    const sourceArchive = this.sourceArchive;
-    if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) {
-      throw new Error(this.verifyZippedSources());
-    }
-    return encodeArchiveBasePath(sourceArchive.fsPath);
-  }
-
-  public verifyZippedSources(): string | undefined {
-    const sourceArchive = this.sourceArchive;
-    if (sourceArchive === undefined) {
-      return `${this.name} has no source archive.`;
-    }
-
-    if (!sourceArchive.fsPath.endsWith(".zip")) {
-      return `${this.name} has a source folder that is unzipped.`;
-    }
-    return;
-  }
-
-  /**
-   * Holds if `uri` belongs to this database's source archive.
-   */
-  public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean {
-    if (this.sourceArchive === undefined) return false;
-    return (
-      uri.scheme === zipArchiveScheme &&
-      decodeSourceArchiveUri(uri).sourceArchiveZipPath ===
-        this.sourceArchive.fsPath
-    );
-  }
-
-  public async isAffectedByTest(testPath: string): Promise<boolean> {
-    const databasePath = this.databaseUri.fsPath;
-    if (!databasePath.endsWith(".testproj")) {
-      return false;
-    }
-    try {
-      const stats = await stat(testPath);
-      if (stats.isDirectory()) {
-        return !relative(testPath, databasePath).startsWith("..");
-      } else {
-        // database for /one/two/three/test.ql is at /one/two/three/three.testproj
-        const testdir = dirname(testPath);
-        const testdirbase = basename(testdir);
-        return pathsEqual(
-          databasePath,
-          join(testdir, `${testdirbase}.testproj`),
-          process.platform,
-        );
-      }
-    } catch {
-      // No information available for test path - assume database is unaffected.
-      return false;
-    }
-  }
-}
-
 /**
  * A promise that resolves to an event's result value when the event
  * `event` fires. If waiting for the event takes too long (by default
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
new file mode 100644
index 000000000..460db6940
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -0,0 +1,247 @@
+// Exported for testing
+import * as cli from "../../codeql-cli/cli";
+import vscode from "vscode";
+import { FullDatabaseOptions } from "./database-options";
+import { basename, dirname, join, relative } from "path";
+import { asError } from "../../pure/helpers-pure";
+import {
+  decodeSourceArchiveUri,
+  encodeArchiveBasePath,
+  encodeSourceArchiveUri,
+  zipArchiveScheme,
+} from "../../common/vscode/archive-filesystem-provider";
+import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
+import { isLikelyDatabaseRoot } from "../../helpers";
+import { stat } from "fs-extra";
+import { pathsEqual } from "../../pure/files";
+import {
+  DatabaseChangedEvent,
+  DatabaseContents,
+  DatabaseEventKind,
+  DatabaseResolver,
+} from "../local-databases";
+
+export class DatabaseItemImpl implements DatabaseItem {
+  private _error: Error | undefined = undefined;
+  private _contents: DatabaseContents | undefined;
+  /** A cache of database info */
+  private _dbinfo: cli.DbInfo | undefined;
+
+  public constructor(
+    public readonly databaseUri: vscode.Uri,
+    contents: DatabaseContents | undefined,
+    private options: FullDatabaseOptions,
+    private readonly onChanged: (event: DatabaseChangedEvent) => void,
+  ) {
+    this._contents = contents;
+  }
+
+  public get name(): string {
+    if (this.options.displayName) {
+      return this.options.displayName;
+    } else if (this._contents) {
+      return this._contents.name;
+    } else {
+      return basename(this.databaseUri.fsPath);
+    }
+  }
+
+  public set name(newName: string) {
+    this.options.displayName = newName;
+  }
+
+  public get sourceArchive(): vscode.Uri | undefined {
+    if (this.options.ignoreSourceArchive || this._contents === undefined) {
+      return undefined;
+    } else {
+      return this._contents.sourceArchiveUri;
+    }
+  }
+
+  public get contents(): DatabaseContents | undefined {
+    return this._contents;
+  }
+
+  public get dateAdded(): number | undefined {
+    return this.options.dateAdded;
+  }
+
+  public get error(): Error | undefined {
+    return this._error;
+  }
+
+  public async refresh(): Promise<void> {
+    try {
+      try {
+        this._contents = await DatabaseResolver.resolveDatabaseContents(
+          this.databaseUri,
+        );
+        this._error = undefined;
+      } catch (e) {
+        this._contents = undefined;
+        this._error = asError(e);
+        throw e;
+      }
+    } finally {
+      this.onChanged({
+        kind: DatabaseEventKind.Refresh,
+        item: this,
+      });
+    }
+  }
+
+  public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
+    const sourceArchive = this.sourceArchive;
+    const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
+    if (uri && uri.scheme !== "file") {
+      throw new Error(
+        `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`,
+      );
+    }
+    if (!sourceArchive) {
+      if (uri) {
+        return uri;
+      } else {
+        return this.databaseUri;
+      }
+    }
+
+    if (uri) {
+      const relativeFilePath = decodeURI(uri.path)
+        .replace(":", "_")
+        .replace(/^\/*/, "");
+      if (sourceArchive.scheme === zipArchiveScheme) {
+        const zipRef = decodeSourceArchiveUri(sourceArchive);
+        const pathWithinSourceArchive =
+          zipRef.pathWithinSourceArchive === "/"
+            ? relativeFilePath
+            : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`;
+        return encodeSourceArchiveUri({
+          pathWithinSourceArchive,
+          sourceArchiveZipPath: zipRef.sourceArchiveZipPath,
+        });
+      } else {
+        let newPath = sourceArchive.path;
+        if (!newPath.endsWith("/")) {
+          // Ensure a trailing slash.
+          newPath += "/";
+        }
+        newPath += relativeFilePath;
+
+        return sourceArchive.with({ path: newPath });
+      }
+    } else {
+      return sourceArchive;
+    }
+  }
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  public getPersistedState(): PersistedDatabaseItem {
+    return {
+      uri: this.databaseUri.toString(true),
+      options: this.options,
+    };
+  }
+
+  /**
+   * Holds if the database item refers to an exported snapshot
+   */
+  public async hasMetadataFile(): Promise<boolean> {
+    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
+  }
+
+  /**
+   * Returns information about a database.
+   */
+  private async getDbInfo(server: cli.CodeQLCliServer): Promise<cli.DbInfo> {
+    if (this._dbinfo === undefined) {
+      this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath);
+    }
+    return this._dbinfo;
+  }
+
+  /**
+   * Returns `sourceLocationPrefix` of database. Requires that the database
+   * has a `.dbinfo` file, which is the source of the prefix.
+   */
+  public async getSourceLocationPrefix(
+    server: cli.CodeQLCliServer,
+  ): Promise<string> {
+    const dbInfo = await this.getDbInfo(server);
+    return dbInfo.sourceLocationPrefix;
+  }
+
+  /**
+   * Returns path to dataset folder of database.
+   */
+  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
+    const dbInfo = await this.getDbInfo(server);
+    return dbInfo.datasetFolder;
+  }
+
+  public get language() {
+    return this.options.language || "";
+  }
+
+  /**
+   * Returns the root uri of the virtual filesystem for this database's source archive.
+   */
+  public getSourceArchiveExplorerUri(): vscode.Uri {
+    const sourceArchive = this.sourceArchive;
+    if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) {
+      throw new Error(this.verifyZippedSources());
+    }
+    return encodeArchiveBasePath(sourceArchive.fsPath);
+  }
+
+  public verifyZippedSources(): string | undefined {
+    const sourceArchive = this.sourceArchive;
+    if (sourceArchive === undefined) {
+      return `${this.name} has no source archive.`;
+    }
+
+    if (!sourceArchive.fsPath.endsWith(".zip")) {
+      return `${this.name} has a source folder that is unzipped.`;
+    }
+    return;
+  }
+
+  /**
+   * Holds if `uri` belongs to this database's source archive.
+   */
+  public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean {
+    if (this.sourceArchive === undefined) return false;
+    return (
+      uri.scheme === zipArchiveScheme &&
+      decodeSourceArchiveUri(uri).sourceArchiveZipPath ===
+        this.sourceArchive.fsPath
+    );
+  }
+
+  public async isAffectedByTest(testPath: string): Promise<boolean> {
+    const databasePath = this.databaseUri.fsPath;
+    if (!databasePath.endsWith(".testproj")) {
+      return false;
+    }
+    try {
+      const stats = await stat(testPath);
+      if (stats.isDirectory()) {
+        return !relative(testPath, databasePath).startsWith("..");
+      } else {
+        // database for /one/two/three/test.ql is at /one/two/three/three.testproj
+        const testdir = dirname(testPath);
+        const testdirbase = basename(testdir);
+        return pathsEqual(
+          databasePath,
+          join(testdir, `${testdirbase}.testproj`),
+          process.platform,
+        );
+      }
+    } catch {
+      // No information available for test path - assume database is unaffected.
+      return false;
+    }
+  }
+}

From 67983c64caa381555c016999ce72902437f28991 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 16:55:31 +0200
Subject: [PATCH 054/119] Move DatabaseContents to separate file

---
 .../src/databases/local-databases.ts          | 35 ++++---------------
 .../local-databases/database-contents.ts      | 30 ++++++++++++++++
 .../local-databases/database-item-impl.ts     |  2 +-
 .../local-databases/database-item.ts          |  2 +-
 4 files changed, 38 insertions(+), 31 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-contents.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index c52f5ae0d..abc5f1e61 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -35,7 +35,13 @@ import {
   PersistedDatabaseItem,
 } from "./local-databases/database-item";
 import { DatabaseItemImpl } from "./local-databases/database-item-impl";
+import {
+  DatabaseContents,
+  DatabaseContentsWithDbScheme,
+  DatabaseKind,
+} from "./local-databases/database-contents";
 
+export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents";
 export { DatabaseItem } from "./local-databases/database-item";
 
 /**
@@ -60,35 +66,6 @@ const CURRENT_DB = "currentDatabase";
  */
 const DB_LIST = "databaseList";
 
-/**
- * The layout of the database.
- */
-export enum DatabaseKind {
-  /** A CodeQL database */
-  Database,
-  /** A raw QL dataset */
-  RawDataset,
-}
-
-export interface DatabaseContents {
-  /** The layout of the database */
-  kind: DatabaseKind;
-  /**
-   * The name of the database.
-   */
-  name: string;
-  /** The URI of the QL dataset within the database. */
-  datasetUri: vscode.Uri;
-  /** The URI of the source archive within the database, if one exists. */
-  sourceArchiveUri?: vscode.Uri;
-  /** The URI of the CodeQL database scheme within the database, if exactly one exists. */
-  dbSchemeUri?: vscode.Uri;
-}
-
-export interface DatabaseContentsWithDbScheme extends DatabaseContents {
-  dbSchemeUri: vscode.Uri; // Always present
-}
-
 /**
  * An error thrown when we cannot find a valid database in a putative
  * database directory.
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-contents.ts b/extensions/ql-vscode/src/databases/local-databases/database-contents.ts
new file mode 100644
index 000000000..ce9f5d760
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-contents.ts
@@ -0,0 +1,30 @@
+import vscode from "vscode";
+
+/**
+ * The layout of the database.
+ */
+export enum DatabaseKind {
+  /** A CodeQL database */
+  Database,
+  /** A raw QL dataset */
+  RawDataset,
+}
+
+export interface DatabaseContents {
+  /** The layout of the database */
+  kind: DatabaseKind;
+  /**
+   * The name of the database.
+   */
+  name: string;
+  /** The URI of the QL dataset within the database. */
+  datasetUri: vscode.Uri;
+  /** The URI of the source archive within the database, if one exists. */
+  sourceArchiveUri?: vscode.Uri;
+  /** The URI of the CodeQL database scheme within the database, if exactly one exists. */
+  dbSchemeUri?: vscode.Uri;
+}
+
+export interface DatabaseContentsWithDbScheme extends DatabaseContents {
+  dbSchemeUri: vscode.Uri; // Always present
+}
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
index 460db6940..7e9fc22c9 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -16,10 +16,10 @@ import { stat } from "fs-extra";
 import { pathsEqual } from "../../pure/files";
 import {
   DatabaseChangedEvent,
-  DatabaseContents,
   DatabaseEventKind,
   DatabaseResolver,
 } from "../local-databases";
+import { DatabaseContents } from "./database-contents";
 
 export class DatabaseItemImpl implements DatabaseItem {
   private _error: Error | undefined = undefined;
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
index 494ae3744..0da295afb 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
@@ -1,6 +1,6 @@
 import vscode from "vscode";
 import * as cli from "../../codeql-cli/cli";
-import { DatabaseContents } from "../local-databases";
+import { DatabaseContents } from "./database-contents";
 import { DatabaseOptions } from "./database-options";
 
 /** An item in the list of available databases */

From 7888d210c48861c6f73d92d1b96e61051e74bbf3 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 16:58:24 +0200
Subject: [PATCH 055/119] Move DatabaseResolver to separate file

---
 .../src/databases/local-databases.ts          | 145 +-----------------
 .../local-databases/database-item-impl.ts     |   7 +-
 .../local-databases/database-resolver.ts      | 144 +++++++++++++++++
 3 files changed, 150 insertions(+), 146 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-resolver.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index abc5f1e61..d26849b8a 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -1,19 +1,15 @@
-import { pathExists, remove } from "fs-extra";
-import { glob } from "glob";
-import { join, basename, resolve, dirname, extname } from "path";
+import { remove } from "fs-extra";
+import { join, dirname, extname } from "path";
 import * as vscode from "vscode";
 import * as cli from "../codeql-cli/cli";
 import { ExtensionContext } from "vscode";
 import {
-  showAndLogWarningMessage,
-  showAndLogInformationMessage,
   showAndLogExceptionWithTelemetry,
   isFolderAlreadyInWorkspace,
   getFirstWorkspaceFolder,
   showNeverAskAgainDialog,
 } from "../helpers";
 import { ProgressCallback, withProgress } from "../common/vscode/progress";
-import { encodeArchiveBasePath } from "../common/vscode/archive-filesystem-provider";
 import { DisposableObject } from "../pure/disposable-object";
 import { Logger, extLogger } from "../common";
 import { asError, getErrorMessage } from "../pure/helpers-pure";
@@ -35,14 +31,11 @@ import {
   PersistedDatabaseItem,
 } from "./local-databases/database-item";
 import { DatabaseItemImpl } from "./local-databases/database-item-impl";
-import {
-  DatabaseContents,
-  DatabaseContentsWithDbScheme,
-  DatabaseKind,
-} from "./local-databases/database-contents";
+import { DatabaseResolver } from "./local-databases/database-resolver";
 
 export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents";
 export { DatabaseItem } from "./local-databases/database-item";
+export { DatabaseResolver } from "./local-databases/database-resolver";
 
 /**
  * databases.ts
@@ -66,136 +59,6 @@ const CURRENT_DB = "currentDatabase";
  */
 const DB_LIST = "databaseList";
 
-/**
- * An error thrown when we cannot find a valid database in a putative
- * database directory.
- */
-class InvalidDatabaseError extends Error {}
-
-async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
-  /*
-   * Look directly in the root
-   */
-  let dbRelativePaths = await glob("db-*/", {
-    cwd: parentDirectory,
-  });
-
-  if (dbRelativePaths.length === 0) {
-    /*
-     * Check If they are in the old location
-     */
-    dbRelativePaths = await glob("working/db-*/", {
-      cwd: parentDirectory,
-    });
-  }
-  if (dbRelativePaths.length === 0) {
-    throw new InvalidDatabaseError(
-      `'${parentDirectory}' does not contain a dataset directory.`,
-    );
-  }
-
-  const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]);
-  if (dbRelativePaths.length > 1) {
-    void showAndLogWarningMessage(
-      `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`,
-    );
-  }
-
-  return vscode.Uri.file(dbAbsolutePath);
-}
-
-// exported for testing
-export async function findSourceArchive(
-  databasePath: string,
-): Promise<vscode.Uri | undefined> {
-  const relativePaths = ["src", "output/src_archive"];
-
-  for (const relativePath of relativePaths) {
-    const basePath = join(databasePath, relativePath);
-    const zipPath = `${basePath}.zip`;
-
-    // Prefer using a zip archive over a directory.
-    if (await pathExists(zipPath)) {
-      return encodeArchiveBasePath(zipPath);
-    } else if (await pathExists(basePath)) {
-      return vscode.Uri.file(basePath);
-    }
-  }
-
-  void showAndLogInformationMessage(
-    `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`,
-  );
-  return undefined;
-}
-
-/** Gets the relative paths of all `.dbscheme` files in the given directory. */
-async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
-  return await glob("*.dbscheme", { cwd: dbDirectory });
-}
-
-export class DatabaseResolver {
-  public static async resolveDatabaseContents(
-    uri: vscode.Uri,
-  ): Promise<DatabaseContentsWithDbScheme> {
-    if (uri.scheme !== "file") {
-      throw new Error(
-        `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`,
-      );
-    }
-    const databasePath = uri.fsPath;
-    if (!(await pathExists(databasePath))) {
-      throw new InvalidDatabaseError(
-        `Database '${databasePath}' does not exist.`,
-      );
-    }
-
-    const contents = await this.resolveDatabase(databasePath);
-
-    if (contents === undefined) {
-      throw new InvalidDatabaseError(
-        `'${databasePath}' is not a valid database.`,
-      );
-    }
-
-    // Look for a single dbscheme file within the database.
-    // This should be found in the dataset directory, regardless of the form of database.
-    const dbPath = contents.datasetUri.fsPath;
-    const dbSchemeFiles = await getDbSchemeFiles(dbPath);
-    if (dbSchemeFiles.length === 0) {
-      throw new InvalidDatabaseError(
-        `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`,
-      );
-    } else if (dbSchemeFiles.length > 1) {
-      throw new InvalidDatabaseError(
-        `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`,
-      );
-    } else {
-      const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
-      return {
-        ...contents,
-        dbSchemeUri,
-      };
-    }
-  }
-
-  public static async resolveDatabase(
-    databasePath: string,
-  ): Promise<DatabaseContents> {
-    const name = basename(databasePath);
-
-    // Look for dataset and source archive.
-    const datasetUri = await findDataset(databasePath);
-    const sourceArchiveUri = await findSourceArchive(databasePath);
-
-    return {
-      kind: DatabaseKind.Database,
-      name,
-      datasetUri,
-      sourceArchiveUri,
-    };
-  }
-}
-
 export enum DatabaseEventKind {
   Add = "Add",
   Remove = "Remove",
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
index 7e9fc22c9..3b2397038 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -14,12 +14,9 @@ import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
 import { isLikelyDatabaseRoot } from "../../helpers";
 import { stat } from "fs-extra";
 import { pathsEqual } from "../../pure/files";
-import {
-  DatabaseChangedEvent,
-  DatabaseEventKind,
-  DatabaseResolver,
-} from "../local-databases";
+import { DatabaseChangedEvent, DatabaseEventKind } from "../local-databases";
 import { DatabaseContents } from "./database-contents";
+import { DatabaseResolver } from "./database-resolver";
 
 export class DatabaseItemImpl implements DatabaseItem {
   private _error: Error | undefined = undefined;
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts b/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts
new file mode 100644
index 000000000..aa758c773
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-resolver.ts
@@ -0,0 +1,144 @@
+import vscode from "vscode";
+import { pathExists } from "fs-extra";
+import { basename, join, resolve } from "path";
+import {
+  DatabaseContents,
+  DatabaseContentsWithDbScheme,
+  DatabaseKind,
+} from "./database-contents";
+import { glob } from "glob";
+import {
+  showAndLogInformationMessage,
+  showAndLogWarningMessage,
+} from "../../helpers";
+import { encodeArchiveBasePath } from "../../common/vscode/archive-filesystem-provider";
+
+export class DatabaseResolver {
+  public static async resolveDatabaseContents(
+    uri: vscode.Uri,
+  ): Promise<DatabaseContentsWithDbScheme> {
+    if (uri.scheme !== "file") {
+      throw new Error(
+        `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`,
+      );
+    }
+    const databasePath = uri.fsPath;
+    if (!(await pathExists(databasePath))) {
+      throw new InvalidDatabaseError(
+        `Database '${databasePath}' does not exist.`,
+      );
+    }
+
+    const contents = await this.resolveDatabase(databasePath);
+
+    if (contents === undefined) {
+      throw new InvalidDatabaseError(
+        `'${databasePath}' is not a valid database.`,
+      );
+    }
+
+    // Look for a single dbscheme file within the database.
+    // This should be found in the dataset directory, regardless of the form of database.
+    const dbPath = contents.datasetUri.fsPath;
+    const dbSchemeFiles = await getDbSchemeFiles(dbPath);
+    if (dbSchemeFiles.length === 0) {
+      throw new InvalidDatabaseError(
+        `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`,
+      );
+    } else if (dbSchemeFiles.length > 1) {
+      throw new InvalidDatabaseError(
+        `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`,
+      );
+    } else {
+      const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
+      return {
+        ...contents,
+        dbSchemeUri,
+      };
+    }
+  }
+
+  public static async resolveDatabase(
+    databasePath: string,
+  ): Promise<DatabaseContents> {
+    const name = basename(databasePath);
+
+    // Look for dataset and source archive.
+    const datasetUri = await findDataset(databasePath);
+    const sourceArchiveUri = await findSourceArchive(databasePath);
+
+    return {
+      kind: DatabaseKind.Database,
+      name,
+      datasetUri,
+      sourceArchiveUri,
+    };
+  }
+}
+
+/**
+ * An error thrown when we cannot find a valid database in a putative
+ * database directory.
+ */
+class InvalidDatabaseError extends Error {}
+
+async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
+  /*
+   * Look directly in the root
+   */
+  let dbRelativePaths = await glob("db-*/", {
+    cwd: parentDirectory,
+  });
+
+  if (dbRelativePaths.length === 0) {
+    /*
+     * Check If they are in the old location
+     */
+    dbRelativePaths = await glob("working/db-*/", {
+      cwd: parentDirectory,
+    });
+  }
+  if (dbRelativePaths.length === 0) {
+    throw new InvalidDatabaseError(
+      `'${parentDirectory}' does not contain a dataset directory.`,
+    );
+  }
+
+  const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]);
+  if (dbRelativePaths.length > 1) {
+    void showAndLogWarningMessage(
+      `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`,
+    );
+  }
+
+  return vscode.Uri.file(dbAbsolutePath);
+}
+
+/** Gets the relative paths of all `.dbscheme` files in the given directory. */
+async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
+  return await glob("*.dbscheme", { cwd: dbDirectory });
+}
+
+// exported for testing
+export async function findSourceArchive(
+  databasePath: string,
+): Promise<vscode.Uri | undefined> {
+  const relativePaths = ["src", "output/src_archive"];
+
+  for (const relativePath of relativePaths) {
+    const basePath = join(databasePath, relativePath);
+    const zipPath = `${basePath}.zip`;
+
+    // Prefer using a zip archive over a directory.
+    if (await pathExists(zipPath)) {
+      return encodeArchiveBasePath(zipPath);
+    } else if (await pathExists(basePath)) {
+      return vscode.Uri.file(basePath);
+    }
+  }
+
+  void showAndLogInformationMessage(
+    `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`,
+  );
+  return undefined;
+}

From 59482c2b2c65feb764c9cdce280993c8d5c096dd Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 17:00:52 +0200
Subject: [PATCH 056/119] Move DatabaseChangedEvent to separate file

---
 .../src/databases/local-databases.ts          | 26 ++++++-------------
 .../local-databases/database-events.ts        | 19 ++++++++++++++
 .../local-databases/database-item-impl.ts     |  2 +-
 3 files changed, 28 insertions(+), 19 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-events.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index d26849b8a..9d306c529 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -32,8 +32,16 @@ import {
 } from "./local-databases/database-item";
 import { DatabaseItemImpl } from "./local-databases/database-item-impl";
 import { DatabaseResolver } from "./local-databases/database-resolver";
+import {
+  DatabaseChangedEvent,
+  DatabaseEventKind,
+} from "./local-databases/database-events";
 
 export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents";
+export {
+  DatabaseChangedEvent,
+  DatabaseEventKind,
+} from "./local-databases/database-events";
 export { DatabaseItem } from "./local-databases/database-item";
 export { DatabaseResolver } from "./local-databases/database-resolver";
 
@@ -59,24 +67,6 @@ const CURRENT_DB = "currentDatabase";
  */
 const DB_LIST = "databaseList";
 
-export enum DatabaseEventKind {
-  Add = "Add",
-  Remove = "Remove",
-
-  // Fired when databases are refreshed from persisted state
-  Refresh = "Refresh",
-
-  // Fired when the current database changes
-  Change = "Change",
-
-  Rename = "Rename",
-}
-
-export interface DatabaseChangedEvent {
-  kind: DatabaseEventKind;
-  item: DatabaseItem | undefined;
-}
-
 /**
  * A promise that resolves to an event's result value when the event
  * `event` fires. If waiting for the event takes too long (by default
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-events.ts b/extensions/ql-vscode/src/databases/local-databases/database-events.ts
new file mode 100644
index 000000000..a48766ffb
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-events.ts
@@ -0,0 +1,19 @@
+import { DatabaseItem } from "./database-item";
+
+export enum DatabaseEventKind {
+  Add = "Add",
+  Remove = "Remove",
+
+  // Fired when databases are refreshed from persisted state
+  Refresh = "Refresh",
+
+  // Fired when the current database changes
+  Change = "Change",
+
+  Rename = "Rename",
+}
+
+export interface DatabaseChangedEvent {
+  kind: DatabaseEventKind;
+  item: DatabaseItem | undefined;
+}
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
index 3b2397038..bd2c76786 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -14,9 +14,9 @@ import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
 import { isLikelyDatabaseRoot } from "../../helpers";
 import { stat } from "fs-extra";
 import { pathsEqual } from "../../pure/files";
-import { DatabaseChangedEvent, DatabaseEventKind } from "../local-databases";
 import { DatabaseContents } from "./database-contents";
 import { DatabaseResolver } from "./database-resolver";
+import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 
 export class DatabaseItemImpl implements DatabaseItem {
   private _error: Error | undefined = undefined;

From 60cfc311e5e895f88663e3d2138919b7d57acca2 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 17:02:39 +0200
Subject: [PATCH 057/119] Move DatabaseManager to separate file

---
 .../src/databases/local-databases.ts          | 655 +-----------------
 .../local-databases/database-manager.ts       | 647 +++++++++++++++++
 2 files changed, 649 insertions(+), 653 deletions(-)
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/database-manager.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index 9d306c529..194286b30 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -1,41 +1,5 @@
-import { remove } from "fs-extra";
-import { join, dirname, extname } from "path";
+import { dirname } from "path";
 import * as vscode from "vscode";
-import * as cli from "../codeql-cli/cli";
-import { ExtensionContext } from "vscode";
-import {
-  showAndLogExceptionWithTelemetry,
-  isFolderAlreadyInWorkspace,
-  getFirstWorkspaceFolder,
-  showNeverAskAgainDialog,
-} from "../helpers";
-import { ProgressCallback, withProgress } from "../common/vscode/progress";
-import { DisposableObject } from "../pure/disposable-object";
-import { Logger, extLogger } from "../common";
-import { asError, getErrorMessage } from "../pure/helpers-pure";
-import { QueryRunner } from "../query-server";
-import { containsPath } from "../pure/files";
-import { redactableError } from "../pure/errors";
-import {
-  getAutogenerateQlPacks,
-  isCodespacesTemplate,
-  setAutogenerateQlPacks,
-} from "../config";
-import { QlPackGenerator } from "../qlpack-generator";
-import { QueryLanguage } from "../common/query-language";
-import { App } from "../common/app";
-import { existsSync } from "fs";
-import { FullDatabaseOptions } from "./local-databases/database-options";
-import {
-  DatabaseItem,
-  PersistedDatabaseItem,
-} from "./local-databases/database-item";
-import { DatabaseItemImpl } from "./local-databases/database-item-impl";
-import { DatabaseResolver } from "./local-databases/database-resolver";
-import {
-  DatabaseChangedEvent,
-  DatabaseEventKind,
-} from "./local-databases/database-events";
 
 export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents";
 export {
@@ -43,6 +7,7 @@ export {
   DatabaseEventKind,
 } from "./local-databases/database-events";
 export { DatabaseItem } from "./local-databases/database-item";
+export { DatabaseManager } from "./local-databases/database-manager";
 export { DatabaseResolver } from "./local-databases/database-resolver";
 
 /**
@@ -55,622 +20,6 @@ export { DatabaseResolver } from "./local-databases/database-resolver";
  * `DatabaseManager` class below.
  */
 
-/**
- * The name of the key in the workspaceState dictionary in which we
- * persist the current database across sessions.
- */
-const CURRENT_DB = "currentDatabase";
-
-/**
- * The name of the key in the workspaceState dictionary in which we
- * persist the list of databases across sessions.
- */
-const DB_LIST = "databaseList";
-
-/**
- * A promise that resolves to an event's result value when the event
- * `event` fires. If waiting for the event takes too long (by default
- * >1000ms) log a warning, and resolve to undefined.
- */
-function eventFired<T>(
-  event: vscode.Event<T>,
-  timeoutMs = 1000,
-): Promise<T | undefined> {
-  return new Promise((res, _rej) => {
-    const timeout = setTimeout(() => {
-      void extLogger.log(
-        `Waiting for event ${event} timed out after ${timeoutMs}ms`,
-      );
-      res(undefined);
-      dispose();
-    }, timeoutMs);
-    const disposable = event((e) => {
-      res(e);
-      dispose();
-    });
-    function dispose() {
-      clearTimeout(timeout);
-      disposable.dispose();
-    }
-  });
-}
-
-export class DatabaseManager extends DisposableObject {
-  private readonly _onDidChangeDatabaseItem = this.push(
-    new vscode.EventEmitter<DatabaseChangedEvent>(),
-  );
-
-  readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
-
-  private readonly _onDidChangeCurrentDatabaseItem = this.push(
-    new vscode.EventEmitter<DatabaseChangedEvent>(),
-  );
-  readonly onDidChangeCurrentDatabaseItem =
-    this._onDidChangeCurrentDatabaseItem.event;
-
-  private readonly _databaseItems: DatabaseItem[] = [];
-  private _currentDatabaseItem: DatabaseItem | undefined = undefined;
-
-  constructor(
-    private readonly ctx: ExtensionContext,
-    private readonly app: App,
-    private readonly qs: QueryRunner,
-    private readonly cli: cli.CodeQLCliServer,
-    public logger: Logger,
-  ) {
-    super();
-
-    qs.onStart(this.reregisterDatabases.bind(this));
-  }
-
-  /**
-   * Creates a {@link DatabaseItem} for the specified database, and adds it to the list of open
-   * databases.
-   */
-  public async openDatabase(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    uri: vscode.Uri,
-    makeSelected = true,
-    displayName?: string,
-    isTutorialDatabase?: boolean,
-  ): Promise<DatabaseItem> {
-    const databaseItem = await this.createDatabaseItem(uri, displayName);
-
-    return await this.addExistingDatabaseItem(
-      databaseItem,
-      progress,
-      makeSelected,
-      token,
-      isTutorialDatabase,
-    );
-  }
-
-  /**
-   * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
-   * the list.
-   *
-   * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}.
-   */
-  public async addExistingDatabaseItem(
-    databaseItem: DatabaseItem,
-    progress: ProgressCallback,
-    makeSelected: boolean,
-    token: vscode.CancellationToken,
-    isTutorialDatabase?: boolean,
-  ): Promise<DatabaseItem> {
-    const existingItem = this.findDatabaseItem(databaseItem.databaseUri);
-    if (existingItem !== undefined) {
-      if (makeSelected) {
-        await this.setCurrentDatabaseItem(existingItem);
-      }
-      return existingItem;
-    }
-
-    await this.addDatabaseItem(progress, token, databaseItem);
-    if (makeSelected) {
-      await this.setCurrentDatabaseItem(databaseItem);
-    }
-    await this.addDatabaseSourceArchiveFolder(databaseItem);
-
-    if (isCodespacesTemplate() && !isTutorialDatabase) {
-      await this.createSkeletonPacks(databaseItem);
-    }
-
-    return databaseItem;
-  }
-
-  /**
-   * Creates a {@link DatabaseItem} for the specified database, without adding it to the list of
-   * open databases.
-   */
-  private async createDatabaseItem(
-    uri: vscode.Uri,
-    displayName: string | undefined,
-  ): Promise<DatabaseItem> {
-    const contents = await DatabaseResolver.resolveDatabaseContents(uri);
-    // Ignore the source archive for QLTest databases by default.
-    const isQLTestDatabase = extname(uri.fsPath) === ".testproj";
-    const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: isQLTestDatabase,
-      // If a displayName is not passed in, the basename of folder containing the database is used.
-      displayName,
-      dateAdded: Date.now(),
-      language: await this.getPrimaryLanguage(uri.fsPath),
-    };
-    const databaseItem = new DatabaseItemImpl(
-      uri,
-      contents,
-      fullOptions,
-      (event) => {
-        this._onDidChangeDatabaseItem.fire(event);
-      },
-    );
-
-    return databaseItem;
-  }
-
-  /**
-   * If the specified database is already on the list of open databases, returns that database's
-   * {@link DatabaseItem}. Otherwise, creates a new {@link DatabaseItem} without adding it to the
-   * list of open databases.
-   *
-   * The {@link DatabaseItem} can be added to the list of open databases later, via {@link addExistingDatabaseItem}.
-   */
-  public async createOrOpenDatabaseItem(
-    uri: vscode.Uri,
-  ): Promise<DatabaseItem> {
-    const existingItem = this.findDatabaseItem(uri);
-    if (existingItem !== undefined) {
-      // Use the one we already have.
-      return existingItem;
-    }
-
-    // We don't add this to the list automatically, but the user can add it later.
-    return this.createDatabaseItem(uri, undefined);
-  }
-
-  public async createSkeletonPacks(databaseItem: DatabaseItem) {
-    if (databaseItem === undefined) {
-      void this.logger.log(
-        "Could not create QL pack because no database is selected. Please add a database.",
-      );
-      return;
-    }
-
-    if (databaseItem.language === "") {
-      void this.logger.log(
-        "Could not create skeleton QL pack because the selected database's language is not set.",
-      );
-      return;
-    }
-
-    const firstWorkspaceFolder = getFirstWorkspaceFolder();
-    const folderName = `codeql-custom-queries-${databaseItem.language}`;
-
-    if (
-      existsSync(join(firstWorkspaceFolder, folderName)) ||
-      isFolderAlreadyInWorkspace(folderName)
-    ) {
-      return;
-    }
-
-    if (getAutogenerateQlPacks() === "never") {
-      return;
-    }
-
-    const answer = await showNeverAskAgainDialog(
-      `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`,
-    );
-
-    if (answer === "No") {
-      return;
-    }
-
-    if (answer === "No, and never ask me again") {
-      await setAutogenerateQlPacks("never");
-      return;
-    }
-
-    try {
-      const qlPackGenerator = new QlPackGenerator(
-        folderName,
-        databaseItem.language as QueryLanguage,
-        this.cli,
-        firstWorkspaceFolder,
-      );
-      await qlPackGenerator.generate();
-    } catch (e: unknown) {
-      void this.logger.log(
-        `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
-      );
-    }
-  }
-
-  private async reregisterDatabases(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-  ) {
-    let completed = 0;
-    await Promise.all(
-      this._databaseItems.map(async (databaseItem) => {
-        await this.registerDatabase(progress, token, databaseItem);
-        completed++;
-        progress({
-          maxStep: this._databaseItems.length,
-          step: completed,
-          message: "Re-registering databases",
-        });
-      }),
-    );
-  }
-
-  public async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
-    // The folder may already be in workspace state from a previous
-    // session. If not, add it.
-    const index = this.getDatabaseWorkspaceFolderIndex(item);
-    if (index === -1) {
-      // Add that filesystem as a folder to the current workspace.
-      //
-      // It's important that we add workspace folders to the end,
-      // rather than beginning of the list, because the first
-      // workspace folder is special; if it gets updated, the entire
-      // extension host is restarted. (cf.
-      // https://github.com/microsoft/vscode/blob/e0d2ed907d1b22808c56127678fb436d604586a7/src/vs/workbench/contrib/relauncher/browser/relauncher.contribution.ts#L209-L214)
-      //
-      // This is undesirable, as we might be adding and removing many
-      // workspace folders as the user adds and removes databases.
-      const end = (vscode.workspace.workspaceFolders || []).length;
-
-      const msg = item.verifyZippedSources();
-      if (msg) {
-        void extLogger.log(`Could not add source folder because ${msg}`);
-        return;
-      }
-
-      const uri = item.getSourceArchiveExplorerUri();
-      void extLogger.log(
-        `Adding workspace folder for ${item.name} source archive at index ${end}`,
-      );
-      if ((vscode.workspace.workspaceFolders || []).length < 2) {
-        // Adding this workspace folder makes the workspace
-        // multi-root, which may surprise the user. Let them know
-        // we're doing this.
-        void vscode.window.showInformationMessage(
-          `Adding workspace folder for source archive of database ${item.name}.`,
-        );
-      }
-      vscode.workspace.updateWorkspaceFolders(end, 0, {
-        name: `[${item.name} source archive]`,
-        uri,
-      });
-      // vscode api documentation says we must to wait for this event
-      // between multiple `updateWorkspaceFolders` calls.
-      await eventFired(vscode.workspace.onDidChangeWorkspaceFolders);
-    }
-  }
-
-  private async createDatabaseItemFromPersistedState(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    state: PersistedDatabaseItem,
-  ): Promise<DatabaseItem> {
-    let displayName: string | undefined = undefined;
-    let ignoreSourceArchive = false;
-    let dateAdded = undefined;
-    let language = undefined;
-    if (state.options) {
-      if (typeof state.options.displayName === "string") {
-        displayName = state.options.displayName;
-      }
-      if (typeof state.options.ignoreSourceArchive === "boolean") {
-        ignoreSourceArchive = state.options.ignoreSourceArchive;
-      }
-      if (typeof state.options.dateAdded === "number") {
-        dateAdded = state.options.dateAdded;
-      }
-      language = state.options.language;
-    }
-
-    const dbBaseUri = vscode.Uri.parse(state.uri, true);
-    if (language === undefined) {
-      // we haven't been successful yet at getting the language. try again
-      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
-    }
-
-    const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive,
-      displayName,
-      dateAdded,
-      language,
-    };
-    const item = new DatabaseItemImpl(
-      dbBaseUri,
-      undefined,
-      fullOptions,
-      (event) => {
-        this._onDidChangeDatabaseItem.fire(event);
-      },
-    );
-
-    // Avoid persisting the database state after adding since that should happen only after
-    // all databases have been added.
-    await this.addDatabaseItem(progress, token, item, false);
-    return item;
-  }
-
-  public async loadPersistedState(): Promise<void> {
-    return withProgress(async (progress, token) => {
-      const currentDatabaseUri =
-        this.ctx.workspaceState.get<string>(CURRENT_DB);
-      const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(
-        DB_LIST,
-        [],
-      );
-      let step = 0;
-      progress({
-        maxStep: databases.length,
-        message: "Loading persisted databases",
-        step,
-      });
-      try {
-        void this.logger.log(
-          `Found ${databases.length} persisted databases: ${databases
-            .map((db) => db.uri)
-            .join(", ")}`,
-        );
-        for (const database of databases) {
-          progress({
-            maxStep: databases.length,
-            message: `Loading ${database.options?.displayName || "databases"}`,
-            step: ++step,
-          });
-
-          const databaseItem = await this.createDatabaseItemFromPersistedState(
-            progress,
-            token,
-            database,
-          );
-          try {
-            await databaseItem.refresh();
-            await this.registerDatabase(progress, token, databaseItem);
-            if (currentDatabaseUri === database.uri) {
-              await this.setCurrentDatabaseItem(databaseItem, true);
-            }
-            void this.logger.log(
-              `Loaded database ${databaseItem.name} at URI ${database.uri}.`,
-            );
-          } catch (e) {
-            // When loading from persisted state, leave invalid databases in the list. They will be
-            // marked as invalid, and cannot be set as the current database.
-            void this.logger.log(
-              `Error loading database ${database.uri}: ${e}.`,
-            );
-          }
-        }
-        await this.updatePersistedDatabaseList();
-      } catch (e) {
-        // database list had an unexpected type - nothing to be done?
-        void showAndLogExceptionWithTelemetry(
-          redactableError(
-            asError(e),
-          )`Database list loading failed: ${getErrorMessage(e)}`,
-        );
-      }
-
-      void this.logger.log("Finished loading persisted databases.");
-    });
-  }
-
-  public get databaseItems(): readonly DatabaseItem[] {
-    return this._databaseItems;
-  }
-
-  public get currentDatabaseItem(): DatabaseItem | undefined {
-    return this._currentDatabaseItem;
-  }
-
-  public async setCurrentDatabaseItem(
-    item: DatabaseItem | undefined,
-    skipRefresh = false,
-  ): Promise<void> {
-    if (!skipRefresh && item !== undefined) {
-      await item.refresh(); // Will throw on invalid database.
-    }
-    if (this._currentDatabaseItem !== item) {
-      this._currentDatabaseItem = item;
-      this.updatePersistedCurrentDatabaseItem();
-
-      await this.app.commands.execute(
-        "setContext",
-        "codeQL.currentDatabaseItem",
-        item?.name,
-      );
-
-      this._onDidChangeCurrentDatabaseItem.fire({
-        item,
-        kind: DatabaseEventKind.Change,
-      });
-    }
-  }
-
-  /**
-   * Returns the index of the workspace folder that corresponds to the source archive of `item`
-   * if there is one, and -1 otherwise.
-   */
-  private getDatabaseWorkspaceFolderIndex(item: DatabaseItem): number {
-    return (vscode.workspace.workspaceFolders || []).findIndex((folder) =>
-      item.belongsToSourceArchiveExplorerUri(folder.uri),
-    );
-  }
-
-  public findDatabaseItem(uri: vscode.Uri): DatabaseItem | undefined {
-    const uriString = uri.toString(true);
-    return this._databaseItems.find(
-      (item) => item.databaseUri.toString(true) === uriString,
-    );
-  }
-
-  public findDatabaseItemBySourceArchive(
-    uri: vscode.Uri,
-  ): DatabaseItem | undefined {
-    const uriString = uri.toString(true);
-    return this._databaseItems.find(
-      (item) =>
-        item.sourceArchive && item.sourceArchive.toString(true) === uriString,
-    );
-  }
-
-  private async addDatabaseItem(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    item: DatabaseItem,
-    updatePersistedState = true,
-  ) {
-    this._databaseItems.push(item);
-
-    if (updatePersistedState) {
-      await this.updatePersistedDatabaseList();
-    }
-
-    // Add this database item to the allow-list
-    // Database items reconstituted from persisted state
-    // will not have their contents yet.
-    if (item.contents?.datasetUri) {
-      await this.registerDatabase(progress, token, item);
-    }
-    // note that we use undefined as the item in order to reset the entire tree
-    this._onDidChangeDatabaseItem.fire({
-      item: undefined,
-      kind: DatabaseEventKind.Add,
-    });
-  }
-
-  public async renameDatabaseItem(item: DatabaseItem, newName: string) {
-    item.name = newName;
-    await this.updatePersistedDatabaseList();
-    this._onDidChangeDatabaseItem.fire({
-      // pass undefined so that the entire tree is rebuilt in order to re-sort
-      item: undefined,
-      kind: DatabaseEventKind.Rename,
-    });
-  }
-
-  public async removeDatabaseItem(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    item: DatabaseItem,
-  ) {
-    if (this._currentDatabaseItem === item) {
-      this._currentDatabaseItem = undefined;
-    }
-    const index = this.databaseItems.findIndex(
-      (searchItem) => searchItem === item,
-    );
-    if (index >= 0) {
-      this._databaseItems.splice(index, 1);
-    }
-    await this.updatePersistedDatabaseList();
-
-    // Delete folder from workspace, if it is still there
-    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
-      (folder) => item.belongsToSourceArchiveExplorerUri(folder.uri),
-    );
-    if (folderIndex >= 0) {
-      void extLogger.log(`Removing workspace folder at index ${folderIndex}`);
-      vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
-    }
-
-    // Remove this database item from the allow-list
-    await this.deregisterDatabase(progress, token, item);
-
-    // Delete folder from file system only if it is controlled by the extension
-    if (this.isExtensionControlledLocation(item.databaseUri)) {
-      void extLogger.log("Deleting database from filesystem.");
-      await remove(item.databaseUri.fsPath).then(
-        () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`),
-        (e: unknown) =>
-          void extLogger.log(
-            `Failed to delete '${
-              item.databaseUri.fsPath
-            }'. Reason: ${getErrorMessage(e)}`,
-          ),
-      );
-    }
-
-    // note that we use undefined as the item in order to reset the entire tree
-    this._onDidChangeDatabaseItem.fire({
-      item: undefined,
-      kind: DatabaseEventKind.Remove,
-    });
-  }
-
-  public async removeAllDatabases(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-  ) {
-    for (const item of this.databaseItems) {
-      await this.removeDatabaseItem(progress, token, item);
-    }
-  }
-
-  private async deregisterDatabase(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    dbItem: DatabaseItem,
-  ) {
-    try {
-      await this.qs.deregisterDatabase(progress, token, dbItem);
-    } catch (e) {
-      const message = getErrorMessage(e);
-      if (message === "Connection is disposed.") {
-        // This is expected if the query server is not running.
-        void extLogger.log(
-          `Could not de-register database '${dbItem.name}' because query server is not running.`,
-        );
-        return;
-      }
-      throw e;
-    }
-  }
-  private async registerDatabase(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    dbItem: DatabaseItem,
-  ) {
-    await this.qs.registerDatabase(progress, token, dbItem);
-  }
-
-  private updatePersistedCurrentDatabaseItem(): void {
-    void this.ctx.workspaceState.update(
-      CURRENT_DB,
-      this._currentDatabaseItem
-        ? this._currentDatabaseItem.databaseUri.toString(true)
-        : undefined,
-    );
-  }
-
-  private async updatePersistedDatabaseList(): Promise<void> {
-    await this.ctx.workspaceState.update(
-      DB_LIST,
-      this._databaseItems.map((item) => item.getPersistedState()),
-    );
-  }
-
-  private isExtensionControlledLocation(uri: vscode.Uri) {
-    const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri;
-    if (storageUri) {
-      return containsPath(storageUri.fsPath, uri.fsPath, process.platform);
-    }
-    return false;
-  }
-
-  private async getPrimaryLanguage(dbPath: string) {
-    const dbInfo = await this.cli.resolveDatabase(dbPath);
-    return dbInfo.languages?.[0] || "";
-  }
-}
-
 /**
  * Get the set of directories containing upgrades, given a list of
  * scripts returned by the cli's upgrade resolution.
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
new file mode 100644
index 000000000..1383b179e
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
@@ -0,0 +1,647 @@
+import vscode, { ExtensionContext } from "vscode";
+import { extLogger, Logger } from "../../common";
+import { DisposableObject } from "../../pure/disposable-object";
+import { App } from "../../common/app";
+import { QueryRunner } from "../../query-server";
+import * as cli from "../../codeql-cli/cli";
+import { ProgressCallback, withProgress } from "../../common/vscode/progress";
+import {
+  getAutogenerateQlPacks,
+  isCodespacesTemplate,
+  setAutogenerateQlPacks,
+} from "../../config";
+import { extname, join } from "path";
+import { FullDatabaseOptions } from "./database-options";
+import { DatabaseItemImpl } from "./database-item-impl";
+import {
+  getFirstWorkspaceFolder,
+  isFolderAlreadyInWorkspace,
+  showAndLogExceptionWithTelemetry,
+  showNeverAskAgainDialog,
+} from "../../helpers";
+import { existsSync } from "fs";
+import { QlPackGenerator } from "../../qlpack-generator";
+import { QueryLanguage } from "../../common/query-language";
+import { asError, getErrorMessage } from "../../pure/helpers-pure";
+import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
+import { redactableError } from "../../pure/errors";
+import { remove } from "fs-extra";
+import { containsPath } from "../../pure/files";
+import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
+import { DatabaseResolver } from "./database-resolver";
+
+/**
+ * The name of the key in the workspaceState dictionary in which we
+ * persist the current database across sessions.
+ */
+const CURRENT_DB = "currentDatabase";
+
+/**
+ * The name of the key in the workspaceState dictionary in which we
+ * persist the list of databases across sessions.
+ */
+const DB_LIST = "databaseList";
+
+/**
+ * A promise that resolves to an event's result value when the event
+ * `event` fires. If waiting for the event takes too long (by default
+ * >1000ms) log a warning, and resolve to undefined.
+ */
+function eventFired<T>(
+  event: vscode.Event<T>,
+  timeoutMs = 1000,
+): Promise<T | undefined> {
+  return new Promise((res, _rej) => {
+    const timeout = setTimeout(() => {
+      void extLogger.log(
+        `Waiting for event ${event} timed out after ${timeoutMs}ms`,
+      );
+      res(undefined);
+      dispose();
+    }, timeoutMs);
+    const disposable = event((e) => {
+      res(e);
+      dispose();
+    });
+    function dispose() {
+      clearTimeout(timeout);
+      disposable.dispose();
+    }
+  });
+}
+
+export class DatabaseManager extends DisposableObject {
+  private readonly _onDidChangeDatabaseItem = this.push(
+    new vscode.EventEmitter<DatabaseChangedEvent>(),
+  );
+
+  readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
+
+  private readonly _onDidChangeCurrentDatabaseItem = this.push(
+    new vscode.EventEmitter<DatabaseChangedEvent>(),
+  );
+  readonly onDidChangeCurrentDatabaseItem =
+    this._onDidChangeCurrentDatabaseItem.event;
+
+  private readonly _databaseItems: DatabaseItem[] = [];
+  private _currentDatabaseItem: DatabaseItem | undefined = undefined;
+
+  constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly app: App,
+    private readonly qs: QueryRunner,
+    private readonly cli: cli.CodeQLCliServer,
+    public logger: Logger,
+  ) {
+    super();
+
+    qs.onStart(this.reregisterDatabases.bind(this));
+  }
+
+  /**
+   * Creates a {@link DatabaseItem} for the specified database, and adds it to the list of open
+   * databases.
+   */
+  public async openDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    uri: vscode.Uri,
+    makeSelected = true,
+    displayName?: string,
+    isTutorialDatabase?: boolean,
+  ): Promise<DatabaseItem> {
+    const databaseItem = await this.createDatabaseItem(uri, displayName);
+
+    return await this.addExistingDatabaseItem(
+      databaseItem,
+      progress,
+      makeSelected,
+      token,
+      isTutorialDatabase,
+    );
+  }
+
+  /**
+   * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
+   * the list.
+   *
+   * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}.
+   */
+  public async addExistingDatabaseItem(
+    databaseItem: DatabaseItem,
+    progress: ProgressCallback,
+    makeSelected: boolean,
+    token: vscode.CancellationToken,
+    isTutorialDatabase?: boolean,
+  ): Promise<DatabaseItem> {
+    const existingItem = this.findDatabaseItem(databaseItem.databaseUri);
+    if (existingItem !== undefined) {
+      if (makeSelected) {
+        await this.setCurrentDatabaseItem(existingItem);
+      }
+      return existingItem;
+    }
+
+    await this.addDatabaseItem(progress, token, databaseItem);
+    if (makeSelected) {
+      await this.setCurrentDatabaseItem(databaseItem);
+    }
+    await this.addDatabaseSourceArchiveFolder(databaseItem);
+
+    if (isCodespacesTemplate() && !isTutorialDatabase) {
+      await this.createSkeletonPacks(databaseItem);
+    }
+
+    return databaseItem;
+  }
+
+  /**
+   * Creates a {@link DatabaseItem} for the specified database, without adding it to the list of
+   * open databases.
+   */
+  private async createDatabaseItem(
+    uri: vscode.Uri,
+    displayName: string | undefined,
+  ): Promise<DatabaseItem> {
+    const contents = await DatabaseResolver.resolveDatabaseContents(uri);
+    // Ignore the source archive for QLTest databases by default.
+    const isQLTestDatabase = extname(uri.fsPath) === ".testproj";
+    const fullOptions: FullDatabaseOptions = {
+      ignoreSourceArchive: isQLTestDatabase,
+      // If a displayName is not passed in, the basename of folder containing the database is used.
+      displayName,
+      dateAdded: Date.now(),
+      language: await this.getPrimaryLanguage(uri.fsPath),
+    };
+    const databaseItem = new DatabaseItemImpl(
+      uri,
+      contents,
+      fullOptions,
+      (event) => {
+        this._onDidChangeDatabaseItem.fire(event);
+      },
+    );
+
+    return databaseItem;
+  }
+
+  /**
+   * If the specified database is already on the list of open databases, returns that database's
+   * {@link DatabaseItem}. Otherwise, creates a new {@link DatabaseItem} without adding it to the
+   * list of open databases.
+   *
+   * The {@link DatabaseItem} can be added to the list of open databases later, via {@link addExistingDatabaseItem}.
+   */
+  public async createOrOpenDatabaseItem(
+    uri: vscode.Uri,
+  ): Promise<DatabaseItem> {
+    const existingItem = this.findDatabaseItem(uri);
+    if (existingItem !== undefined) {
+      // Use the one we already have.
+      return existingItem;
+    }
+
+    // We don't add this to the list automatically, but the user can add it later.
+    return this.createDatabaseItem(uri, undefined);
+  }
+
+  public async createSkeletonPacks(databaseItem: DatabaseItem) {
+    if (databaseItem === undefined) {
+      void this.logger.log(
+        "Could not create QL pack because no database is selected. Please add a database.",
+      );
+      return;
+    }
+
+    if (databaseItem.language === "") {
+      void this.logger.log(
+        "Could not create skeleton QL pack because the selected database's language is not set.",
+      );
+      return;
+    }
+
+    const firstWorkspaceFolder = getFirstWorkspaceFolder();
+    const folderName = `codeql-custom-queries-${databaseItem.language}`;
+
+    if (
+      existsSync(join(firstWorkspaceFolder, folderName)) ||
+      isFolderAlreadyInWorkspace(folderName)
+    ) {
+      return;
+    }
+
+    if (getAutogenerateQlPacks() === "never") {
+      return;
+    }
+
+    const answer = await showNeverAskAgainDialog(
+      `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`,
+    );
+
+    if (answer === "No") {
+      return;
+    }
+
+    if (answer === "No, and never ask me again") {
+      await setAutogenerateQlPacks("never");
+      return;
+    }
+
+    try {
+      const qlPackGenerator = new QlPackGenerator(
+        folderName,
+        databaseItem.language as QueryLanguage,
+        this.cli,
+        firstWorkspaceFolder,
+      );
+      await qlPackGenerator.generate();
+    } catch (e: unknown) {
+      void this.logger.log(
+        `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+
+  private async reregisterDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+  ) {
+    let completed = 0;
+    await Promise.all(
+      this._databaseItems.map(async (databaseItem) => {
+        await this.registerDatabase(progress, token, databaseItem);
+        completed++;
+        progress({
+          maxStep: this._databaseItems.length,
+          step: completed,
+          message: "Re-registering databases",
+        });
+      }),
+    );
+  }
+
+  public async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
+    // The folder may already be in workspace state from a previous
+    // session. If not, add it.
+    const index = this.getDatabaseWorkspaceFolderIndex(item);
+    if (index === -1) {
+      // Add that filesystem as a folder to the current workspace.
+      //
+      // It's important that we add workspace folders to the end,
+      // rather than beginning of the list, because the first
+      // workspace folder is special; if it gets updated, the entire
+      // extension host is restarted. (cf.
+      // https://github.com/microsoft/vscode/blob/e0d2ed907d1b22808c56127678fb436d604586a7/src/vs/workbench/contrib/relauncher/browser/relauncher.contribution.ts#L209-L214)
+      //
+      // This is undesirable, as we might be adding and removing many
+      // workspace folders as the user adds and removes databases.
+      const end = (vscode.workspace.workspaceFolders || []).length;
+
+      const msg = item.verifyZippedSources();
+      if (msg) {
+        void extLogger.log(`Could not add source folder because ${msg}`);
+        return;
+      }
+
+      const uri = item.getSourceArchiveExplorerUri();
+      void extLogger.log(
+        `Adding workspace folder for ${item.name} source archive at index ${end}`,
+      );
+      if ((vscode.workspace.workspaceFolders || []).length < 2) {
+        // Adding this workspace folder makes the workspace
+        // multi-root, which may surprise the user. Let them know
+        // we're doing this.
+        void vscode.window.showInformationMessage(
+          `Adding workspace folder for source archive of database ${item.name}.`,
+        );
+      }
+      vscode.workspace.updateWorkspaceFolders(end, 0, {
+        name: `[${item.name} source archive]`,
+        uri,
+      });
+      // vscode api documentation says we must to wait for this event
+      // between multiple `updateWorkspaceFolders` calls.
+      await eventFired(vscode.workspace.onDidChangeWorkspaceFolders);
+    }
+  }
+
+  private async createDatabaseItemFromPersistedState(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    state: PersistedDatabaseItem,
+  ): Promise<DatabaseItem> {
+    let displayName: string | undefined = undefined;
+    let ignoreSourceArchive = false;
+    let dateAdded = undefined;
+    let language = undefined;
+    if (state.options) {
+      if (typeof state.options.displayName === "string") {
+        displayName = state.options.displayName;
+      }
+      if (typeof state.options.ignoreSourceArchive === "boolean") {
+        ignoreSourceArchive = state.options.ignoreSourceArchive;
+      }
+      if (typeof state.options.dateAdded === "number") {
+        dateAdded = state.options.dateAdded;
+      }
+      language = state.options.language;
+    }
+
+    const dbBaseUri = vscode.Uri.parse(state.uri, true);
+    if (language === undefined) {
+      // we haven't been successful yet at getting the language. try again
+      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
+    }
+
+    const fullOptions: FullDatabaseOptions = {
+      ignoreSourceArchive,
+      displayName,
+      dateAdded,
+      language,
+    };
+    const item = new DatabaseItemImpl(
+      dbBaseUri,
+      undefined,
+      fullOptions,
+      (event) => {
+        this._onDidChangeDatabaseItem.fire(event);
+      },
+    );
+
+    // Avoid persisting the database state after adding since that should happen only after
+    // all databases have been added.
+    await this.addDatabaseItem(progress, token, item, false);
+    return item;
+  }
+
+  public async loadPersistedState(): Promise<void> {
+    return withProgress(async (progress, token) => {
+      const currentDatabaseUri =
+        this.ctx.workspaceState.get<string>(CURRENT_DB);
+      const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(
+        DB_LIST,
+        [],
+      );
+      let step = 0;
+      progress({
+        maxStep: databases.length,
+        message: "Loading persisted databases",
+        step,
+      });
+      try {
+        void this.logger.log(
+          `Found ${databases.length} persisted databases: ${databases
+            .map((db) => db.uri)
+            .join(", ")}`,
+        );
+        for (const database of databases) {
+          progress({
+            maxStep: databases.length,
+            message: `Loading ${database.options?.displayName || "databases"}`,
+            step: ++step,
+          });
+
+          const databaseItem = await this.createDatabaseItemFromPersistedState(
+            progress,
+            token,
+            database,
+          );
+          try {
+            await databaseItem.refresh();
+            await this.registerDatabase(progress, token, databaseItem);
+            if (currentDatabaseUri === database.uri) {
+              await this.setCurrentDatabaseItem(databaseItem, true);
+            }
+            void this.logger.log(
+              `Loaded database ${databaseItem.name} at URI ${database.uri}.`,
+            );
+          } catch (e) {
+            // When loading from persisted state, leave invalid databases in the list. They will be
+            // marked as invalid, and cannot be set as the current database.
+            void this.logger.log(
+              `Error loading database ${database.uri}: ${e}.`,
+            );
+          }
+        }
+        await this.updatePersistedDatabaseList();
+      } catch (e) {
+        // database list had an unexpected type - nothing to be done?
+        void showAndLogExceptionWithTelemetry(
+          redactableError(
+            asError(e),
+          )`Database list loading failed: ${getErrorMessage(e)}`,
+        );
+      }
+
+      void this.logger.log("Finished loading persisted databases.");
+    });
+  }
+
+  public get databaseItems(): readonly DatabaseItem[] {
+    return this._databaseItems;
+  }
+
+  public get currentDatabaseItem(): DatabaseItem | undefined {
+    return this._currentDatabaseItem;
+  }
+
+  public async setCurrentDatabaseItem(
+    item: DatabaseItem | undefined,
+    skipRefresh = false,
+  ): Promise<void> {
+    if (!skipRefresh && item !== undefined) {
+      await item.refresh(); // Will throw on invalid database.
+    }
+    if (this._currentDatabaseItem !== item) {
+      this._currentDatabaseItem = item;
+      this.updatePersistedCurrentDatabaseItem();
+
+      await this.app.commands.execute(
+        "setContext",
+        "codeQL.currentDatabaseItem",
+        item?.name,
+      );
+
+      this._onDidChangeCurrentDatabaseItem.fire({
+        item,
+        kind: DatabaseEventKind.Change,
+      });
+    }
+  }
+
+  /**
+   * Returns the index of the workspace folder that corresponds to the source archive of `item`
+   * if there is one, and -1 otherwise.
+   */
+  private getDatabaseWorkspaceFolderIndex(item: DatabaseItem): number {
+    return (vscode.workspace.workspaceFolders || []).findIndex((folder) =>
+      item.belongsToSourceArchiveExplorerUri(folder.uri),
+    );
+  }
+
+  public findDatabaseItem(uri: vscode.Uri): DatabaseItem | undefined {
+    const uriString = uri.toString(true);
+    return this._databaseItems.find(
+      (item) => item.databaseUri.toString(true) === uriString,
+    );
+  }
+
+  public findDatabaseItemBySourceArchive(
+    uri: vscode.Uri,
+  ): DatabaseItem | undefined {
+    const uriString = uri.toString(true);
+    return this._databaseItems.find(
+      (item) =>
+        item.sourceArchive && item.sourceArchive.toString(true) === uriString,
+    );
+  }
+
+  private async addDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem,
+    updatePersistedState = true,
+  ) {
+    this._databaseItems.push(item);
+
+    if (updatePersistedState) {
+      await this.updatePersistedDatabaseList();
+    }
+
+    // Add this database item to the allow-list
+    // Database items reconstituted from persisted state
+    // will not have their contents yet.
+    if (item.contents?.datasetUri) {
+      await this.registerDatabase(progress, token, item);
+    }
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Add,
+    });
+  }
+
+  public async renameDatabaseItem(item: DatabaseItem, newName: string) {
+    item.name = newName;
+    await this.updatePersistedDatabaseList();
+    this._onDidChangeDatabaseItem.fire({
+      // pass undefined so that the entire tree is rebuilt in order to re-sort
+      item: undefined,
+      kind: DatabaseEventKind.Rename,
+    });
+  }
+
+  public async removeDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem,
+  ) {
+    if (this._currentDatabaseItem === item) {
+      this._currentDatabaseItem = undefined;
+    }
+    const index = this.databaseItems.findIndex(
+      (searchItem) => searchItem === item,
+    );
+    if (index >= 0) {
+      this._databaseItems.splice(index, 1);
+    }
+    await this.updatePersistedDatabaseList();
+
+    // Delete folder from workspace, if it is still there
+    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
+      (folder) => item.belongsToSourceArchiveExplorerUri(folder.uri),
+    );
+    if (folderIndex >= 0) {
+      void extLogger.log(`Removing workspace folder at index ${folderIndex}`);
+      vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
+    }
+
+    // Remove this database item from the allow-list
+    await this.deregisterDatabase(progress, token, item);
+
+    // Delete folder from file system only if it is controlled by the extension
+    if (this.isExtensionControlledLocation(item.databaseUri)) {
+      void extLogger.log("Deleting database from filesystem.");
+      await remove(item.databaseUri.fsPath).then(
+        () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`),
+        (e: unknown) =>
+          void extLogger.log(
+            `Failed to delete '${
+              item.databaseUri.fsPath
+            }'. Reason: ${getErrorMessage(e)}`,
+          ),
+      );
+    }
+
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Remove,
+    });
+  }
+
+  public async removeAllDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+  ) {
+    for (const item of this.databaseItems) {
+      await this.removeDatabaseItem(progress, token, item);
+    }
+  }
+
+  private async deregisterDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    try {
+      await this.qs.deregisterDatabase(progress, token, dbItem);
+    } catch (e) {
+      const message = getErrorMessage(e);
+      if (message === "Connection is disposed.") {
+        // This is expected if the query server is not running.
+        void extLogger.log(
+          `Could not de-register database '${dbItem.name}' because query server is not running.`,
+        );
+        return;
+      }
+      throw e;
+    }
+  }
+  private async registerDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    await this.qs.registerDatabase(progress, token, dbItem);
+  }
+
+  private updatePersistedCurrentDatabaseItem(): void {
+    void this.ctx.workspaceState.update(
+      CURRENT_DB,
+      this._currentDatabaseItem
+        ? this._currentDatabaseItem.databaseUri.toString(true)
+        : undefined,
+    );
+  }
+
+  private async updatePersistedDatabaseList(): Promise<void> {
+    await this.ctx.workspaceState.update(
+      DB_LIST,
+      this._databaseItems.map((item) => item.getPersistedState()),
+    );
+  }
+
+  private isExtensionControlledLocation(uri: vscode.Uri) {
+    const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri;
+    if (storageUri) {
+      return containsPath(storageUri.fsPath, uri.fsPath, process.platform);
+    }
+    return false;
+  }
+
+  private async getPrimaryLanguage(dbPath: string) {
+    const dbInfo = await this.cli.resolveDatabase(dbPath);
+    return dbInfo.languages?.[0] || "";
+  }
+}

From d608c057a4669bf1cdd249245625dc35c38ff505 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 17:03:14 +0200
Subject: [PATCH 058/119] Remove unused getUpgradesDirectories function

---
 .../ql-vscode/src/databases/local-databases.ts    | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
index 194286b30..038afb056 100644
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ b/extensions/ql-vscode/src/databases/local-databases.ts
@@ -1,6 +1,3 @@
-import { dirname } from "path";
-import * as vscode from "vscode";
-
 export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents";
 export {
   DatabaseChangedEvent,
@@ -19,15 +16,3 @@ export { DatabaseResolver } from "./local-databases/database-resolver";
  * The source of truth of the current state resides inside the
  * `DatabaseManager` class below.
  */
-
-/**
- * Get the set of directories containing upgrades, given a list of
- * scripts returned by the cli's upgrade resolution.
- */
-export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] {
-  const parentDirs = scripts.map((dir) => dirname(dir));
-  const uniqueParentDirs = new Set(parentDirs);
-  return Array.from(uniqueParentDirs).map((filePath) =>
-    vscode.Uri.file(filePath),
-  );
-}

From 6b9c3491ec4324d6c2f65b9d31aeda261862487c Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 17:04:14 +0200
Subject: [PATCH 059/119] Move local-databases.ts to local-databases/index.ts

---
 .../ql-vscode/src/databases/local-databases.ts | 18 ------------------
 .../src/databases/local-databases/index.ts     | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 18 deletions(-)
 delete mode 100644 extensions/ql-vscode/src/databases/local-databases.ts
 create mode 100644 extensions/ql-vscode/src/databases/local-databases/index.ts

diff --git a/extensions/ql-vscode/src/databases/local-databases.ts b/extensions/ql-vscode/src/databases/local-databases.ts
deleted file mode 100644
index 038afb056..000000000
--- a/extensions/ql-vscode/src/databases/local-databases.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-export { DatabaseContentsWithDbScheme } from "./local-databases/database-contents";
-export {
-  DatabaseChangedEvent,
-  DatabaseEventKind,
-} from "./local-databases/database-events";
-export { DatabaseItem } from "./local-databases/database-item";
-export { DatabaseManager } from "./local-databases/database-manager";
-export { DatabaseResolver } from "./local-databases/database-resolver";
-
-/**
- * databases.ts
- * ------------
- * Managing state of what the current database is, and what other
- * databases have been recently selected.
- *
- * The source of truth of the current state resides inside the
- * `DatabaseManager` class below.
- */
diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts
new file mode 100644
index 000000000..5a8e994db
--- /dev/null
+++ b/extensions/ql-vscode/src/databases/local-databases/index.ts
@@ -0,0 +1,15 @@
+export { DatabaseContentsWithDbScheme } from "./database-contents";
+export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
+export { DatabaseItem } from "./database-item";
+export { DatabaseManager } from "./database-manager";
+export { DatabaseResolver } from "./database-resolver";
+
+/**
+ * databases.ts
+ * ------------
+ * Managing state of what the current database is, and what other
+ * databases have been recently selected.
+ *
+ * The source of truth of the current state resides inside the
+ * `DatabaseManager` class below.
+ */

From c004f187208bb848ceb96cd45fe16cfc6565c3cf Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 17:04:36 +0200
Subject: [PATCH 060/119] Remove outdated comment

---
 .../ql-vscode/src/databases/local-databases/index.ts   | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts
index 5a8e994db..44c67a080 100644
--- a/extensions/ql-vscode/src/databases/local-databases/index.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/index.ts
@@ -3,13 +3,3 @@ export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 export { DatabaseItem } from "./database-item";
 export { DatabaseManager } from "./database-manager";
 export { DatabaseResolver } from "./database-resolver";
-
-/**
- * databases.ts
- * ------------
- * Managing state of what the current database is, and what other
- * databases have been recently selected.
- *
- * The source of truth of the current state resides inside the
- * `DatabaseManager` class below.
- */

From 070715560362875b9dd6ba2529ec43c5100d6fa6 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 24 May 2023 17:08:00 +0200
Subject: [PATCH 061/119] Fix imports for tests

---
 .../ql-vscode/src/databases/local-databases/index.ts      | 8 +++++++-
 .../minimal-workspace/local-databases.test.ts             | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/local-databases/index.ts b/extensions/ql-vscode/src/databases/local-databases/index.ts
index 44c67a080..fbca66f64 100644
--- a/extensions/ql-vscode/src/databases/local-databases/index.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/index.ts
@@ -1,5 +1,11 @@
-export { DatabaseContentsWithDbScheme } from "./database-contents";
+export {
+  DatabaseContents,
+  DatabaseContentsWithDbScheme,
+  DatabaseKind,
+} from "./database-contents";
 export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 export { DatabaseItem } from "./database-item";
+export { DatabaseItemImpl } from "./database-item-impl";
 export { DatabaseManager } from "./database-manager";
 export { DatabaseResolver } from "./database-resolver";
+export { DatabaseOptions, FullDatabaseOptions } from "./database-options";
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts
index 034c86f66..00bad35c2 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts
@@ -9,7 +9,6 @@ import {
   DatabaseItemImpl,
   DatabaseManager,
   DatabaseResolver,
-  findSourceArchive,
   FullDatabaseOptions,
 } from "../../../src/databases/local-databases";
 import { Logger } from "../../../src/common";
@@ -32,6 +31,7 @@ import {
   mockDbOptions,
   sourceLocationUri,
 } from "../../factories/databases/databases";
+import { findSourceArchive } from "../../../src/databases/local-databases/database-resolver";
 
 describe("local databases", () => {
   let databaseManager: DatabaseManager;

From 49f3f5673d70204d5d7bd6ec15db2938481a641d Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 24 May 2023 16:26:08 +0000
Subject: [PATCH 062/119] Show progress information when sending search api
 request

---
 .../ql-vscode/src/databases/ui/db-panel.ts    | 45 ++++++++++++++-----
 .../variant-analysis/gh-api/gh-api-client.ts  | 23 ++++++++--
 2 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index 61daf987c..dcf1cf453 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -1,4 +1,5 @@
 import {
+  ProgressLocation,
   QuickPickItem,
   TreeView,
   TreeViewExpansionEvent,
@@ -13,7 +14,10 @@ import {
   getOwnerFromGitHubUrl,
   isValidGitHubOwner,
 } from "../../common/github-url-identifier-helper";
-import { showAndLogErrorMessage } from "../../helpers";
+import {
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+} from "../../helpers";
 import { DisposableObject } from "../../pure/disposable-object";
 import {
   DbItem,
@@ -343,6 +347,8 @@ export class DbPanel extends DisposableObject {
       throw new Error("Please select a valid list to add code search results.");
     }
 
+    const listName = treeViewItem.dbItem.listName;
+
     const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values(
       QueryLanguage,
     ).map((language) => ({
@@ -375,18 +381,33 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    const repositories = await getCodeSearchRepositories(
-      this.app.credentials,
-      `${codeSearchQuery} language:${codeSearchLanguage.language}`,
-    );
+    void window.withProgress(
+      {
+        location: ProgressLocation.Notification,
+        title: "Searching for repositories... This might take a while",
+        cancellable: true,
+      },
+      async (progress, token) => {
+        progress.report({ increment: 10 });
 
-    const truncatedRepositories = await this.dbManager.addNewRemoteReposToList(
-      repositories,
-      treeViewItem.dbItem.listName,
-    );
-    this.truncatedReposNote(
-      truncatedRepositories,
-      treeViewItem.dbItem.listName,
+        const repositories = await getCodeSearchRepositories(
+          this.app.credentials,
+          `${codeSearchQuery} language:${codeSearchLanguage.language}`,
+          progress,
+          token,
+        );
+
+        token.onCancellationRequested(() => {
+          void showAndLogInformationMessage("Code search cancelled");
+          return;
+        });
+
+        progress.report({ increment: 10, message: "Processing results..." });
+
+        const truncatedRepositories =
+          await this.dbManager.addNewRemoteReposToList(repositories, listName);
+        this.truncatedReposNote(truncatedRepositories, listName);
+      },
     );
   }
 
diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
index 8c348d207..2746de16c 100644
--- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
+++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
@@ -7,21 +7,36 @@ import {
   VariantAnalysisSubmissionRequest,
 } from "./variant-analysis";
 import { Repository } from "./repository";
+import { Progress } from "vscode";
+import { CancellationToken } from "vscode-jsonrpc";
 
 export async function getCodeSearchRepositories(
   credentials: Credentials,
   query: string,
+  progress: Progress<{
+    message?: string | undefined;
+    increment?: number | undefined;
+  }>,
+  token: CancellationToken,
 ): Promise<string[]> {
+  let nwos: string[] = [];
   const octokit = await credentials.getOctokit();
-
-  const nwos = await octokit.paginate(
+  for await (const response of octokit.paginate.iterator(
     octokit.rest.search.repos,
     {
       q: query,
       per_page: 100,
     },
-    (response) => response.data.map((item) => item.full_name),
-  );
+  )) {
+    nwos.push(...response.data.map((item) => item.full_name));
+    const numberOfRequests = Math.ceil(response.data.total_count / 99);
+    const increment = numberOfRequests < 10 ? 80 / numberOfRequests : 8;
+    progress.report({ increment });
+    if (token.isCancellationRequested) {
+      nwos = [];
+      break;
+    }
+  }
 
   return [...new Set(nwos)];
 }

From b4506cf6e50a99799dd87df99c911809234bcc20 Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Wed, 24 May 2023 21:46:54 +0000
Subject: [PATCH 063/119] Use the original pack name for generated pack

Generated variant analysis packs will use the original name
of the pack that the query is located in. This is to support
some future work where we do extra validation of data extensions.

If the query is not in a pack, the default name is used.
---
 .../ql-vscode/src/variant-analysis/run-remote-query.ts    | 3 ++-
 .../variant-analysis/variant-analysis-manager.test.ts     | 8 +++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
index aa6285cfe..c8d493bf0 100644
--- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
+++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
@@ -379,7 +379,8 @@ async function fixPackFile(
   }
   const qlpack = load(await readFile(packPath, "utf8")) as QlPack;
 
-  qlpack.name = QUERY_PACK_NAME;
+  // Use original name
+  // qlpack.name = QUERY_PACK_NAME;
   updateDefaultSuite(qlpack, packRelativePath);
   removeWorkspaceRefs(qlpack);
 
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
index 8d8bc9aed..733553e5f 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
@@ -222,6 +222,7 @@ describe("Variant Analysis Manager", () => {
       it("should run a remote query that is part of a qlpack", async () => {
         await doVariantAnalysisTest({
           queryPath: "data-remote-qlpack/in-pack.ql",
+          expectedPackName: "codeql-remote/query",
           filesThatExist: ["in-pack.ql", "lib.qll"],
           filesThatDoNotExist: [],
           qlxFilesThatExist: ["in-pack.qlx"],
@@ -231,6 +232,7 @@ describe("Variant Analysis Manager", () => {
       it("should run a remote query that is not part of a qlpack", async () => {
         await doVariantAnalysisTest({
           queryPath: "data-remote-no-qlpack/in-pack.ql",
+          expectedPackName: "",
           filesThatExist: ["in-pack.ql"],
           filesThatDoNotExist: ["lib.qll", "not-in-pack.ql"],
           qlxFilesThatExist: ["in-pack.qlx"],
@@ -240,6 +242,7 @@ describe("Variant Analysis Manager", () => {
       it("should run a remote query that is nested inside a qlpack", async () => {
         await doVariantAnalysisTest({
           queryPath: "data-remote-qlpack-nested/subfolder/in-pack.ql",
+          expectedPackName: "github/remote-query-pack",
           filesThatExist: ["subfolder/in-pack.ql", "otherfolder/lib.qll"],
           filesThatDoNotExist: ["subfolder/not-in-pack.ql"],
           qlxFilesThatExist: ["subfolder/in-pack.qlx"],
@@ -256,6 +259,7 @@ describe("Variant Analysis Manager", () => {
         await cli.setUseExtensionPacks(true);
         await doVariantAnalysisTest({
           queryPath: "data-remote-qlpack-nested/subfolder/in-pack.ql",
+          expectedPackName: "github/remote-query-pack",
           filesThatExist: [
             "subfolder/in-pack.ql",
             "otherfolder/lib.qll",
@@ -273,12 +277,14 @@ describe("Variant Analysis Manager", () => {
 
     async function doVariantAnalysisTest({
       queryPath,
+      expectedPackName,
       filesThatExist,
       qlxFilesThatExist,
       filesThatDoNotExist,
       dependenciesToCheck = ["codeql/javascript-all"],
     }: {
       queryPath: string;
+      expectedPackName: string;
       filesThatExist: string[];
       qlxFilesThatExist: string[];
       filesThatDoNotExist: string[];
@@ -332,7 +338,7 @@ describe("Variant Analysis Manager", () => {
       const qlpackContents = load(
         packFS.fileContents(packFileName).toString("utf-8"),
       );
-      expect(qlpackContents.name).toEqual("codeql-remote/query");
+      expect(qlpackContents.name).toEqual(expectedPackName);
       expect(qlpackContents.version).toEqual("0.0.0");
       expect(qlpackContents.dependencies?.["codeql/javascript-all"]).toEqual(
         "*",

From 301149fd323d9f3ddf75969278bcf6b9a791f3d3 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 24 May 2023 17:49:41 +0100
Subject: [PATCH 064/119] Don't create array entry if workspace contains no
 queries

---
 .../src/queries-panel/query-discovery.ts         | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index 00322d634..b9369a3a3 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -99,22 +99,26 @@ export class QueryDiscovery
   ): Promise<FileTreeDirectory[]> {
     const rootDirectories = [];
     for (const workspaceFolder of workspaceFolders) {
-      rootDirectories.push(
-        await this.discoverQueriesInWorkspace(workspaceFolder),
-      );
+      const root = await this.discoverQueriesInWorkspace(workspaceFolder);
+      if (root !== undefined) {
+        rootDirectories.push(root);
+      }
     }
     return rootDirectories;
   }
 
   private async discoverQueriesInWorkspace(
     workspaceFolder: WorkspaceFolder,
-  ): Promise<FileTreeDirectory> {
+  ): Promise<FileTreeDirectory | undefined> {
     const fullPath = workspaceFolder.uri.fsPath;
     const name = workspaceFolder.name;
 
-    const rootDirectory = new FileTreeDirectory(fullPath, name);
-
     const resolvedQueries = await this.cliServer.resolveQueries(fullPath);
+    if (resolvedQueries.length === 0) {
+      return undefined;
+    }
+
+    const rootDirectory = new FileTreeDirectory(fullPath, name);
     for (const queryPath of resolvedQueries) {
       const relativePath = normalize(relative(fullPath, queryPath));
       const dirName = dirname(relativePath);

From f236e65f684697d836d7c239f112d3a17df9af92 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 24 May 2023 16:23:25 +0100
Subject: [PATCH 065/119] Add integration tests of QueryDiscovery

---
 .../src/queries-panel/query-discovery.ts      |   2 +-
 .../queries-panel/query-discovery.test.ts     | 221 ++++++++++++++++++
 2 files changed, 222 insertions(+), 1 deletion(-)
 create mode 100644 extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts

diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index b9369a3a3..bde646fbb 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -12,7 +12,7 @@ import { QueryDiscoverer } from "./query-tree-data-provider";
 /**
  * The results of discovering queries.
  */
-interface QueryDiscoveryResults {
+export interface QueryDiscoveryResults {
   /**
    * A tree of directories and query files.
    * May have multiple roots because of multiple workspaces.
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
new file mode 100644
index 000000000..0496f8712
--- /dev/null
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
@@ -0,0 +1,221 @@
+import {
+  EventEmitter,
+  FileSystemWatcher,
+  Uri,
+  WorkspaceFoldersChangeEvent,
+  workspace,
+} from "vscode";
+import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
+import {
+  QueryDiscovery,
+  QueryDiscoveryResults,
+} from "../../../../src/queries-panel/query-discovery";
+import { createMockApp } from "../../../__mocks__/appMock";
+import { mockedObject } from "../../utils/mocking.helpers";
+import { basename, join } from "path";
+import { sleep } from "../../../../src/pure/time";
+
+describe("QueryDiscovery", () => {
+  beforeEach(() => {
+    expect(workspace.workspaceFolders?.length).toEqual(1);
+  });
+
+  describe("queries", () => {
+    it("should return empty list when no QL files are present", async () => {
+      const resolveQueries = jest.fn().mockResolvedValue([]);
+      const cli = mockedObject<CodeQLCliServer>({
+        resolveQueries,
+      });
+
+      const discovery = new QueryDiscovery(createMockApp({}), cli);
+      const results: QueryDiscoveryResults = await (
+        discovery as any
+      ).discover();
+
+      expect(results.queries).toEqual([]);
+      expect(resolveQueries).toHaveBeenCalledTimes(1);
+    });
+
+    it("should organise query files into directories", async () => {
+      const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath;
+      const cli = mockedObject<CodeQLCliServer>({
+        resolveQueries: jest
+          .fn()
+          .mockResolvedValue([
+            join(workspaceRoot, "dir1/query1.ql"),
+            join(workspaceRoot, "dir2/query2.ql"),
+            join(workspaceRoot, "query3.ql"),
+          ]),
+      });
+
+      const discovery = new QueryDiscovery(createMockApp({}), cli);
+      const results: QueryDiscoveryResults = await (
+        discovery as any
+      ).discover();
+
+      expect(results.queries[0].children.length).toEqual(3);
+      expect(results.queries[0].children[0].name).toEqual("dir1");
+      expect(results.queries[0].children[0].children.length).toEqual(1);
+      expect(results.queries[0].children[0].children[0].name).toEqual(
+        "query1.ql",
+      );
+      expect(results.queries[0].children[1].name).toEqual("dir2");
+      expect(results.queries[0].children[1].children.length).toEqual(1);
+      expect(results.queries[0].children[1].children[0].name).toEqual(
+        "query2.ql",
+      );
+      expect(results.queries[0].children[2].name).toEqual("query3.ql");
+    });
+
+    it("should collapse directories containing only a single element", async () => {
+      const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath;
+      const cli = mockedObject<CodeQLCliServer>({
+        resolveQueries: jest
+          .fn()
+          .mockResolvedValue([
+            join(workspaceRoot, "dir1/query1.ql"),
+            join(workspaceRoot, "dir1/dir2/dir3/dir3/query2.ql"),
+          ]),
+      });
+
+      const discovery = new QueryDiscovery(createMockApp({}), cli);
+      const results: QueryDiscoveryResults = await (
+        discovery as any
+      ).discover();
+
+      expect(results.queries[0].children.length).toEqual(1);
+      expect(results.queries[0].children[0].name).toEqual("dir1");
+      expect(results.queries[0].children[0].children.length).toEqual(2);
+      expect(results.queries[0].children[0].children[0].name).toEqual(
+        "dir2 / dir3 / dir3",
+      );
+      expect(
+        results.queries[0].children[0].children[0].children.length,
+      ).toEqual(1);
+      expect(
+        results.queries[0].children[0].children[0].children[0].name,
+      ).toEqual("query2.ql");
+      expect(results.queries[0].children[0].children[1].name).toEqual(
+        "query1.ql",
+      );
+    });
+
+    it("calls resolveQueries once for each workspace", async () => {
+      const workspaceRoots = ["/workspace1", "/workspace2", "/workspace3"];
+      jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValueOnce(
+        workspaceRoots.map((root, index) => ({
+          uri: Uri.file(root),
+          name: basename(root),
+          index,
+        })),
+      );
+
+      const resolveQueries = jest.fn().mockImplementation((queryDir) => {
+        const workspaceIndex = workspaceRoots.indexOf(queryDir);
+        if (workspaceIndex === -1) {
+          throw new Error("Unexpected workspace");
+        }
+        return Promise.resolve([
+          join(queryDir, `query${workspaceIndex + 1}.ql`),
+        ]);
+      });
+      const cli = mockedObject<CodeQLCliServer>({
+        resolveQueries,
+      });
+
+      const discovery = new QueryDiscovery(createMockApp({}), cli);
+      const results: QueryDiscoveryResults = await (
+        discovery as any
+      ).discover();
+
+      expect(results.queries.length).toEqual(3);
+      expect(results.queries[0].children[0].name).toEqual("query1.ql");
+      expect(results.queries[1].children[0].name).toEqual("query2.ql");
+      expect(results.queries[2].children[0].name).toEqual("query3.ql");
+
+      expect(resolveQueries).toHaveBeenCalledTimes(3);
+    });
+  });
+
+  describe("onDidChangeQueries", () => {
+    it("should fire onDidChangeQueries when a watcher fires", async () => {
+      const onWatcherDidChangeEvent = new EventEmitter<Uri>();
+      const watcher: FileSystemWatcher = {
+        ignoreCreateEvents: false,
+        ignoreChangeEvents: false,
+        ignoreDeleteEvents: false,
+        onDidCreate: onWatcherDidChangeEvent.event,
+        onDidChange: onWatcherDidChangeEvent.event,
+        onDidDelete: onWatcherDidChangeEvent.event,
+        dispose: () => undefined,
+      };
+      const createFileSystemWatcherSpy = jest.spyOn(
+        workspace,
+        "createFileSystemWatcher",
+      );
+      createFileSystemWatcherSpy.mockReturnValue(watcher);
+
+      const workspaceRoot = workspace.workspaceFolders![0].uri.fsPath;
+      const cli = mockedObject<CodeQLCliServer>({
+        resolveQueries: jest
+          .fn()
+          .mockResolvedValue([join(workspaceRoot, "query1.ql")]),
+      });
+
+      const discovery = new QueryDiscovery(
+        createMockApp({
+          createEventEmitter: () => new EventEmitter(),
+        }),
+        cli,
+      );
+
+      const onDidChangeQueriesSpy = jest.fn();
+      discovery.onDidChangeQueries(onDidChangeQueriesSpy);
+
+      const results = await (discovery as any).discover();
+      (discovery as any).update(results);
+
+      expect(createFileSystemWatcherSpy).toHaveBeenCalledTimes(2);
+      expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1);
+
+      onWatcherDidChangeEvent.fire(workspace.workspaceFolders![0].uri);
+
+      // Wait for refresh to finish
+      await sleep(100);
+
+      expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe("onDidChangeWorkspaceFolders", () => {
+    it("should refresh when workspace folders change", async () => {
+      const onDidChangeWorkspaceFoldersEvent =
+        new EventEmitter<WorkspaceFoldersChangeEvent>();
+
+      const discovery = new QueryDiscovery(
+        createMockApp({
+          createEventEmitter: () => new EventEmitter(),
+          onDidChangeWorkspaceFolders: onDidChangeWorkspaceFoldersEvent.event,
+        }),
+        mockedObject<CodeQLCliServer>({
+          resolveQueries: jest.fn().mockResolvedValue([]),
+        }),
+      );
+
+      const onDidChangeQueriesSpy = jest.fn();
+      discovery.onDidChangeQueries(onDidChangeQueriesSpy);
+
+      const results = await (discovery as any).discover();
+      (discovery as any).update(results);
+
+      expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1);
+
+      onDidChangeWorkspaceFoldersEvent.fire({ added: [], removed: [] });
+
+      // Wait for refresh to finish
+      await sleep(100);
+
+      expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
+    });
+  });
+});

From 2e12b8d756a32d96e22164d064201689930f70c1 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Thu, 25 May 2023 09:11:37 +0000
Subject: [PATCH 066/119] Rename command from importCodeSearch to
 importFromCodeSearch

---
 extensions/ql-vscode/package.json                 | 6 +++---
 extensions/ql-vscode/src/common/commands.ts       | 2 +-
 extensions/ql-vscode/src/databases/ui/db-panel.ts | 8 +++++---
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json
index b6ba2b393..997073dd7 100644
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@@ -517,7 +517,7 @@
         "icon": "$(new-folder)"
       },
       {
-        "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+        "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
         "title": "Import Repos from GitHub Code Search"
       },
       {
@@ -966,7 +966,7 @@
           "group": "2_qlContextMenu@1"
         },
         {
-          "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
           "group": "2_qlContextMenu@1"
         },
@@ -1307,7 +1307,7 @@
           "when": "false"
         },
         {
-          "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
           "when": "false"
         },
         {
diff --git a/extensions/ql-vscode/src/common/commands.ts b/extensions/ql-vscode/src/common/commands.ts
index 15d566bfc..74d5560af 100644
--- a/extensions/ql-vscode/src/common/commands.ts
+++ b/extensions/ql-vscode/src/common/commands.ts
@@ -275,7 +275,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
-  "codeQLVariantAnalysisRepositories.importCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {
diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index dcf1cf453..6414871f4 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -103,8 +103,8 @@ export class DbPanel extends DisposableObject {
         this.renameItem.bind(this),
       "codeQLVariantAnalysisRepositories.removeItemContextMenu":
         this.removeItem.bind(this),
-      "codeQLVariantAnalysisRepositories.importCodeSearch":
-        this.importCodeSearch.bind(this),
+      "codeQLVariantAnalysisRepositories.importFromCodeSearch":
+        this.importFromCodeSearch.bind(this),
     };
   }
 
@@ -342,7 +342,9 @@ export class DbPanel extends DisposableObject {
     await this.dbManager.removeDbItem(treeViewItem.dbItem);
   }
 
-  private async importCodeSearch(treeViewItem: DbTreeViewItem): Promise<void> {
+  private async importFromCodeSearch(
+    treeViewItem: DbTreeViewItem,
+  ): Promise<void> {
     if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) {
       throw new Error("Please select a valid list to add code search results.");
     }

From 250089a9e394502ad011f24aa932851ae11563d9 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Thu, 25 May 2023 10:02:22 +0000
Subject: [PATCH 067/119] Renaming and smaller merge comments

---
 .../ql-vscode/src/databases/config/db-config-store.ts |  2 +-
 extensions/ql-vscode/src/databases/ui/db-panel.ts     | 10 ++++------
 .../databases/config/db-config-store.test.ts          | 11 ++++++-----
 .../test/unit-tests/databases/db-manager.test.ts      |  4 ++--
 4 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts
index ef62c61b3..49e5c5288 100644
--- a/extensions/ql-vscode/src/databases/config/db-config-store.ts
+++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts
@@ -163,7 +163,7 @@ export class DbConfigStore extends DisposableObject {
 
     // Remove duplicates from the list of repositories.
     const newRepositoriesList = [
-      ...new Set([...new Set(parent.repositories), ...new Set(repoNwoList)]),
+      ...new Set([...parent.repositories, ...repoNwoList]),
     ];
 
     parent.repositories = newRepositoriesList.slice(0, 1000);
diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index 6414871f4..c649d8619 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -189,7 +189,7 @@ export class DbPanel extends DisposableObject {
     );
 
     if (parentList) {
-      this.truncatedReposNote(truncatedRepositories, parentList);
+      this.reportAnyTruncatedRepos(truncatedRepositories, parentList);
     }
   }
 
@@ -369,9 +369,7 @@ export class DbPanel extends DisposableObject {
         },
       );
     if (!codeSearchLanguage) {
-      // We don't need to display a warning pop-up in this case, since the user just escaped out of the operation.
-      // We set 'true' to make this a silent exception.
-      throw new UserCancellationException("No language selected", true);
+      return;
     }
 
     const codeSearchQuery = await window.showInputBox({
@@ -408,12 +406,12 @@ export class DbPanel extends DisposableObject {
 
         const truncatedRepositories =
           await this.dbManager.addNewRemoteReposToList(repositories, listName);
-        this.truncatedReposNote(truncatedRepositories, listName);
+        this.reportAnyTruncatedRepos(truncatedRepositories, listName);
       },
     );
   }
 
-  private truncatedReposNote(
+  private reportAnyTruncatedRepos(
     truncatedRepositories: string[],
     listName: string,
   ) {
diff --git a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
index 6673fb07a..b7a3fd997 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/config/db-config-store.test.ts
@@ -262,7 +262,7 @@ describe("db config store", () => {
       });
 
       // Add
-      await configStore.addRemoteReposToList(
+      const response = await configStore.addRemoteReposToList(
         ["owner/repo1", "owner/repo2"],
         "list1",
       );
@@ -278,11 +278,12 @@ describe("db config store", () => {
         name: "list1",
         repositories: ["owner/repo1", "owner/repo2"],
       });
+      expect(response).toEqual([]);
 
       configStore.dispose();
     });
 
-    it("should add no more than 1000 repositories to a remote list using #addRemoteReposToList", async () => {
+    it("should add no more than 1000 repositories to a remote list when adding multiple repos", async () => {
       // Initial set up
       const dbConfig = createDbConfig({
         remoteLists: [
@@ -296,7 +297,7 @@ describe("db config store", () => {
       const configStore = await initializeConfig(dbConfig, configPath, app);
 
       // Add
-      const reponse = await configStore.addRemoteReposToList(
+      const response = await configStore.addRemoteReposToList(
         [...Array(1001).keys()].map((i) => `owner/db${i}`),
         "list1",
       );
@@ -311,12 +312,12 @@ describe("db config store", () => {
       expect(updatedRemoteDbs.repositoryLists[0].repositories).toHaveLength(
         1000,
       );
-      expect(reponse).toEqual(["owner/db1000"]);
+      expect(response).toEqual(["owner/db1000"]);
 
       configStore.dispose();
     });
 
-    it("should add no more than 1000 repositories to a remote list using #addRemoteRepo", async () => {
+    it("should add no more than 1000 repositories to a remote list when adding one repo", async () => {
       // Initial set up
       const dbConfig = createDbConfig({
         remoteLists: [
diff --git a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
index ff0bc8ba6..60c87c356 100644
--- a/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/databases/db-manager.test.ts
@@ -115,7 +115,7 @@ describe("db manager", () => {
         });
       });
 
-      it("should return truncated repos when adding to a user defined list using #addNewRemoteReposToList", async () => {
+      it("should return truncated repos when adding multiple repos to a user defined list", async () => {
         const dbConfig: DbConfig = createDbConfig({
           remoteLists: [
             {
@@ -135,7 +135,7 @@ describe("db manager", () => {
         expect(response).toEqual(["owner2/repo2"]);
       });
 
-      it("should return truncated repos when adding to a user defined list using #addNewRemoteRepo", async () => {
+      it("should return truncated repos when adding one repo to a user defined list", async () => {
         const dbConfig: DbConfig = createDbConfig({
           remoteLists: [
             {

From a88e683ebf1d585f624a7f133fb283840c17f279 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Thu, 25 May 2023 13:21:20 +0200
Subject: [PATCH 068/119] Add initial implementation of auto-modeling

---
 extensions/ql-vscode/src/config.ts            |   7 +
 .../data-extensions-editor/auto-model-api.ts  |  54 ++++
 .../src/data-extensions-editor/auto-model.ts  | 117 ++++++++
 .../data-extensions-editor-view.ts            |  48 +++
 .../shared/view-state.ts                      |   1 +
 .../ql-vscode/src/pure/interface-types.ts     |   9 +-
 .../DataExtensionsEditor.stories.tsx          |   1 +
 .../DataExtensionsEditor.tsx                  |  16 +
 .../data-extensions-editor/auto-model.test.ts | 283 ++++++++++++++++++
 9 files changed, 535 insertions(+), 1 deletion(-)
 create mode 100644 extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts
 create mode 100644 extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
 create mode 100644 extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts

diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts
index bb00a1a4b..48c168773 100644
--- a/extensions/ql-vscode/src/config.ts
+++ b/extensions/ql-vscode/src/config.ts
@@ -711,3 +711,10 @@ const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
 export function showQueriesPanel(): boolean {
   return !!QUERIES_PANEL.getValue<boolean>();
 }
+
+const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
+const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
+
+export function showLlmGeneration(): boolean {
+  return !!LLM_GENERATION.getValue<boolean>();
+}
diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts
new file mode 100644
index 000000000..671c71096
--- /dev/null
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts
@@ -0,0 +1,54 @@
+import { Credentials } from "../common/authentication";
+import { OctokitResponse } from "@octokit/types";
+
+export enum ClassificationType {
+  Unknown = "CLASSIFICATION_TYPE_UNKNOWN",
+  Neutral = "CLASSIFICATION_TYPE_NEUTRAL",
+  Source = "CLASSIFICATION_TYPE_SOURCE",
+  Sink = "CLASSIFICATION_TYPE_SINK",
+  Summary = "CLASSIFICATION_TYPE_SUMMARY",
+}
+
+export interface Classification {
+  type: ClassificationType;
+  kind: string;
+  explanation: string;
+}
+
+export interface Method {
+  package: string;
+  type: string;
+  name: string;
+  signature: string;
+  usages: string[];
+  classification?: Classification;
+  input?: string;
+  output?: string;
+}
+
+export interface ModelRequest {
+  language: string;
+  candidates: Method[];
+  samples: Method[];
+}
+
+export interface ModelResponse {
+  language: string;
+  predicted: Method[];
+}
+
+export async function autoModel(
+  credentials: Credentials,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<ModelResponse> = await octokit.request(
+    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+    {
+      data: request,
+    },
+  );
+
+  return response.data;
+}
diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
new file mode 100644
index 000000000..16d14ed75
--- /dev/null
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -0,0 +1,117 @@
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+import {
+  Classification,
+  ClassificationType,
+  Method,
+  ModelRequest,
+} from "./auto-model-api";
+
+export function createAutoModelRequest(
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+): ModelRequest {
+  const request: ModelRequest = {
+    language,
+    samples: [],
+    candidates: [],
+  };
+
+  // Sort by number of usages so we always send the most used methods first
+  externalApiUsages = [...externalApiUsages];
+  externalApiUsages.sort((a, b) => b.usages.length - a.usages.length);
+
+  for (const externalApiUsage of externalApiUsages) {
+    const modeledMethod: ModeledMethod = modeledMethods[
+      externalApiUsage.signature
+    ] ?? {
+      type: "none",
+    };
+
+    const numberOfArguments =
+      externalApiUsage.methodParameters === "()"
+        ? 0
+        : externalApiUsage.methodParameters.split(",").length;
+
+    for (
+      let argumentIndex = 0;
+      argumentIndex < numberOfArguments;
+      argumentIndex++
+    ) {
+      const method: Method = {
+        package: externalApiUsage.packageName,
+        type: externalApiUsage.typeName,
+        name: externalApiUsage.methodName,
+        signature: externalApiUsage.methodParameters,
+        classification:
+          modeledMethod.type === "none"
+            ? undefined
+            : toMethodClassification(modeledMethod),
+        usages: externalApiUsage.usages.map((usage) => usage.label),
+        input: `Argument[${argumentIndex}]`,
+      };
+
+      if (method.usages.length > 10) {
+        method.usages = method.usages.slice(0, 10);
+      }
+
+      if (modeledMethod.type === "none") {
+        request.candidates.push(method);
+      } else {
+        request.samples.push(method);
+      }
+    }
+  }
+
+  if (request.candidates.length > 100) {
+    request.candidates = request.candidates.slice(0, 100);
+  }
+  if (request.samples.length > 20) {
+    request.samples = request.samples.slice(0, 20);
+  }
+
+  return request;
+}
+
+function toMethodClassificationType(
+  type: ModeledMethodType,
+): ClassificationType {
+  switch (type) {
+    case "source":
+      return ClassificationType.Source;
+    case "sink":
+      return ClassificationType.Sink;
+    case "summary":
+      return ClassificationType.Summary;
+    case "neutral":
+      return ClassificationType.Neutral;
+    default:
+      return ClassificationType.Unknown;
+  }
+}
+
+function toMethodClassification(modeledMethod: ModeledMethod): Classification {
+  return {
+    type: toMethodClassificationType(modeledMethod.type),
+    kind: modeledMethod.kind,
+    explanation: "",
+  };
+}
+
+export function classificationTypeToModeledMethodType(
+  type: ClassificationType,
+): ModeledMethodType {
+  switch (type) {
+    case ClassificationType.Source:
+      return "source";
+    case ClassificationType.Sink:
+      return "sink";
+    case ClassificationType.Summary:
+      return "summary";
+    case ClassificationType.Neutral:
+      return "neutral";
+    default:
+      return "none";
+  }
+}
diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
index c60983129..4adb5b392 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
@@ -39,6 +39,12 @@ import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
 import { ExternalApiUsage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
 import { ExtensionPackModelFile } from "./shared/extension-pack";
+import { autoModel } from "./auto-model-api";
+import {
+  classificationTypeToModeledMethodType,
+  createAutoModelRequest,
+} from "./auto-model";
+import { showLlmGeneration } from "../config";
 
 function getQlSubmoduleFolder(): WorkspaceFolder | undefined {
   const workspaceFolder = workspace.workspaceFolders?.find(
@@ -127,6 +133,13 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "generateExternalApi":
         await this.generateModeledMethods();
 
+        break;
+      case "generateExternalApiFromLlm":
+        await this.generateModeledMethodsFromLlm(
+          msg.externalApiUsages,
+          msg.modeledMethods,
+        );
+
         break;
       default:
         assertNever(msg);
@@ -149,6 +162,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
       viewState: {
         extensionPackModelFile: this.modelFile,
         modelFileExists: await pathExists(this.modelFile.filename),
+        showLlmButton: showLlmGeneration(),
       },
     });
   }
@@ -367,6 +381,40 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.clearProgress();
   }
 
+  private async generateModeledMethodsFromLlm(
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+  ): Promise<void> {
+    const request = createAutoModelRequest(
+      this.databaseItem.language,
+      externalApiUsages,
+      modeledMethods,
+    );
+
+    const response = await autoModel(this.app.credentials, request);
+
+    const modeledMethodsByName: Record<string, ModeledMethod> = {};
+
+    for (const method of response.predicted) {
+      if (method.classification === undefined) {
+        continue;
+      }
+
+      modeledMethodsByName[method.signature] = {
+        type: classificationTypeToModeledMethodType(method.classification.type),
+        kind: method.classification.kind,
+        input: method.input ?? "",
+        output: method.output ?? "",
+      };
+    }
+
+    await this.postMessage({
+      t: "addModeledMethods",
+      modeledMethods: modeledMethodsByName,
+      overrideNone: true,
+    });
+  }
+
   /*
    * Progress in this class is a bit weird. Most of the progress is based on running the query.
    * Query progress is always between 0 and 1000. However, we still have some steps that need
diff --git a/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts b/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts
index 0da3f0d25..ece8af174 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts
@@ -3,4 +3,5 @@ import { ExtensionPackModelFile } from "./extension-pack";
 export interface DataExtensionEditorViewState {
   extensionPackModelFile: ExtensionPackModelFile;
   modelFileExists: boolean;
+  showLlmButton: boolean;
 }
diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts
index c04644b73..a71591745 100644
--- a/extensions/ql-vscode/src/pure/interface-types.ts
+++ b/extensions/ql-vscode/src/pure/interface-types.ts
@@ -544,6 +544,12 @@ export interface GenerateExternalApiMessage {
   t: "generateExternalApi";
 }
 
+export interface GenerateExternalApiFromLlmMessage {
+  t: "generateExternalApiFromLlm";
+  externalApiUsages: ExternalApiUsage[];
+  modeledMethods: Record<string, ModeledMethod>;
+}
+
 export type ToDataExtensionsEditorMessage =
   | SetExtensionPackStateMessage
   | SetExternalApiUsagesMessage
@@ -556,4 +562,5 @@ export type FromDataExtensionsEditorMessage =
   | OpenExtensionPackMessage
   | JumpToUsageMessage
   | SaveModeledMethods
-  | GenerateExternalApiMessage;
+  | GenerateExternalApiMessage
+  | GenerateExternalApiFromLlmMessage;
diff --git a/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx b/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx
index 922aa38fa..789c0ff06 100644
--- a/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx
+++ b/extensions/ql-vscode/src/stories/data-extensions-editor/DataExtensionsEditor.stories.tsx
@@ -30,6 +30,7 @@ DataExtensionsEditor.args = {
         "/home/user/vscode-codeql-starter/codeql-custom-queries-java/sql2o/models/sql2o.yml",
     },
     modelFileExists: true,
+    showLlmButton: true,
   },
   initialExternalApiUsages: [
     {
diff --git a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx
index 592625e5f..4b09a8424 100644
--- a/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx
+++ b/extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx
@@ -157,6 +157,14 @@ export function DataExtensionsEditor({
     });
   }, []);
 
+  const onGenerateFromLlmClick = useCallback(() => {
+    vscode.postMessage({
+      t: "generateExternalApiFromLlm",
+      externalApiUsages,
+      modeledMethods,
+    });
+  }, [externalApiUsages, modeledMethods]);
+
   const onOpenExtensionPackClick = useCallback(() => {
     vscode.postMessage({
       t: "openExtensionPack",
@@ -214,6 +222,14 @@ export function DataExtensionsEditor({
             <VSCodeButton onClick={onGenerateClick}>
               Download and generate
             </VSCodeButton>
+            {viewState?.showLlmButton && (
+              <>
+                &nbsp;
+                <VSCodeButton onClick={onGenerateFromLlmClick}>
+                  Generate using LLM
+                </VSCodeButton>
+              </>
+            )}
             <br />
             <br />
             <VSCodeDataGrid>
diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
new file mode 100644
index 000000000..d56fcda07
--- /dev/null
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
@@ -0,0 +1,283 @@
+import { createAutoModelRequest } from "../../../src/data-extensions-editor/auto-model";
+import { ExternalApiUsage } from "../../../src/data-extensions-editor/external-api-usage";
+import { ModeledMethod } from "../../../src/data-extensions-editor/modeled-method";
+
+describe("createAutoModelRequest", () => {
+  const externalApiUsages: ExternalApiUsage[] = [
+    {
+      signature:
+        "org.springframework.boot.SpringApplication#run(Class,String[])",
+      packageName: "org.springframework.boot",
+      typeName: "SpringApplication",
+      methodName: "run",
+      methodParameters: "(Class,String[])",
+      supported: false,
+      usages: [
+        {
+          label: "run(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java",
+            startLine: 9,
+            startColumn: 9,
+            endLine: 9,
+            endColumn: 66,
+          },
+        },
+      ],
+    },
+    {
+      signature: "org.sql2o.Connection#createQuery(String)",
+      packageName: "org.sql2o",
+      typeName: "Connection",
+      methodName: "createQuery",
+      methodParameters: "(String)",
+      supported: true,
+      usages: [
+        {
+          label: "createQuery(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 15,
+            startColumn: 13,
+            endLine: 15,
+            endColumn: 56,
+          },
+        },
+        {
+          label: "createQuery(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 26,
+            startColumn: 13,
+            endLine: 26,
+            endColumn: 39,
+          },
+        },
+      ],
+    },
+    {
+      signature: "org.sql2o.Query#executeScalar(Class)",
+      packageName: "org.sql2o",
+      typeName: "Query",
+      methodName: "executeScalar",
+      methodParameters: "(Class)",
+      supported: true,
+      usages: [
+        {
+          label: "executeScalar(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 15,
+            startColumn: 13,
+            endLine: 15,
+            endColumn: 85,
+          },
+        },
+        {
+          label: "executeScalar(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 26,
+            startColumn: 13,
+            endLine: 26,
+            endColumn: 68,
+          },
+        },
+      ],
+    },
+    {
+      signature: "org.sql2o.Sql2o#open()",
+      packageName: "org.sql2o",
+      typeName: "Sql2o",
+      methodName: "open",
+      methodParameters: "()",
+      supported: true,
+      usages: [
+        {
+          label: "open(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 14,
+            startColumn: 24,
+            endLine: 14,
+            endColumn: 35,
+          },
+        },
+        {
+          label: "open(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 25,
+            startColumn: 24,
+            endLine: 25,
+            endColumn: 35,
+          },
+        },
+      ],
+    },
+    {
+      signature: "java.io.PrintStream#println(String)",
+      packageName: "java.io",
+      typeName: "PrintStream",
+      methodName: "println",
+      methodParameters: "(String)",
+      supported: true,
+      usages: [
+        {
+          label: "println(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 29,
+            startColumn: 9,
+            endLine: 29,
+            endColumn: 49,
+          },
+        },
+      ],
+    },
+    {
+      signature: "org.sql2o.Sql2o#Sql2o(String,String,String)",
+      packageName: "org.sql2o",
+      typeName: "Sql2o",
+      methodName: "Sql2o",
+      methodParameters: "(String,String,String)",
+      supported: true,
+      usages: [
+        {
+          label: "new Sql2o(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 10,
+            startColumn: 33,
+            endLine: 10,
+            endColumn: 88,
+          },
+        },
+      ],
+    },
+    {
+      signature: "org.sql2o.Sql2o#Sql2o(String)",
+      packageName: "org.sql2o",
+      typeName: "Sql2o",
+      methodName: "Sql2o",
+      methodParameters: "(String)",
+      supported: true,
+      usages: [
+        {
+          label: "new Sql2o(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 23,
+            startColumn: 23,
+            endLine: 23,
+            endColumn: 36,
+          },
+        },
+      ],
+    },
+  ];
+
+  const modeledMethods: Record<string, ModeledMethod> = {
+    "org.sql2o.Sql2o#open()": {
+      type: "neutral",
+      kind: "",
+      input: "",
+      output: "",
+    },
+    "org.sql2o.Sql2o#Sql2o(String)": {
+      type: "sink",
+      kind: "jndi-injection",
+      input: "Argument[0]",
+      output: "",
+    },
+  };
+
+  it("creates a matching request", () => {
+    expect(
+      createAutoModelRequest("java", externalApiUsages, modeledMethods),
+    ).toEqual({
+      language: "java",
+      samples: [
+        {
+          package: "org.sql2o",
+          type: "Sql2o",
+          name: "Sql2o",
+          signature: "(String)",
+          classification: {
+            type: "CLASSIFICATION_TYPE_SINK",
+            kind: "jndi-injection",
+            explanation: "",
+          },
+          usages: ["new Sql2o(...)"],
+          input: "Argument[0]",
+        },
+      ],
+      candidates: [
+        {
+          package: "org.sql2o",
+          type: "Connection",
+          name: "createQuery",
+          signature: "(String)",
+          usages: ["createQuery(...)", "createQuery(...)"],
+          input: "Argument[0]",
+        },
+        {
+          package: "org.sql2o",
+          type: "Query",
+          name: "executeScalar",
+          signature: "(Class)",
+          usages: ["executeScalar(...)", "executeScalar(...)"],
+          input: "Argument[0]",
+        },
+        {
+          package: "org.springframework.boot",
+          type: "SpringApplication",
+          name: "run",
+          signature: "(Class,String[])",
+          usages: ["run(...)"],
+          input: "Argument[0]",
+        },
+        {
+          package: "org.springframework.boot",
+          type: "SpringApplication",
+          name: "run",
+          signature: "(Class,String[])",
+          usages: ["run(...)"],
+          input: "Argument[1]",
+        },
+        {
+          package: "java.io",
+          type: "PrintStream",
+          name: "println",
+          signature: "(String)",
+          usages: ["println(...)"],
+          input: "Argument[0]",
+        },
+        {
+          package: "org.sql2o",
+          type: "Sql2o",
+          name: "Sql2o",
+          signature: "(String,String,String)",
+          usages: ["new Sql2o(...)"],
+          input: "Argument[0]",
+        },
+        {
+          package: "org.sql2o",
+          type: "Sql2o",
+          name: "Sql2o",
+          signature: "(String,String,String)",
+          usages: ["new Sql2o(...)"],
+          input: "Argument[1]",
+        },
+        {
+          package: "org.sql2o",
+          type: "Sql2o",
+          name: "Sql2o",
+          signature: "(String,String,String)",
+          usages: ["new Sql2o(...)"],
+          input: "Argument[2]",
+        },
+      ],
+    });
+  });
+});

From 394b51f4f3bd722d053182fa71c373755a9fad77 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 12:49:01 +0100
Subject: [PATCH 069/119] Add welcome text

---
 extensions/ql-vscode/package.json | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json
index 87788fe8d..89b9ef9ea 100644
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@@ -1593,6 +1593,10 @@
         "view": "codeQLQueryHistory",
         "contents": "You have no query history items at the moment.\n\nSelect a database to run a CodeQL query and get your first results."
       },
+      {
+        "view": "codeQLQueries",
+        "contents": "This workspace doesn't contain any CodeQL queries at the moment."
+      },
       {
         "view": "codeQLDatabases",
         "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)"

From 295a08f85a1d61dd526d3f33074f07b4bda1a2bd Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Thu, 25 May 2023 14:36:38 +0200
Subject: [PATCH 070/119] Use `DatabaseItemImpl` where possible in
 `DatabaseManager`

This will allow us to implement specific behavior on the
`DatabaseItemImpl` which is not available on the `DatabaseItem`. This
will allow us to make the surface area of the `DatabaseItem` smaller.
---
 .../databases/local-databases/database-manager.ts    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
index 1383b179e..29d910e1a 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
@@ -83,7 +83,7 @@ export class DatabaseManager extends DisposableObject {
   readonly onDidChangeCurrentDatabaseItem =
     this._onDidChangeCurrentDatabaseItem.event;
 
-  private readonly _databaseItems: DatabaseItem[] = [];
+  private readonly _databaseItems: DatabaseItemImpl[] = [];
   private _currentDatabaseItem: DatabaseItem | undefined = undefined;
 
   constructor(
@@ -127,8 +127,8 @@ export class DatabaseManager extends DisposableObject {
    *
    * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}.
    */
-  public async addExistingDatabaseItem(
-    databaseItem: DatabaseItem,
+  private async addExistingDatabaseItem(
+    databaseItem: DatabaseItemImpl,
     progress: ProgressCallback,
     makeSelected: boolean,
     token: vscode.CancellationToken,
@@ -162,7 +162,7 @@ export class DatabaseManager extends DisposableObject {
   private async createDatabaseItem(
     uri: vscode.Uri,
     displayName: string | undefined,
-  ): Promise<DatabaseItem> {
+  ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = extname(uri.fsPath) === ".testproj";
@@ -329,7 +329,7 @@ export class DatabaseManager extends DisposableObject {
     progress: ProgressCallback,
     token: vscode.CancellationToken,
     state: PersistedDatabaseItem,
-  ): Promise<DatabaseItem> {
+  ): Promise<DatabaseItemImpl> {
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
     let dateAdded = undefined;
@@ -499,7 +499,7 @@ export class DatabaseManager extends DisposableObject {
   private async addDatabaseItem(
     progress: ProgressCallback,
     token: vscode.CancellationToken,
-    item: DatabaseItem,
+    item: DatabaseItemImpl,
     updatePersistedState = true,
   ) {
     this._databaseItems.push(item);

From 09fc0f3040e08bb0b66afc2c5e6b958e568f403a Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Thu, 25 May 2023 15:03:25 +0200
Subject: [PATCH 071/119] Extract `DatabaseItem.refresh` to `DatabaseManager`

This moves the `refresh` method from `DatabaseItem` to `DatabaseManager`
and makes it private. This makes the `DatabaseItem` interface smaller
and more focused and ensures that `refresh` cannot be called from
outside of the `DatabaseManager`.
---
 .../local-databases/database-item-impl.ts     | 47 +++-------------
 .../local-databases/database-item.ts          | 10 +---
 .../local-databases/database-manager.ts       | 56 ++++++++++++-------
 .../test/factories/databases/databases.ts     |  1 -
 .../skeleton-query-wizard.test.ts             |  4 +-
 .../minimal-workspace/local-databases.test.ts |  8 +--
 .../query-testing/test-runner.test.ts         |  6 --
 7 files changed, 51 insertions(+), 81 deletions(-)

diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
index bd2c76786..16f184324 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -3,7 +3,6 @@ import * as cli from "../../codeql-cli/cli";
 import vscode from "vscode";
 import { FullDatabaseOptions } from "./database-options";
 import { basename, dirname, join, relative } from "path";
-import { asError } from "../../pure/helpers-pure";
 import {
   decodeSourceArchiveUri,
   encodeArchiveBasePath,
@@ -15,12 +14,11 @@ import { isLikelyDatabaseRoot } from "../../helpers";
 import { stat } from "fs-extra";
 import { pathsEqual } from "../../pure/files";
 import { DatabaseContents } from "./database-contents";
-import { DatabaseResolver } from "./database-resolver";
-import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 
 export class DatabaseItemImpl implements DatabaseItem {
-  private _error: Error | undefined = undefined;
-  private _contents: DatabaseContents | undefined;
+  // These are only public in the implementation, they are readonly in the interface
+  public error: Error | undefined = undefined;
+  public contents: DatabaseContents | undefined;
   /** A cache of database info */
   private _dbinfo: cli.DbInfo | undefined;
 
@@ -28,16 +26,15 @@ export class DatabaseItemImpl implements DatabaseItem {
     public readonly databaseUri: vscode.Uri,
     contents: DatabaseContents | undefined,
     private options: FullDatabaseOptions,
-    private readonly onChanged: (event: DatabaseChangedEvent) => void,
   ) {
-    this._contents = contents;
+    this.contents = contents;
   }
 
   public get name(): string {
     if (this.options.displayName) {
       return this.options.displayName;
-    } else if (this._contents) {
-      return this._contents.name;
+    } else if (this.contents) {
+      return this.contents.name;
     } else {
       return basename(this.databaseUri.fsPath);
     }
@@ -48,45 +45,17 @@ export class DatabaseItemImpl implements DatabaseItem {
   }
 
   public get sourceArchive(): vscode.Uri | undefined {
-    if (this.options.ignoreSourceArchive || this._contents === undefined) {
+    if (this.options.ignoreSourceArchive || this.contents === undefined) {
       return undefined;
     } else {
-      return this._contents.sourceArchiveUri;
+      return this.contents.sourceArchiveUri;
     }
   }
 
-  public get contents(): DatabaseContents | undefined {
-    return this._contents;
-  }
-
   public get dateAdded(): number | undefined {
     return this.options.dateAdded;
   }
 
-  public get error(): Error | undefined {
-    return this._error;
-  }
-
-  public async refresh(): Promise<void> {
-    try {
-      try {
-        this._contents = await DatabaseResolver.resolveDatabaseContents(
-          this.databaseUri,
-        );
-        this._error = undefined;
-      } catch (e) {
-        this._contents = undefined;
-        this._error = asError(e);
-        throw e;
-      }
-    } finally {
-      this.onChanged({
-        kind: DatabaseEventKind.Refresh,
-        item: this,
-      });
-    }
-  }
-
   public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
index 0da295afb..1794d8e75 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
@@ -27,15 +27,7 @@ export interface DatabaseItem {
 
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
-  /**
-   * Resolves the contents of the database.
-   *
-   * @remarks
-   * The contents include the database directory, source archive, and metadata about the database.
-   * If the database is invalid, `this.error` is updated with the error object that describes why
-   * the database is invalid. This error is also thrown.
-   */
-  refresh(): Promise<void>;
+
   /**
    * Resolves a filename to its URI in the source archive.
    *
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
index 29d910e1a..726ab554a 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-manager.ts
@@ -173,14 +173,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
     };
-    const databaseItem = new DatabaseItemImpl(
-      uri,
-      contents,
-      fullOptions,
-      (event) => {
-        this._onDidChangeDatabaseItem.fire(event);
-      },
-    );
+    const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
     return databaseItem;
   }
@@ -359,14 +352,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded,
       language,
     };
-    const item = new DatabaseItemImpl(
-      dbBaseUri,
-      undefined,
-      fullOptions,
-      (event) => {
-        this._onDidChangeDatabaseItem.fire(event);
-      },
-    );
+    const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 
     // Avoid persisting the database state after adding since that should happen only after
     // all databases have been added.
@@ -407,7 +393,7 @@ export class DatabaseManager extends DisposableObject {
             database,
           );
           try {
-            await databaseItem.refresh();
+            await this.refreshDatabase(databaseItem);
             await this.registerDatabase(progress, token, databaseItem);
             if (currentDatabaseUri === database.uri) {
               await this.setCurrentDatabaseItem(databaseItem, true);
@@ -449,8 +435,12 @@ export class DatabaseManager extends DisposableObject {
     item: DatabaseItem | undefined,
     skipRefresh = false,
   ): Promise<void> {
-    if (!skipRefresh && item !== undefined) {
-      await item.refresh(); // Will throw on invalid database.
+    if (
+      !skipRefresh &&
+      item !== undefined &&
+      item instanceof DatabaseItemImpl
+    ) {
+      await this.refreshDatabase(item); // Will throw on invalid database.
     }
     if (this._currentDatabaseItem !== item) {
       this._currentDatabaseItem = item;
@@ -616,6 +606,34 @@ export class DatabaseManager extends DisposableObject {
     await this.qs.registerDatabase(progress, token, dbItem);
   }
 
+  /**
+   * Resolves the contents of the database.
+   *
+   * @remarks
+   * The contents include the database directory, source archive, and metadata about the database.
+   * If the database is invalid, `databaseItem.error` is updated with the error object that describes why
+   * the database is invalid. This error is also thrown.
+   */
+  private async refreshDatabase(databaseItem: DatabaseItemImpl) {
+    try {
+      try {
+        databaseItem.contents = await DatabaseResolver.resolveDatabaseContents(
+          databaseItem.databaseUri,
+        );
+        databaseItem.error = undefined;
+      } catch (e) {
+        databaseItem.contents = undefined;
+        databaseItem.error = asError(e);
+        throw e;
+      }
+    } finally {
+      this._onDidChangeDatabaseItem.fire({
+        kind: DatabaseEventKind.Refresh,
+        item: databaseItem,
+      });
+    }
+  }
+
   private updatePersistedCurrentDatabaseItem(): void {
     void this.ctx.workspaceState.update(
       CURRENT_DB,
diff --git a/extensions/ql-vscode/test/factories/databases/databases.ts b/extensions/ql-vscode/test/factories/databases/databases.ts
index f6aa2aebf..46826917a 100644
--- a/extensions/ql-vscode/test/factories/databases/databases.ts
+++ b/extensions/ql-vscode/test/factories/databases/databases.ts
@@ -33,7 +33,6 @@ export function createMockDB(
       datasetUri: databaseUri,
     } as DatabaseContents,
     dbOptions,
-    () => void 0,
   );
 }
 
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts
index a50a08c15..949d79580 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/skeleton-query-wizard.test.ts
@@ -546,9 +546,7 @@ describe("SkeletonQueryWizard", () => {
           dateAdded: 123,
         } as FullDatabaseOptions);
 
-        jest
-          .spyOn(mockDbItem, "error", "get")
-          .mockReturnValue(asError("database go boom!"));
+        mockDbItem.error = asError("database go boom!");
 
         const sortedList =
           await SkeletonQueryWizard.sortDatabaseItemsByDateAdded([
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts
index 00bad35c2..a05418c39 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts
@@ -327,7 +327,7 @@ describe("local databases", () => {
         mockDbOptions(),
         Uri.parse("file:/sourceArchive-uri/"),
       );
-      (db as any)._contents.sourceArchiveUri = undefined;
+      (db as any).contents.sourceArchiveUri = undefined;
       expect(() => db.resolveSourceFile("abc")).toThrowError(
         "Scheme is missing",
       );
@@ -339,7 +339,7 @@ describe("local databases", () => {
         mockDbOptions(),
         Uri.parse("file:/sourceArchive-uri/"),
       );
-      (db as any)._contents.sourceArchiveUri = undefined;
+      (db as any).contents.sourceArchiveUri = undefined;
       expect(() => db.resolveSourceFile("http://abc")).toThrowError(
         "Invalid uri scheme",
       );
@@ -352,7 +352,7 @@ describe("local databases", () => {
           mockDbOptions(),
           Uri.parse("file:/sourceArchive-uri/"),
         );
-        (db as any)._contents.sourceArchiveUri = undefined;
+        (db as any).contents.sourceArchiveUri = undefined;
         const resolved = db.resolveSourceFile(undefined);
         expect(resolved.toString(true)).toBe(dbLocationUri(dir).toString(true));
       });
@@ -363,7 +363,7 @@ describe("local databases", () => {
           mockDbOptions(),
           Uri.parse("file:/sourceArchive-uri/"),
         );
-        (db as any)._contents.sourceArchiveUri = undefined;
+        (db as any).contents.sourceArchiveUri = undefined;
         const resolved = db.resolveSourceFile("file:");
         expect(resolved.toString()).toBe("file:///");
       });
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts
index 628355fb7..f4d9a3902 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-testing/test-runner.test.ts
@@ -40,17 +40,11 @@ describe("test-runner", () => {
     Uri.file("/path/to/test/dir/dir.testproj"),
     undefined,
     mockedObject<FullDatabaseOptions>({ displayName: "custom display name" }),
-    (_) => {
-      /* no change event listener */
-    },
   );
   const postTestDatabaseItem = new DatabaseItemImpl(
     Uri.file("/path/to/test/dir/dir.testproj"),
     undefined,
     mockedObject<FullDatabaseOptions>({ displayName: "default name" }),
-    (_) => {
-      /* no change event listener */
-    },
   );
 
   beforeEach(() => {

From 8be2dd805b179345730e5b361783fc2e4ee09a23 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 14:44:05 +0100
Subject: [PATCH 072/119] Use path.sep

---
 .../queries-panel/query-discovery.test.ts                 | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
index 0496f8712..de1c79690 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
@@ -12,7 +12,7 @@ import {
 } from "../../../../src/queries-panel/query-discovery";
 import { createMockApp } from "../../../__mocks__/appMock";
 import { mockedObject } from "../../utils/mocking.helpers";
-import { basename, join } from "path";
+import { basename, join, sep } from "path";
 import { sleep } from "../../../../src/pure/time";
 
 describe("QueryDiscovery", () => {
@@ -101,7 +101,11 @@ describe("QueryDiscovery", () => {
     });
 
     it("calls resolveQueries once for each workspace", async () => {
-      const workspaceRoots = ["/workspace1", "/workspace2", "/workspace3"];
+      const workspaceRoots = [
+        `${sep}workspace1`,
+        `${sep}workspace2`,
+        `${sep}workspace3`,
+      ];
       jest.spyOn(workspace, "workspaceFolders", "get").mockReturnValueOnce(
         workspaceRoots.map((root, index) => ({
           uri: Uri.file(root),

From adcc3d0b3950f5e2b92e8583a8e3a3fa2a68c2e4 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Thu, 25 May 2023 15:52:00 +0200
Subject: [PATCH 073/119] Remove unnecessary if-statements before slice

---
 .../src/data-extensions-editor/auto-model.ts     | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index 16d14ed75..116f3e24f 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -48,14 +48,12 @@ export function createAutoModelRequest(
           modeledMethod.type === "none"
             ? undefined
             : toMethodClassification(modeledMethod),
-        usages: externalApiUsage.usages.map((usage) => usage.label),
+        usages: externalApiUsage.usages
+          .slice(0, 10)
+          .map((usage) => usage.label),
         input: `Argument[${argumentIndex}]`,
       };
 
-      if (method.usages.length > 10) {
-        method.usages = method.usages.slice(0, 10);
-      }
-
       if (modeledMethod.type === "none") {
         request.candidates.push(method);
       } else {
@@ -64,12 +62,8 @@ export function createAutoModelRequest(
     }
   }
 
-  if (request.candidates.length > 100) {
-    request.candidates = request.candidates.slice(0, 100);
-  }
-  if (request.samples.length > 20) {
-    request.samples = request.samples.slice(0, 20);
-  }
+  request.candidates = request.candidates.slice(0, 100);
+  request.samples = request.samples.slice(0, 20);
 
   return request;
 }

From 472b1769c5491fd91fbe17080242b1da8bc5bd32 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 15:38:49 +0100
Subject: [PATCH 074/119] Update
 extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts

Co-authored-by: Shati Patel <42641846+shati-patel@users.noreply.github.com>
---
 .../minimal-workspace/queries-panel/query-discovery.test.ts     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
index de1c79690..76052b5bd 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
@@ -100,7 +100,7 @@ describe("QueryDiscovery", () => {
       );
     });
 
-    it("calls resolveQueries once for each workspace", async () => {
+    it("calls resolveQueries once for each workspace folder", async () => {
       const workspaceRoots = [
         `${sep}workspace1`,
         `${sep}workspace2`,

From efabcaefe3dae5ea7fb802933980ed7e133cf1f4 Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Thu, 25 May 2023 08:30:24 -0700
Subject: [PATCH 075/119] Update
 extensions/ql-vscode/src/variant-analysis/run-remote-query.ts

Co-authored-by: Koen Vlaswinkel <koesie10@users.noreply.github.com>
---
 extensions/ql-vscode/src/variant-analysis/run-remote-query.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
index c8d493bf0..1ccf46042 100644
--- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
+++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
@@ -379,8 +379,6 @@ async function fixPackFile(
   }
   const qlpack = load(await readFile(packPath, "utf8")) as QlPack;
 
-  // Use original name
-  // qlpack.name = QUERY_PACK_NAME;
   updateDefaultSuite(qlpack, packRelativePath);
   removeWorkspaceRefs(qlpack);
 

From 5a2cb8bc4194874f0ca595e3d3ec80404d7292ef Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 17:26:16 +0100
Subject: [PATCH 076/119] Convert launchDiscovery to use async

---
 extensions/ql-vscode/src/common/discovery.ts | 53 ++++++++++----------
 1 file changed, 26 insertions(+), 27 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index 8c9f970d5..8f5ef1406 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -42,7 +42,7 @@ export abstract class Discovery<T> extends DisposableObject {
     } else {
       // No discovery in progress, so start one now.
       this.discoveryInProgress = true;
-      this.launchDiscovery();
+      void this.launchDiscovery();
     }
   }
 
@@ -51,34 +51,33 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery operation completes, the `update` function will be invoked with the results of the
    * discovery.
    */
-  private launchDiscovery(): void {
-    const discoveryPromise = this.discover();
-    discoveryPromise
-      .then((results) => {
-        if (!this.retry) {
-          // Update any listeners with the results of the discovery.
-          this.discoveryInProgress = false;
-          this.update(results);
-        }
-      })
+  private async launchDiscovery(): Promise<void> {
+    let results: T | undefined;
+    try {
+      results = await this.discover();
+    } catch (err) {
+      void extLogger.log(
+        `${this.name} failed. Reason: ${getErrorMessage(err)}`,
+      );
+      results = undefined;
+    }
 
-      .catch((err: unknown) => {
-        void extLogger.log(
-          `${this.name} failed. Reason: ${getErrorMessage(err)}`,
-        );
-      })
+    if (this.retry) {
+      // Another refresh request came in while we were still running a previous discovery
+      // operation. Since the discovery results we just computed are now stale, we'll launch
+      // another discovery operation instead of updating.
+      // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+      // initial discovery operation failed.
+      this.retry = false;
+      await this.launchDiscovery();
+    } else {
+      this.discoveryInProgress = false;
 
-      .finally(() => {
-        if (this.retry) {
-          // Another refresh request came in while we were still running a previous discovery
-          // operation. Since the discovery results we just computed are now stale, we'll launch
-          // another discovery operation instead of updating.
-          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-          // initial discovery operation failed.
-          this.retry = false;
-          this.launchDiscovery();
-        }
-      });
+      // If the discovery was successful, then update any listeners with the results.
+      if (results !== undefined) {
+        this.update(results);
+      }
+    }
   }
 
   /**

From 5405b1bf2989d49656dbfd8e5247c1dbc7cfc51a Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 17:36:44 +0100
Subject: [PATCH 077/119] Convert refresh to return a promise

---
 extensions/ql-vscode/src/common/discovery.ts       | 14 ++++++++------
 .../ql-vscode/src/queries-panel/queries-module.ts  |  2 +-
 .../src/query-testing/qltest-discovery.ts          |  2 +-
 .../ql-vscode/src/query-testing/test-adapter.ts    |  2 +-
 .../ql-vscode/src/query-testing/test-manager.ts    |  2 +-
 5 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index 8f5ef1406..735177008 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -9,7 +9,7 @@ import { getErrorMessage } from "../pure/helpers-pure";
  */
 export abstract class Discovery<T> extends DisposableObject {
   private retry = false;
-  private discoveryInProgress = false;
+  private currentDiscoveryPromise: Promise<void> | undefined;
 
   constructor(private readonly name: string) {
     super();
@@ -18,8 +18,10 @@ export abstract class Discovery<T> extends DisposableObject {
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
+   *
+   * Returns a promise that resolves when the refresh is complete, including any retries.
    */
-  public refresh(): void {
+  public refresh(): Promise<void> {
     // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
     // got a storm of refresh requests due to, say, the copying or deletion of a large directory
     // tree, we could potentially spawn a separate simultaneous discovery operation for each
@@ -36,14 +38,14 @@ export abstract class Discovery<T> extends DisposableObject {
     // other change notifications that might be coming along. However, this would create more
     // latency in the common case, in order to save a bit of latency in the uncommon case.
 
-    if (this.discoveryInProgress) {
+    if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
       this.retry = true;
     } else {
       // No discovery in progress, so start one now.
-      this.discoveryInProgress = true;
-      void this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery();
     }
+    return this.currentDiscoveryPromise;
   }
 
   /**
@@ -71,7 +73,7 @@ export abstract class Discovery<T> extends DisposableObject {
       this.retry = false;
       await this.launchDiscovery();
     } else {
-      this.discoveryInProgress = false;
+      this.currentDiscoveryPromise = undefined;
 
       // If the discovery was successful, then update any listeners with the results.
       if (results !== undefined) {
diff --git a/extensions/ql-vscode/src/queries-panel/queries-module.ts b/extensions/ql-vscode/src/queries-panel/queries-module.ts
index d62812f46..aede56a8d 100644
--- a/extensions/ql-vscode/src/queries-panel/queries-module.ts
+++ b/extensions/ql-vscode/src/queries-panel/queries-module.ts
@@ -21,7 +21,7 @@ export class QueriesModule extends DisposableObject {
 
     const queryDiscovery = new QueryDiscovery(app, cliServer);
     this.push(queryDiscovery);
-    queryDiscovery.refresh();
+    void queryDiscovery.refresh();
 
     const queriesPanel = new QueriesPanel(queryDiscovery);
     this.push(queriesPanel);
diff --git a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
index b7d333ff6..6775b9611 100644
--- a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
+++ b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
@@ -64,7 +64,7 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
 
   private handleDidChange(uri: Uri): void {
     if (!QLTestDiscovery.ignoreTestPath(uri.fsPath)) {
-      this.refresh();
+      void this.refresh();
     }
   }
   protected async discover(): Promise<QLTestDiscoveryResults> {
diff --git a/extensions/ql-vscode/src/query-testing/test-adapter.ts b/extensions/ql-vscode/src/query-testing/test-adapter.ts
index 86c37b845..85b30a47c 100644
--- a/extensions/ql-vscode/src/query-testing/test-adapter.ts
+++ b/extensions/ql-vscode/src/query-testing/test-adapter.ts
@@ -115,7 +115,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
     this.qlTestDiscovery = this.push(
       new QLTestDiscovery(workspaceFolder, cliServer),
     );
-    this.qlTestDiscovery.refresh();
+    void this.qlTestDiscovery.refresh();
 
     this.push(this.qlTestDiscovery.onDidChangeTests(this.discoverTests, this));
   }
diff --git a/extensions/ql-vscode/src/query-testing/test-manager.ts b/extensions/ql-vscode/src/query-testing/test-manager.ts
index 94b79432b..073cc7170 100644
--- a/extensions/ql-vscode/src/query-testing/test-manager.ts
+++ b/extensions/ql-vscode/src/query-testing/test-manager.ts
@@ -92,7 +92,7 @@ class WorkspaceFolderHandler extends DisposableObject {
     this.push(
       this.testDiscovery.onDidChangeTests(this.handleDidChangeTests, this),
     );
-    this.testDiscovery.refresh();
+    void this.testDiscovery.refresh();
   }
 
   private handleDidChangeTests(): void {

From 9dd7476c90f15162cebabbcb37081155c571e04b Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 17:39:22 +0100
Subject: [PATCH 078/119] Use refresh promise in tests

---
 .../queries-panel/query-discovery.test.ts     | 84 ++++++++-----------
 .../query-testing/qltest-discovery.test.ts    | 24 +++---
 2 files changed, 49 insertions(+), 59 deletions(-)

diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
index 76052b5bd..fe750c89d 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
@@ -6,10 +6,7 @@ import {
   workspace,
 } from "vscode";
 import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
-import {
-  QueryDiscovery,
-  QueryDiscoveryResults,
-} from "../../../../src/queries-panel/query-discovery";
+import { QueryDiscovery } from "../../../../src/queries-panel/query-discovery";
 import { createMockApp } from "../../../__mocks__/appMock";
 import { mockedObject } from "../../utils/mocking.helpers";
 import { basename, join, sep } from "path";
@@ -28,11 +25,10 @@ describe("QueryDiscovery", () => {
       });
 
       const discovery = new QueryDiscovery(createMockApp({}), cli);
-      const results: QueryDiscoveryResults = await (
-        discovery as any
-      ).discover();
+      await discovery.refresh();
+      const queries = discovery.queries;
 
-      expect(results.queries).toEqual([]);
+      expect(queries).toEqual([]);
       expect(resolveQueries).toHaveBeenCalledTimes(1);
     });
 
@@ -49,22 +45,18 @@ describe("QueryDiscovery", () => {
       });
 
       const discovery = new QueryDiscovery(createMockApp({}), cli);
-      const results: QueryDiscoveryResults = await (
-        discovery as any
-      ).discover();
+      await discovery.refresh();
+      const queries = discovery.queries;
+      expect(queries).toBeDefined();
 
-      expect(results.queries[0].children.length).toEqual(3);
-      expect(results.queries[0].children[0].name).toEqual("dir1");
-      expect(results.queries[0].children[0].children.length).toEqual(1);
-      expect(results.queries[0].children[0].children[0].name).toEqual(
-        "query1.ql",
-      );
-      expect(results.queries[0].children[1].name).toEqual("dir2");
-      expect(results.queries[0].children[1].children.length).toEqual(1);
-      expect(results.queries[0].children[1].children[0].name).toEqual(
-        "query2.ql",
-      );
-      expect(results.queries[0].children[2].name).toEqual("query3.ql");
+      expect(queries![0].children.length).toEqual(3);
+      expect(queries![0].children[0].name).toEqual("dir1");
+      expect(queries![0].children[0].children.length).toEqual(1);
+      expect(queries![0].children[0].children[0].name).toEqual("query1.ql");
+      expect(queries![0].children[1].name).toEqual("dir2");
+      expect(queries![0].children[1].children.length).toEqual(1);
+      expect(queries![0].children[1].children[0].name).toEqual("query2.ql");
+      expect(queries![0].children[2].name).toEqual("query3.ql");
     });
 
     it("should collapse directories containing only a single element", async () => {
@@ -79,25 +71,21 @@ describe("QueryDiscovery", () => {
       });
 
       const discovery = new QueryDiscovery(createMockApp({}), cli);
-      const results: QueryDiscoveryResults = await (
-        discovery as any
-      ).discover();
+      await discovery.refresh();
+      const queries = discovery.queries;
+      expect(queries).toBeDefined();
 
-      expect(results.queries[0].children.length).toEqual(1);
-      expect(results.queries[0].children[0].name).toEqual("dir1");
-      expect(results.queries[0].children[0].children.length).toEqual(2);
-      expect(results.queries[0].children[0].children[0].name).toEqual(
+      expect(queries![0].children.length).toEqual(1);
+      expect(queries![0].children[0].name).toEqual("dir1");
+      expect(queries![0].children[0].children.length).toEqual(2);
+      expect(queries![0].children[0].children[0].name).toEqual(
         "dir2 / dir3 / dir3",
       );
-      expect(
-        results.queries[0].children[0].children[0].children.length,
-      ).toEqual(1);
-      expect(
-        results.queries[0].children[0].children[0].children[0].name,
-      ).toEqual("query2.ql");
-      expect(results.queries[0].children[0].children[1].name).toEqual(
-        "query1.ql",
+      expect(queries![0].children[0].children[0].children.length).toEqual(1);
+      expect(queries![0].children[0].children[0].children[0].name).toEqual(
+        "query2.ql",
       );
+      expect(queries![0].children[0].children[1].name).toEqual("query1.ql");
     });
 
     it("calls resolveQueries once for each workspace folder", async () => {
@@ -128,14 +116,14 @@ describe("QueryDiscovery", () => {
       });
 
       const discovery = new QueryDiscovery(createMockApp({}), cli);
-      const results: QueryDiscoveryResults = await (
-        discovery as any
-      ).discover();
+      await discovery.refresh();
+      const queries = discovery.queries;
+      expect(queries).toBeDefined();
 
-      expect(results.queries.length).toEqual(3);
-      expect(results.queries[0].children[0].name).toEqual("query1.ql");
-      expect(results.queries[1].children[0].name).toEqual("query2.ql");
-      expect(results.queries[2].children[0].name).toEqual("query3.ql");
+      expect(queries!.length).toEqual(3);
+      expect(queries![0].children[0].name).toEqual("query1.ql");
+      expect(queries![1].children[0].name).toEqual("query2.ql");
+      expect(queries![2].children[0].name).toEqual("query3.ql");
 
       expect(resolveQueries).toHaveBeenCalledTimes(3);
     });
@@ -176,8 +164,7 @@ describe("QueryDiscovery", () => {
       const onDidChangeQueriesSpy = jest.fn();
       discovery.onDidChangeQueries(onDidChangeQueriesSpy);
 
-      const results = await (discovery as any).discover();
-      (discovery as any).update(results);
+      await discovery.refresh();
 
       expect(createFileSystemWatcherSpy).toHaveBeenCalledTimes(2);
       expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1);
@@ -209,8 +196,7 @@ describe("QueryDiscovery", () => {
       const onDidChangeQueriesSpy = jest.fn();
       discovery.onDidChangeQueries(onDidChangeQueriesSpy);
 
-      const results = await (discovery as any).discover();
-      (discovery as any).update(results);
+      await discovery.refresh();
 
       expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(1);
 
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts
index e85b670a2..49edbef9e 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/query-testing/qltest-discovery.test.ts
@@ -52,12 +52,14 @@ describe("qltest-discovery", () => {
     });
 
     it("should run discovery", async () => {
-      const result = await (qlTestDiscover as any).discover();
-      expect(result.watchPath).toEqualPath(baseDir);
-      expect(result.testDirectory.path).toEqualPath(baseDir);
-      expect(result.testDirectory.name).toBe("My tests");
+      await qlTestDiscover.refresh();
+      const testDirectory = qlTestDiscover.testDirectory;
+      expect(testDirectory).toBeDefined();
 
-      let children = result.testDirectory.children;
+      expect(testDirectory!.path).toEqualPath(baseDir);
+      expect(testDirectory!.name).toBe("My tests");
+
+      let children = testDirectory!.children;
       expect(children.length).toBe(1);
 
       expect(children[0].path).toEqualPath(cDir);
@@ -83,12 +85,14 @@ describe("qltest-discovery", () => {
     it("should avoid discovery if a folder does not exist", async () => {
       await fs.remove(baseDir);
 
-      const result = await (qlTestDiscover as any).discover();
-      expect(result.watchPath).toEqualPath(baseDir);
-      expect(result.testDirectory.path).toEqualPath(baseDir);
-      expect(result.testDirectory.name).toBe("My tests");
+      await qlTestDiscover.refresh();
+      const testDirectory = qlTestDiscover.testDirectory;
+      expect(testDirectory).toBeDefined();
 
-      expect(result.testDirectory.children.length).toBe(0);
+      expect(testDirectory!.path).toEqualPath(baseDir);
+      expect(testDirectory!.name).toBe("My tests");
+
+      expect(testDirectory!.children.length).toBe(0);
     });
   });
 });

From 1f4b19cd3753a518960946eaab21c2c10d4a244a Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Thu, 25 May 2023 17:40:32 +0100
Subject: [PATCH 079/119] Add getCurrentRefreshPromise

---
 extensions/ql-vscode/src/common/discovery.ts              | 8 ++++++++
 .../queries-panel/query-discovery.test.ts                 | 7 ++-----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index 735177008..dad1400a9 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -15,6 +15,14 @@ export abstract class Discovery<T> extends DisposableObject {
     super();
   }
 
+  /**
+   * Returns the promise of the currently running refresh operation, if one is in progress.
+   * Otherwise returns a promise that resolves immediately.
+   */
+  public getCurrentRefreshPromise(): Promise<void> {
+    return this.currentDiscoveryPromise ?? Promise.resolve();
+  }
+
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
index fe750c89d..55875603c 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
@@ -10,7 +10,6 @@ import { QueryDiscovery } from "../../../../src/queries-panel/query-discovery";
 import { createMockApp } from "../../../__mocks__/appMock";
 import { mockedObject } from "../../utils/mocking.helpers";
 import { basename, join, sep } from "path";
-import { sleep } from "../../../../src/pure/time";
 
 describe("QueryDiscovery", () => {
   beforeEach(() => {
@@ -171,8 +170,7 @@ describe("QueryDiscovery", () => {
 
       onWatcherDidChangeEvent.fire(workspace.workspaceFolders![0].uri);
 
-      // Wait for refresh to finish
-      await sleep(100);
+      await discovery.getCurrentRefreshPromise();
 
       expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
     });
@@ -202,8 +200,7 @@ describe("QueryDiscovery", () => {
 
       onDidChangeWorkspaceFoldersEvent.fire({ added: [], removed: [] });
 
-      // Wait for refresh to finish
-      await sleep(100);
+      await discovery.getCurrentRefreshPromise();
 
       expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
     });

From 4b54e4f31f85ff0bcee5ced6d1570102938d6ab9 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Fri, 26 May 2023 13:40:12 +0200
Subject: [PATCH 080/119] Use 20 candidates and 100 samples

---
 extensions/ql-vscode/src/data-extensions-editor/auto-model.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index 116f3e24f..b4e6f4779 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -62,8 +62,8 @@ export function createAutoModelRequest(
     }
   }
 
-  request.candidates = request.candidates.slice(0, 100);
-  request.samples = request.samples.slice(0, 20);
+  request.candidates = request.candidates.slice(0, 20);
+  request.samples = request.samples.slice(0, 100);
 
   return request;
 }

From f52ad04afeed8014739dac335eb7005d8bdf5031 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Fri, 26 May 2023 16:18:59 +0200
Subject: [PATCH 081/119] Improve parsing of predicted classifications

---
 .../src/data-extensions-editor/auto-model.ts  |  72 ++++++++++---
 .../data-extensions-editor-view.ts            |  21 +---
 .../data-extensions-editor/auto-model.test.ts | 102 +++++++++++++++++-
 3 files changed, 163 insertions(+), 32 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index b4e6f4779..ec5995554 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -68,6 +68,61 @@ export function createAutoModelRequest(
   return request;
 }
 
+export function parsePredictedClassifications(
+  predicted: Method[],
+): Record<string, ModeledMethod> {
+  const predictedBySignature: Record<string, Method[]> = {};
+  for (const method of predicted) {
+    if (!method.classification) {
+      continue;
+    }
+
+    const signature = toFullMethodSignature(method);
+
+    if (!(signature in predictedBySignature)) {
+      predictedBySignature[signature] = [];
+    }
+
+    predictedBySignature[signature].push(method);
+  }
+
+  const modeledMethods: Record<string, ModeledMethod> = {};
+
+  for (const signature in predictedBySignature) {
+    const predictedMethods = predictedBySignature[signature];
+
+    const sinks = predictedMethods.filter(
+      (method) => method.classification?.type === ClassificationType.Sink,
+    );
+    if (sinks.length === 0) {
+      // For now, model any method for which none of its arguments are modeled as sinks as neutral
+      modeledMethods[signature] = {
+        type: "neutral",
+        kind: "",
+        input: "",
+        output: "",
+      };
+      continue;
+    }
+
+    // Order the sinks by the input alphabetically. This will ensure that the first argument is always
+    // first in the list of sinks, the second argument is always second, etc.
+    // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first
+    sinks.sort((a, b) => (a.input ?? "").localeCompare(b.input ?? ""));
+
+    const sink = sinks[0];
+
+    modeledMethods[signature] = {
+      type: "sink",
+      kind: sink.classification?.kind ?? "",
+      input: sink.input ?? "",
+      output: sink.output ?? "",
+    };
+  }
+
+  return modeledMethods;
+}
+
 function toMethodClassificationType(
   type: ModeledMethodType,
 ): ClassificationType {
@@ -93,19 +148,6 @@ function toMethodClassification(modeledMethod: ModeledMethod): Classification {
   };
 }
 
-export function classificationTypeToModeledMethodType(
-  type: ClassificationType,
-): ModeledMethodType {
-  switch (type) {
-    case ClassificationType.Source:
-      return "source";
-    case ClassificationType.Sink:
-      return "sink";
-    case ClassificationType.Summary:
-      return "summary";
-    case ClassificationType.Neutral:
-      return "neutral";
-    default:
-      return "none";
-  }
+function toFullMethodSignature(method: Method): string {
+  return `${method.package}.${method.type}.${method.name}${method.signature}`;
 }
diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
index 4adb5b392..e3f2c7d4c 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
@@ -41,8 +41,8 @@ import { ModeledMethod } from "./modeled-method";
 import { ExtensionPackModelFile } from "./shared/extension-pack";
 import { autoModel } from "./auto-model-api";
 import {
-  classificationTypeToModeledMethodType,
   createAutoModelRequest,
+  parsePredictedClassifications,
 } from "./auto-model";
 import { showLlmGeneration } from "../config";
 
@@ -393,24 +393,13 @@ export class DataExtensionsEditorView extends AbstractWebview<
 
     const response = await autoModel(this.app.credentials, request);
 
-    const modeledMethodsByName: Record<string, ModeledMethod> = {};
-
-    for (const method of response.predicted) {
-      if (method.classification === undefined) {
-        continue;
-      }
-
-      modeledMethodsByName[method.signature] = {
-        type: classificationTypeToModeledMethodType(method.classification.type),
-        kind: method.classification.kind,
-        input: method.input ?? "",
-        output: method.output ?? "",
-      };
-    }
+    const predictedModeledMethods = parsePredictedClassifications(
+      response.predicted,
+    );
 
     await this.postMessage({
       t: "addModeledMethods",
-      modeledMethods: modeledMethodsByName,
+      modeledMethods: predictedModeledMethods,
       overrideNone: true,
     });
   }
diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
index d56fcda07..1eccb6165 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
@@ -1,6 +1,13 @@
-import { createAutoModelRequest } from "../../../src/data-extensions-editor/auto-model";
+import {
+  createAutoModelRequest,
+  parsePredictedClassifications,
+} from "../../../src/data-extensions-editor/auto-model";
 import { ExternalApiUsage } from "../../../src/data-extensions-editor/external-api-usage";
 import { ModeledMethod } from "../../../src/data-extensions-editor/modeled-method";
+import {
+  ClassificationType,
+  Method,
+} from "../../../src/data-extensions-editor/auto-model-api";
 
 describe("createAutoModelRequest", () => {
   const externalApiUsages: ExternalApiUsage[] = [
@@ -281,3 +288,96 @@ describe("createAutoModelRequest", () => {
     });
   });
 });
+
+describe("parsePredictedClassifications", () => {
+  const predictions: Method[] = [
+    {
+      package: "org.sql2o",
+      type: "Sql2o",
+      name: "createQuery",
+      signature: "(String)",
+      usages: ["createQuery(...)", "createQuery(...)"],
+      input: "Argument[0]",
+      classification: {
+        type: ClassificationType.Sink,
+        kind: "sql injection sink",
+        explanation: "",
+      },
+    },
+    {
+      package: "org.sql2o",
+      type: "Sql2o",
+      name: "executeScalar",
+      signature: "(Class)",
+      usages: ["executeScalar(...)", "executeScalar(...)"],
+      input: "Argument[0]",
+      classification: {
+        type: ClassificationType.Neutral,
+        kind: "",
+        explanation: "not a sink",
+      },
+    },
+    {
+      package: "org.sql2o",
+      type: "Sql2o",
+      name: "Sql2o",
+      signature: "(String,String,String)",
+      usages: ["new Sql2o(...)"],
+      input: "Argument[0]",
+      classification: {
+        type: ClassificationType.Neutral,
+        kind: "",
+        explanation: "not a sink",
+      },
+    },
+    {
+      package: "org.sql2o",
+      type: "Sql2o",
+      name: "Sql2o",
+      signature: "(String,String,String)",
+      usages: ["new Sql2o(...)"],
+      input: "Argument[1]",
+      classification: {
+        type: ClassificationType.Sink,
+        kind: "sql injection sink",
+        explanation: "not a sink",
+      },
+    },
+    {
+      package: "org.sql2o",
+      type: "Sql2o",
+      name: "Sql2o",
+      signature: "(String,String,String)",
+      usages: ["new Sql2o(...)"],
+      input: "Argument[2]",
+      classification: {
+        type: ClassificationType.Sink,
+        kind: "sql injection sink",
+        explanation: "not a sink",
+      },
+    },
+  ];
+
+  it("correctly parses the output", () => {
+    expect(parsePredictedClassifications(predictions)).toEqual({
+      "org.sql2o.Sql2o.createQuery(String)": {
+        type: "sink",
+        kind: "sql injection sink",
+        input: "Argument[0]",
+        output: "",
+      },
+      "org.sql2o.Sql2o.executeScalar(Class)": {
+        type: "neutral",
+        kind: "",
+        input: "",
+        output: "",
+      },
+      "org.sql2o.Sql2o.Sql2o(String,String,String)": {
+        type: "sink",
+        kind: "sql injection sink",
+        input: "Argument[1]",
+        output: "",
+      },
+    });
+  });
+});

From deb268465f81b6efa09b4eba2c2549e7fb2174b6 Mon Sep 17 00:00:00 2001
From: Andrew Eisenberg <aeisenberg@github.com>
Date: Thu, 25 May 2023 21:14:05 -0700
Subject: [PATCH 082/119] Fix failing test

---
 .../variant-analysis/variant-analysis-manager.test.ts         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
index 733553e5f..993d0bb1d 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts
@@ -222,7 +222,7 @@ describe("Variant Analysis Manager", () => {
       it("should run a remote query that is part of a qlpack", async () => {
         await doVariantAnalysisTest({
           queryPath: "data-remote-qlpack/in-pack.ql",
-          expectedPackName: "codeql-remote/query",
+          expectedPackName: "github/remote-query-pack",
           filesThatExist: ["in-pack.ql", "lib.qll"],
           filesThatDoNotExist: [],
           qlxFilesThatExist: ["in-pack.qlx"],
@@ -232,7 +232,7 @@ describe("Variant Analysis Manager", () => {
       it("should run a remote query that is not part of a qlpack", async () => {
         await doVariantAnalysisTest({
           queryPath: "data-remote-no-qlpack/in-pack.ql",
-          expectedPackName: "",
+          expectedPackName: "codeql-remote/query",
           filesThatExist: ["in-pack.ql"],
           filesThatDoNotExist: ["lib.qll", "not-in-pack.ql"],
           qlxFilesThatExist: ["in-pack.qlx"],

From b8770a289636325227b303063ce0fde067999f00 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Fri, 26 May 2023 16:06:25 +0100
Subject: [PATCH 083/119] Convert assert-pure.ql to be a path-problem query

---
 .github/codeql/queries/assert-pure.ql | 37 ++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/.github/codeql/queries/assert-pure.ql b/.github/codeql/queries/assert-pure.ql
index cd840d502..7efe31751 100644
--- a/.github/codeql/queries/assert-pure.ql
+++ b/.github/codeql/queries/assert-pure.ql
@@ -1,21 +1,44 @@
 /**
  * @name Unwanted dependency on vscode API
- * @kind problem
+ * @kind path-problem
  * @problem.severity error
  * @id vscode-codeql/assert-pure
  * @description The modules stored under `pure` and tested in the `pure-tests`
  * are intended to be "pure".
  */
+
 import javascript
 
-class VSCodeImport extends ASTNode {
-  VSCodeImport() {
-    this.(Import).getImportedPath().getValue() = "vscode"
+class VSCodeImport extends AstNode {
+  VSCodeImport() { this.(Import).getImportedPath().getValue() = "vscode" }
+}
+
+class PureFile extends File {
+  PureFile() {
+    (
+      this.getRelativePath().regexpMatch(".*/src/pure/.*") or
+      this.getRelativePath().regexpMatch(".*/src/common/.*")
+    ) and
+    not this.getRelativePath().regexpMatch(".*/src/common/vscode/.*")
   }
 }
 
+Import getANonTypeOnlyImport(Module m) {
+  result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly()
+}
+
+Module getANonTypeOnlyImportedModule(Module m) {
+  result = getANonTypeOnlyImport(m).getImportedModule()
+}
+
+query predicate edges(AstNode a, AstNode b) {
+  getANonTypeOnlyImport(a) = b or
+  a.(Import).getImportedModule() = b
+}
+
 from Module m, VSCodeImport v
 where
-  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
-  m.getAnImportedModule*().getAnImport() = v
-select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"
+  m.getFile() instanceof PureFile and
+  getANonTypeOnlyImport(getANonTypeOnlyImportedModule*(m)) = v
+select m, m, v,
+  "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

From 2e29d0cda47f35e9a2998fcdd18d3779ce91ac3e Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Tue, 30 May 2023 10:04:33 +0200
Subject: [PATCH 084/119] Fix incorrect signature reconstruction

---
 .../ql-vscode/src/data-extensions-editor/auto-model.ts      | 2 +-
 .../unit-tests/data-extensions-editor/auto-model.test.ts    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index ec5995554..bb3f94468 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -149,5 +149,5 @@ function toMethodClassification(modeledMethod: ModeledMethod): Classification {
 }
 
 function toFullMethodSignature(method: Method): string {
-  return `${method.package}.${method.type}.${method.name}${method.signature}`;
+  return `${method.package}.${method.type}#${method.name}${method.signature}`;
 }
diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
index 1eccb6165..d93cd4523 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
@@ -360,19 +360,19 @@ describe("parsePredictedClassifications", () => {
 
   it("correctly parses the output", () => {
     expect(parsePredictedClassifications(predictions)).toEqual({
-      "org.sql2o.Sql2o.createQuery(String)": {
+      "org.sql2o.Sql2o#createQuery(String)": {
         type: "sink",
         kind: "sql injection sink",
         input: "Argument[0]",
         output: "",
       },
-      "org.sql2o.Sql2o.executeScalar(Class)": {
+      "org.sql2o.Sql2o#executeScalar(Class)": {
         type: "neutral",
         kind: "",
         input: "",
         output: "",
       },
-      "org.sql2o.Sql2o.Sql2o(String,String,String)": {
+      "org.sql2o.Sql2o#Sql2o(String,String,String)": {
         type: "sink",
         kind: "sql injection sink",
         input: "Argument[1]",

From 558f51b96226e3ec39bf15915bd22e8228b30648 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Tue, 30 May 2023 11:04:13 +0200
Subject: [PATCH 085/119] Add comment to `parsePredictedClassifications` method

---
 .../ql-vscode/src/data-extensions-editor/auto-model.ts    | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index bb3f94468..f1783ab76 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -68,6 +68,14 @@ export function createAutoModelRequest(
   return request;
 }
 
+/**
+ * For now, we have a simplified model that only models methods as sinks. It does not model methods as neutral,
+ * so we aren't actually able to correctly determine that a method is neutral; it could still be a source or summary.
+ * However, to keep this method simple and give output to the user, we will model any method for which none of its
+ * arguments are modeled as sinks as neutral.
+ *
+ * If there are multiple arguments which are modeled as sinks, we will only model the first one.
+ */
 export function parsePredictedClassifications(
   predicted: Method[],
 ): Record<string, ModeledMethod> {

From 695dc3f883dfe395cad069c9b944d84f3aaff650 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Tue, 30 May 2023 11:17:52 +0200
Subject: [PATCH 086/119] Improve sorting of input/output arguments

---
 .../src/data-extensions-editor/auto-model.ts  | 59 ++++++++++++++++++-
 .../data-extensions-editor/auto-model.test.ts | 47 +++++++++++++++
 2 files changed, 105 insertions(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index f1783ab76..c8084a5c9 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -116,7 +116,7 @@ export function parsePredictedClassifications(
     // Order the sinks by the input alphabetically. This will ensure that the first argument is always
     // first in the list of sinks, the second argument is always second, etc.
     // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first
-    sinks.sort((a, b) => (a.input ?? "").localeCompare(b.input ?? ""));
+    sinks.sort((a, b) => compareInputOutput(a.input ?? "", b.input ?? ""));
 
     const sink = sinks[0];
 
@@ -159,3 +159,60 @@ function toMethodClassification(modeledMethod: ModeledMethod): Classification {
 function toFullMethodSignature(method: Method): string {
   return `${method.package}.${method.type}#${method.name}${method.signature}`;
 }
+
+const argumentRegex = /^Argument\[(\d+)]$/;
+
+// Argument[this] is before ReturnValue
+const nonNumericArgumentOrder = ["Argument[this]", "ReturnValue"];
+
+/**
+ * Compare two inputs or outputs matching `Argument[<number>]`, `Argument[this]`, or `ReturnValue`.
+ * If they are the same, return 0. If a is less than b, returns a negative number.
+ * If a is greater than b, returns a positive number.
+ */
+export function compareInputOutput(a: string, b: string): number {
+  if (a === b) {
+    return 0;
+  }
+
+  const aMatch = a.match(argumentRegex);
+  const bMatch = b.match(argumentRegex);
+
+  // Numeric arguments are always first
+  if (aMatch && !bMatch) {
+    return -1;
+  }
+  if (!aMatch && bMatch) {
+    return 1;
+  }
+
+  // Neither is an argument
+  if (!aMatch && !bMatch) {
+    const aIndex = nonNumericArgumentOrder.indexOf(a);
+    const bIndex = nonNumericArgumentOrder.indexOf(b);
+
+    // If either one is unknown, it is sorted last
+    if (aIndex === -1 && bIndex === -1) {
+      return a.localeCompare(b);
+    }
+    if (aIndex === -1) {
+      return 1;
+    }
+    if (bIndex === -1) {
+      return -1;
+    }
+
+    return aIndex - bIndex;
+  }
+
+  // This case shouldn't happen, but makes TypeScript happy
+  if (!aMatch || !bMatch) {
+    return 0;
+  }
+
+  // Both are arguments
+  const aIndex = parseInt(aMatch[1]);
+  const bIndex = parseInt(bMatch[1]);
+
+  return aIndex - bIndex;
+}
diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
index d93cd4523..17a16eb42 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
@@ -1,4 +1,5 @@
 import {
+  compareInputOutput,
   createAutoModelRequest,
   parsePredictedClassifications,
 } from "../../../src/data-extensions-editor/auto-model";
@@ -381,3 +382,49 @@ describe("parsePredictedClassifications", () => {
     });
   });
 });
+
+describe("compareInputOutput", () => {
+  it("with two small numeric arguments", () => {
+    expect(
+      compareInputOutput("Argument[0]", "Argument[1]"),
+    ).toBeLessThanOrEqual(-1);
+  });
+
+  it("with one larger non-alphabetic argument", () => {
+    expect(
+      compareInputOutput("Argument[10]", "Argument[2]"),
+    ).toBeGreaterThanOrEqual(1);
+  });
+
+  it("with one non-numeric arguments", () => {
+    expect(
+      compareInputOutput("Argument[5]", "Argument[this]"),
+    ).toBeLessThanOrEqual(-1);
+  });
+
+  it("with two non-numeric arguments", () => {
+    expect(
+      compareInputOutput("ReturnValue", "Argument[this]"),
+    ).toBeGreaterThanOrEqual(1);
+  });
+
+  it("with one unknown argument in the a position", () => {
+    expect(
+      compareInputOutput("FooBar", "Argument[this]"),
+    ).toBeGreaterThanOrEqual(1);
+  });
+
+  it("with one unknown argument in the b position", () => {
+    expect(compareInputOutput("Argument[this]", "FooBar")).toBeLessThanOrEqual(
+      -1,
+    );
+  });
+
+  it("with one empty string arguments", () => {
+    expect(compareInputOutput("Argument[5]", "")).toBeLessThanOrEqual(-1);
+  });
+
+  it("with two unknown arguments", () => {
+    expect(compareInputOutput("FooBar", "BarFoo")).toBeGreaterThanOrEqual(1);
+  });
+});

From d27f3d2699fedb21cc19aff1026704686ebc2818 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Tue, 30 May 2023 11:21:01 +0200
Subject: [PATCH 087/119] Remove incorrect explanations

---
 .../test/unit-tests/data-extensions-editor/auto-model.test.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
index 17a16eb42..bb7bc07ac 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
@@ -341,7 +341,7 @@ describe("parsePredictedClassifications", () => {
       classification: {
         type: ClassificationType.Sink,
         kind: "sql injection sink",
-        explanation: "not a sink",
+        explanation: "",
       },
     },
     {
@@ -354,7 +354,7 @@ describe("parsePredictedClassifications", () => {
       classification: {
         type: ClassificationType.Sink,
         kind: "sql injection sink",
-        explanation: "not a sink",
+        explanation: "",
       },
     },
   ];

From e1894afb16daf84d8a746e3e9373b50731c35144 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 30 May 2023 11:39:47 +0100
Subject: [PATCH 088/119] Extend ImportDeclaration instead of AstNode

---
 .github/codeql/queries/assert-pure.ql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/codeql/queries/assert-pure.ql b/.github/codeql/queries/assert-pure.ql
index 7efe31751..49535d5d6 100644
--- a/.github/codeql/queries/assert-pure.ql
+++ b/.github/codeql/queries/assert-pure.ql
@@ -9,8 +9,8 @@
 
 import javascript
 
-class VSCodeImport extends AstNode {
-  VSCodeImport() { this.(Import).getImportedPath().getValue() = "vscode" }
+class VSCodeImport extends ImportDeclaration {
+  VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
 }
 
 class PureFile extends File {

From c462bc024352feede7f7fe4f8cd943cec931f01a Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 30 May 2023 11:41:59 +0100
Subject: [PATCH 089/119] Use edges from select clause

---
 .github/codeql/queries/assert-pure.ql | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/codeql/queries/assert-pure.ql b/.github/codeql/queries/assert-pure.ql
index 49535d5d6..63d9c8774 100644
--- a/.github/codeql/queries/assert-pure.ql
+++ b/.github/codeql/queries/assert-pure.ql
@@ -27,10 +27,6 @@ Import getANonTypeOnlyImport(Module m) {
   result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly()
 }
 
-Module getANonTypeOnlyImportedModule(Module m) {
-  result = getANonTypeOnlyImport(m).getImportedModule()
-}
-
 query predicate edges(AstNode a, AstNode b) {
   getANonTypeOnlyImport(a) = b or
   a.(Import).getImportedModule() = b
@@ -39,6 +35,6 @@ query predicate edges(AstNode a, AstNode b) {
 from Module m, VSCodeImport v
 where
   m.getFile() instanceof PureFile and
-  getANonTypeOnlyImport(getANonTypeOnlyImportedModule*(m)) = v
+  edges+(m, v)
 select m, m, v,
   "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

From 5c81671e67a9ab23ed9ffb5c9ac59f7050ece14c Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Tue, 30 May 2023 12:28:44 +0200
Subject: [PATCH 090/119] Retrieve external API usage snippets using SARIF

---
 extensions/ql-vscode/src/codeql-cli/cli.ts    |   2 +
 .../auto-model-usages-query.ts                | 129 ++++++++++++++++++
 .../src/data-extensions-editor/auto-model.ts  |   9 +-
 .../data-extensions-editor-view.ts            |  46 ++++++-
 .../external-api-usage-query.ts               |  22 +--
 .../data-extensions-editor/queries/csharp.ts  |  71 ++++++----
 .../data-extensions-editor/queries/java.ts    |  22 ++-
 .../data-extensions-editor/queries/query.ts   |   1 +
 .../local-databases/database-item-impl.ts     |  10 +-
 .../local-databases/database-item.ts          |   8 +-
 extensions/ql-vscode/src/helpers.ts           |   2 +-
 extensions/ql-vscode/src/query-results.ts     |   4 +-
 .../data-extensions-editor/auto-model.test.ts |  61 +++++++--
 .../external-api-usage-query.test.ts          |   2 +-
 .../no-workspace/query-results.test.ts        |   2 +
 15 files changed, 333 insertions(+), 58 deletions(-)
 create mode 100644 extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts
index c055b4b2b..9dc5e304f 100644
--- a/extensions/ql-vscode/src/codeql-cli/cli.ts
+++ b/extensions/ql-vscode/src/codeql-cli/cli.ts
@@ -1050,6 +1050,7 @@ export class CodeQLCliServer implements Disposable {
     resultsPath: string,
     interpretedResultsPath: string,
     sourceInfo?: SourceInfo,
+    args?: string[],
   ): Promise<sarif.Log> {
     const additionalArgs = [
       // TODO: This flag means that we don't group interpreted results
@@ -1057,6 +1058,7 @@ export class CodeQLCliServer implements Disposable {
       // interpretation with and without this flag, or do some
       // grouping client-side.
       "--no-group-results",
+      ...(args ?? []),
     ];
 
     await this.runInterpretCommand(
diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
new file mode 100644
index 000000000..4f538427a
--- /dev/null
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -0,0 +1,129 @@
+import { CancellationTokenSource } from "vscode";
+import { join } from "path";
+import { runQuery } from "./external-api-usage-query";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { interpretResultsSarif } from "../query-results";
+import { ProgressCallback } from "../common/vscode/progress";
+
+type Options = {
+  cliServer: Pick<
+    CodeQLCliServer,
+    "resolveDatabase" | "resolveQlpacks" | "interpretBqrsSarif"
+  >;
+  queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
+  databaseItem: Pick<
+    DatabaseItem,
+    | "contents"
+    | "databaseUri"
+    | "language"
+    | "sourceArchive"
+    | "getSourceLocationPrefix"
+  >;
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+};
+
+export async function getAutoModelUsages({
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+}: Options): Promise<Record<string, string[]>> {
+  const maxStep = 1500;
+
+  const cancellationTokenSource = new CancellationTokenSource();
+
+  const queryResult = await runQuery("usagesQuery", {
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    progress: (update) =>
+      progress({
+        maxStep,
+        step: update.step,
+        message: update.message,
+      }),
+    token: cancellationTokenSource.token,
+  });
+  if (!queryResult) {
+    throw new Error("Query failed");
+  }
+
+  progress({
+    maxStep,
+    step: 1100,
+    message: "Retrieving source locatin prefix",
+  });
+
+  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
+    cliServer,
+  );
+  const sourceArchiveUri = databaseItem.sourceArchive;
+  const sourceInfo =
+    sourceArchiveUri === undefined
+      ? undefined
+      : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix,
+        };
+
+  progress({
+    maxStep,
+    step: 1200,
+    message: "Interpreting results",
+  });
+
+  const sarif = await interpretResultsSarif(
+    cliServer,
+    {
+      kind: "problem",
+      id: "usage",
+    },
+    {
+      resultsPath: queryResult.outputDir.bqrsPath,
+      interpretedResultsPath: join(
+        queryStorageDir,
+        "interpreted-results.sarif",
+      ),
+    },
+    sourceInfo,
+    ["--sarif-add-snippets"],
+  );
+
+  progress({
+    maxStep,
+    step: 1400,
+    message: "Parsing results",
+  });
+
+  const snippets: Record<string, string[]> = {};
+
+  const results = sarif.runs[0]?.results;
+  if (!results) {
+    throw new Error("No results");
+  }
+
+  for (const result of results) {
+    const signature = result.message.text;
+
+    const snippet =
+      result.locations?.[0]?.physicalLocation?.contextRegion?.snippet?.text;
+
+    if (!signature || !snippet) {
+      continue;
+    }
+
+    if (!(signature in snippets)) {
+      snippets[signature] = [];
+    }
+
+    snippets[signature].push(snippet);
+  }
+
+  return snippets;
+}
diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index c8084a5c9..2cfa0b334 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -11,6 +11,7 @@ export function createAutoModelRequest(
   language: string,
   externalApiUsages: ExternalApiUsage[],
   modeledMethods: Record<string, ModeledMethod>,
+  usages: Record<string, string[]>,
 ): ModelRequest {
   const request: ModelRequest = {
     language,
@@ -29,6 +30,10 @@ export function createAutoModelRequest(
       type: "none",
     };
 
+    const usagesForMethod =
+      usages[externalApiUsage.signature] ??
+      externalApiUsage.usages.map((usage) => usage.label);
+
     const numberOfArguments =
       externalApiUsage.methodParameters === "()"
         ? 0
@@ -48,9 +53,7 @@ export function createAutoModelRequest(
           modeledMethod.type === "none"
             ? undefined
             : toMethodClassification(modeledMethod),
-        usages: externalApiUsage.usages
-          .slice(0, 10)
-          .map((usage) => usage.label),
+        usages: usagesForMethod.slice(0, 10),
         input: `Argument[${argumentIndex}]`,
       };
 
diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
index e3f2c7d4c..8432d699c 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
@@ -45,6 +45,7 @@ import {
   parsePredictedClassifications,
 } from "./auto-model";
 import { showLlmGeneration } from "../config";
+import { getAutoModelUsages } from "./auto-model-usages-query";
 
 function getQlSubmoduleFolder(): WorkspaceFolder | undefined {
   const workspaceFolder = workspace.workspaceFolders?.find(
@@ -242,7 +243,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     const cancellationTokenSource = new CancellationTokenSource();
 
     try {
-      const queryResult = await runQuery({
+      const queryResult = await runQuery("mainQuery", {
         cliServer: this.cliServer,
         queryRunner: this.queryRunner,
         databaseItem: this.databaseItem,
@@ -385,23 +386,66 @@ export class DataExtensionsEditorView extends AbstractWebview<
     externalApiUsages: ExternalApiUsage[],
     modeledMethods: Record<string, ModeledMethod>,
   ): Promise<void> {
+    const maxStep = 3000;
+
+    await this.showProgress({
+      step: 0,
+      maxStep,
+      message: "Retrieving usages",
+    });
+
+    const usages = await getAutoModelUsages({
+      cliServer: this.cliServer,
+      queryRunner: this.queryRunner,
+      queryStorageDir: this.queryStorageDir,
+      databaseItem: this.databaseItem,
+      progress: (update) => this.showProgress(update, maxStep),
+    });
+
+    await this.showProgress({
+      step: 1800,
+      maxStep,
+      message: "Creating request",
+    });
+
     const request = createAutoModelRequest(
       this.databaseItem.language,
       externalApiUsages,
       modeledMethods,
+      usages,
     );
 
+    await this.showProgress({
+      step: 2000,
+      maxStep,
+      message: "Sending request",
+    });
+
     const response = await autoModel(this.app.credentials, request);
 
+    await this.showProgress({
+      step: 2500,
+      maxStep,
+      message: "Parsing response",
+    });
+
     const predictedModeledMethods = parsePredictedClassifications(
       response.predicted,
     );
 
+    await this.showProgress({
+      step: 2800,
+      maxStep,
+      message: "Applying results",
+    });
+
     await this.postMessage({
       t: "addModeledMethods",
       modeledMethods: predictedModeledMethods,
       overrideNone: true,
     });
+
+    await this.clearProgress();
   }
 
   /*
diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
index 9fbc7b293..b8bbd88dd 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
@@ -16,6 +16,7 @@ import { QueryResultType } from "../pure/new-messages";
 import { join } from "path";
 import { redactableError } from "../pure/errors";
 import { QueryLanguage } from "../common/query-language";
+import { Query } from "./queries/query";
 
 export type RunQueryOptions = {
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
@@ -27,14 +28,17 @@ export type RunQueryOptions = {
   token: CancellationToken;
 };
 
-export async function runQuery({
-  cliServer,
-  queryRunner,
-  databaseItem,
-  queryStorageDir,
-  progress,
-  token,
-}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
+export async function runQuery(
+  queryName: keyof Omit<Query, "dependencies">,
+  {
+    cliServer,
+    queryRunner,
+    databaseItem,
+    queryStorageDir,
+    progress,
+    token,
+  }: RunQueryOptions,
+): Promise<CoreCompletedQuery | undefined> {
   // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
   // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
   // This is intentionally not pretty code, as it will be removed soon.
@@ -51,7 +55,7 @@ export async function runQuery({
 
   const queryDir = (await dir({ unsafeCleanup: true })).path;
   const queryFile = join(queryDir, "FetchExternalApis.ql");
-  await writeFile(queryFile, query.mainQuery, "utf8");
+  await writeFile(queryFile, query[queryName], "utf8");
 
   if (query.dependencies) {
     for (const [filename, contents] of Object.entries(query.dependencies)) {
diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
index 5dc25a9eb..7bdcfe483 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
@@ -2,33 +2,52 @@ import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
   mainQuery: `/**
- * @name Usage of APIs coming from external libraries
- * @description A list of 3rd party APIs used in the codebase.
- * @tags telemetry
- * @id cs/telemetry/fetch-external-apis
- */
+* @name Usage of APIs coming from external libraries
+* @description A list of 3rd party APIs used in the codebase.
+* @tags telemetry
+* @id cs/telemetry/fetch-external-apis
+*/
 
- import csharp
- import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
- import ExternalApi
- 
- private Call aUsage(ExternalApi api) {
-   result.getTarget().getUnboundDeclaration() = api
- }
- 
- private boolean isSupported(ExternalApi api) {
-   api.isSupported() and result = true
-   or
-   not api.isSupported() and
-   result = false
- }
- 
- from ExternalApi api, string apiName, boolean supported, Call usage
- where
-   apiName = api.getApiName() and
-   supported = isSupported(api) and
-   usage = aUsage(api)
- select apiName, supported, usage
+import csharp
+import ExternalApi
+
+private Call aUsage(ExternalApi api) {
+ result.getTarget().getUnboundDeclaration() = api
+}
+
+private boolean isSupported(ExternalApi api) {
+ api.isSupported() and result = true
+ or
+ not api.isSupported() and
+ result = false
+}
+
+from ExternalApi api, string apiName, boolean supported, Call usage
+where
+ apiName = api.getApiName() and
+ supported = isSupported(api) and
+ usage = aUsage(api)
+select apiName, supported, usage
+`,
+  usagesQuery: `/**
+* @name Usage of APIs coming from external libraries
+* @description A list of 3rd party APIs used in the codebase.
+* @kind problem
+* @id cs/telemetry/fetch-external-api-usages
+*/
+
+import csharp
+import ExternalApi
+
+private Call aUsage(ExternalApi api) {
+ result.getTarget().getUnboundDeclaration() = api
+}
+
+from ExternalApi api, string apiName, Call usage
+where
+ apiName = api.getApiName() and
+ usage = aUsage(api)
+select usage, apiName
 `,
   dependencies: {
     "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
index 721673e40..4664dfaa8 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
@@ -9,7 +9,6 @@ export const fetchExternalApisQuery: Query = {
  */
 
 import java
-import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 import ExternalApi
 
 private Call aUsage(ExternalApi api) {
@@ -29,6 +28,27 @@ where
   supported = isSupported(api) and
   usage = aUsage(api)
 select apiName, supported, usage
+`,
+  usagesQuery: `/**
+ * @name Usage of APIs coming from external libraries
+ * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
+ * @kind problem
+ * @id java/telemetry/fetch-external-api-usages
+ */
+
+import java
+import ExternalApi
+
+private Call aUsage(ExternalApi api) {
+  result.getCallee().getSourceDeclaration() = api and
+  not result.getFile() instanceof GeneratedFile
+}
+
+from ExternalApi api, string apiName, Call usage
+where
+  apiName = api.getApiName() and
+  usage = aUsage(api)
+select usage, apiName
 `,
   dependencies: {
     "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
index 72f239529..7c46bec49 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
@@ -1,5 +1,6 @@
 export type Query = {
   mainQuery: string;
+  usagesQuery: string;
   dependencies?: {
     [filename: string]: string;
   };
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
index 16f184324..efbbb9b96 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -121,7 +121,9 @@ export class DatabaseItemImpl implements DatabaseItem {
   /**
    * Returns information about a database.
    */
-  private async getDbInfo(server: cli.CodeQLCliServer): Promise<cli.DbInfo> {
+  private async getDbInfo(
+    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
+  ): Promise<cli.DbInfo> {
     if (this._dbinfo === undefined) {
       this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath);
     }
@@ -133,7 +135,7 @@ export class DatabaseItemImpl implements DatabaseItem {
    * has a `.dbinfo` file, which is the source of the prefix.
    */
   public async getSourceLocationPrefix(
-    server: cli.CodeQLCliServer,
+    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
   ): Promise<string> {
     const dbInfo = await this.getDbInfo(server);
     return dbInfo.sourceLocationPrefix;
@@ -142,7 +144,9 @@ export class DatabaseItemImpl implements DatabaseItem {
   /**
    * Returns path to dataset folder of database.
    */
-  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
+  public async getDatasetFolder(
+    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
+  ): Promise<string> {
     const dbInfo = await this.getDbInfo(server);
     return dbInfo.datasetFolder;
   }
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
index 1794d8e75..ac5ed47d9 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
@@ -43,12 +43,16 @@ export interface DatabaseItem {
   /**
    * Returns `sourceLocationPrefix` of exported database.
    */
-  getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
+  getSourceLocationPrefix(
+    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
+  ): Promise<string>;
 
   /**
    * Returns dataset folder of exported database.
    */
-  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
+  getDatasetFolder(
+    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
+  ): Promise<string>;
 
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive,
diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts
index d4e8a9a8c..374e066d7 100644
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@@ -786,7 +786,7 @@ export async function askForLanguage(
  * @returns A promise that resolves to the query metadata, if available.
  */
 export async function tryGetQueryMetadata(
-  cliServer: CodeQLCliServer,
+  cliServer: Pick<CodeQLCliServer, "resolveMetadata">,
   queryPath: string,
 ): Promise<QueryMetadata | undefined> {
   try {
diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts
index 0cab84e9c..767729e23 100644
--- a/extensions/ql-vscode/src/query-results.ts
+++ b/extensions/ql-vscode/src/query-results.ts
@@ -135,10 +135,11 @@ export class CompletedQueryInfo implements QueryWithResults {
  * Call cli command to interpret SARIF results.
  */
 export async function interpretResultsSarif(
-  cli: cli.CodeQLCliServer,
+  cli: Pick<cli.CodeQLCliServer, "interpretBqrsSarif">,
   metadata: QueryMetadata | undefined,
   resultsPaths: ResultsPaths,
   sourceInfo?: cli.SourceInfo,
+  args?: string[],
 ): Promise<SarifInterpretationData> {
   const { resultsPath, interpretedResultsPath } = resultsPaths;
   let res;
@@ -150,6 +151,7 @@ export async function interpretResultsSarif(
       resultsPath,
       interpretedResultsPath,
       sourceInfo,
+      args,
     );
   }
   return { ...res, t: "SarifInterpretationData" };
diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
index bb7bc07ac..c2f372f01 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/auto-model.test.ts
@@ -200,9 +200,36 @@ describe("createAutoModelRequest", () => {
     },
   };
 
+  const usages: Record<string, string[]> = {
+    "org.springframework.boot.SpringApplication#run(Class,String[])": [
+      "public class Sql2oExampleApplication {\n    public static void main(String[] args) {\n        SpringApplication.run(Sql2oExampleApplication.class, args);\n    }\n}",
+    ],
+    "org.sql2o.Connection#createQuery(String)": [
+      '    public String index(@RequestParam("id") String id) {\n        try (var con = sql2o.open()) {\n            con.createQuery("select 1 where id = " + id).executeScalar(Integer.class);\n        }\n\n',
+      '\n        try (var con = sql2o.open()) {\n            con.createQuery("select 1").executeScalar(Integer.class);\n        }\n\n',
+    ],
+    "org.sql2o.Query#executeScalar(Class)": [
+      '    public String index(@RequestParam("id") String id) {\n        try (var con = sql2o.open()) {\n            con.createQuery("select 1 where id = " + id).executeScalar(Integer.class);\n        }\n\n',
+      '\n        try (var con = sql2o.open()) {\n            con.createQuery("select 1").executeScalar(Integer.class);\n        }\n\n',
+    ],
+    "org.sql2o.Sql2o#open()": [
+      '    @GetMapping("/")\n    public String index(@RequestParam("id") String id) {\n        try (var con = sql2o.open()) {\n            con.createQuery("select 1 where id = " + id).executeScalar(Integer.class);\n        }\n',
+      '        Sql2o sql2o = new Sql2o(url);\n\n        try (var con = sql2o.open()) {\n            con.createQuery("select 1").executeScalar(Integer.class);\n        }\n',
+    ],
+    "java.io.PrintStream#println(String)": [
+      '        }\n\n        System.out.println("Connected to " + url);\n\n        return "Greetings from Spring Boot!";\n',
+    ],
+    "org.sql2o.Sql2o#Sql2o(String,String,String)": [
+      '@RestController\npublic class HelloController {\n    private final Sql2o sql2o = new Sql2o("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1","sa", "");\n\n    @GetMapping("/")\n',
+    ],
+    "org.sql2o.Sql2o#Sql2o(String)": [
+      '    @GetMapping("/connect")\n    public String connect(@RequestParam("url") String url) {\n        Sql2o sql2o = new Sql2o(url);\n\n        try (var con = sql2o.open()) {\n',
+    ],
+  };
+
   it("creates a matching request", () => {
     expect(
-      createAutoModelRequest("java", externalApiUsages, modeledMethods),
+      createAutoModelRequest("java", externalApiUsages, modeledMethods, usages),
     ).toEqual({
       language: "java",
       samples: [
@@ -216,7 +243,7 @@ describe("createAutoModelRequest", () => {
             kind: "jndi-injection",
             explanation: "",
           },
-          usages: ["new Sql2o(...)"],
+          usages: usages["org.sql2o.Sql2o#Sql2o(String)"],
           input: "Argument[0]",
         },
       ],
@@ -226,64 +253,78 @@ describe("createAutoModelRequest", () => {
           type: "Connection",
           name: "createQuery",
           signature: "(String)",
-          usages: ["createQuery(...)", "createQuery(...)"],
+          usages: usages["org.sql2o.Connection#createQuery(String)"],
           input: "Argument[0]",
+          classification: undefined,
         },
         {
           package: "org.sql2o",
           type: "Query",
           name: "executeScalar",
           signature: "(Class)",
-          usages: ["executeScalar(...)", "executeScalar(...)"],
+          usages: usages["org.sql2o.Query#executeScalar(Class)"],
           input: "Argument[0]",
+          classification: undefined,
         },
         {
           package: "org.springframework.boot",
           type: "SpringApplication",
           name: "run",
           signature: "(Class,String[])",
-          usages: ["run(...)"],
+          usages:
+            usages[
+              "org.springframework.boot.SpringApplication#run(Class,String[])"
+            ],
           input: "Argument[0]",
+          classification: undefined,
         },
         {
           package: "org.springframework.boot",
           type: "SpringApplication",
           name: "run",
           signature: "(Class,String[])",
-          usages: ["run(...)"],
+          usages:
+            usages[
+              "org.springframework.boot.SpringApplication#run(Class,String[])"
+            ],
           input: "Argument[1]",
+          classification: undefined,
         },
         {
           package: "java.io",
           type: "PrintStream",
           name: "println",
           signature: "(String)",
-          usages: ["println(...)"],
+          usages: usages["java.io.PrintStream#println(String)"],
           input: "Argument[0]",
+          classification: undefined,
         },
         {
           package: "org.sql2o",
           type: "Sql2o",
           name: "Sql2o",
           signature: "(String,String,String)",
-          usages: ["new Sql2o(...)"],
+          usages: usages["org.sql2o.Sql2o#Sql2o(String,String,String)"],
           input: "Argument[0]",
+          classification: undefined,
         },
         {
           package: "org.sql2o",
           type: "Sql2o",
           name: "Sql2o",
           signature: "(String,String,String)",
-          usages: ["new Sql2o(...)"],
+          usages: usages["org.sql2o.Sql2o#Sql2o(String,String,String)"],
           input: "Argument[1]",
+          classification: undefined,
         },
         {
           package: "org.sql2o",
           type: "Sql2o",
           name: "Sql2o",
           signature: "(String,String,String)",
-          usages: ["new Sql2o(...)"],
+          usages: usages["org.sql2o.Sql2o#Sql2o(String,String,String)"],
           input: "Argument[2]",
+          classification: undefined,
         },
       ],
     });
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
index a4b5ec0e0..24a4f6e48 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
@@ -66,7 +66,7 @@ describe("runQuery", () => {
           onCancellationRequested: jest.fn(),
         },
       };
-      const result = await runQuery(options);
+      const result = await runQuery("mainQuery", options);
 
       expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
 
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts
index 1e7108abf..fe9a51c22 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/query-results.test.ts
@@ -225,6 +225,7 @@ describe("query-results", () => {
           resultsPath,
           interpretedResultsPath,
           sourceInfo,
+          undefined,
         );
       },
       2 * 60 * 1000, // up to 2 minutes per test
@@ -249,6 +250,7 @@ describe("query-results", () => {
           resultsPath,
           interpretedResultsPath,
           sourceInfo,
+          undefined,
         );
       },
       2 * 60 * 1000, // up to 2 minutes per test

From af3be543f515b4028d5692cc2ba30c42a1543c81 Mon Sep 17 00:00:00 2001
From: Alexander Eyers-Taylor <alexet@semmle.com>
Date: Tue, 30 May 2023 14:12:26 +0100
Subject: [PATCH 091/119] Cache MRVA queries correctly (#2435)

---
 extensions/ql-vscode/src/codeql-cli/cli.ts       |  6 ++++++
 .../src/variant-analysis/run-remote-query.ts     | 16 ++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts
index c055b4b2b..fe9e02115 100644
--- a/extensions/ql-vscode/src/codeql-cli/cli.ts
+++ b/extensions/ql-vscode/src/codeql-cli/cli.ts
@@ -1818,6 +1818,8 @@ export class CliVersionConstraint {
     "2.12.4",
   );
 
+  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1887,4 +1889,8 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
     );
   }
+
+  async usesGlobalCompilationCache() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
+  }
 }
diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
index 1ccf46042..a45418e3b 100644
--- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
+++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
@@ -116,12 +116,16 @@ async function generateQueryPack(
 
   let precompilationOpts: string[] = [];
   if (await cliServer.cliConstraints.supportsQlxRemote()) {
-    const ccache = join(originalPackRoot, ".cache");
-    precompilationOpts = [
-      "--qlx",
-      "--no-default-compilation-cache",
-      `--compilation-cache=${ccache}`,
-    ];
+    if (await cliServer.cliConstraints.usesGlobalCompilationCache()) {
+      precompilationOpts = ["--qlx"];
+    } else {
+      const ccache = join(originalPackRoot, ".cache");
+      precompilationOpts = [
+        "--qlx",
+        "--no-default-compilation-cache",
+        `--compilation-cache=${ccache}`,
+      ];
+    }
   } else {
     precompilationOpts = ["--no-precompile"];
   }

From 22172d5d74e5633dd65c47fb04ea3da0183e7140 Mon Sep 17 00:00:00 2001
From: Alexander Eyers-Taylor <alexet@semmle.com>
Date: Tue, 30 May 2023 14:12:47 +0100
Subject: [PATCH 092/119] Add support for quick eval count to the query runner
 (#2417)

* Add support for quick eval count to the query runner

This only adds support internally to the query runner,
without any UI support.

* Fix some tests
---
 .../external-api-usage-query.ts               |  6 ++-
 .../generate-flow-model.ts                    |  6 ++-
 .../src/debugger/debug-configuration.ts       |  2 +-
 .../ql-vscode/src/debugger/debug-session.ts   |  1 +
 .../ql-vscode/src/debugger/debugger-ui.ts     |  2 +-
 .../ast-viewer/ast-cfg-commands.ts            |  4 +-
 .../src/local-queries/local-queries.ts        | 44 ++++++++++++++-----
 .../ql-vscode/src/pure/messages-shared.ts     |  8 ++++
 .../src/query-server/query-runner.ts          |  4 ++
 .../ql-vscode/src/query-server/run-queries.ts |  5 ++-
 .../ql-vscode/src/run-queries-shared.ts       |  3 ++
 .../cli-integration/queries.test.ts           | 10 ++---
 .../determining-selected-query-test.ts        |  6 +--
 .../external-api-usage-query.test.ts          |  1 +
 14 files changed, 77 insertions(+), 25 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
index 9fbc7b293..659f8d519 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
@@ -78,7 +78,11 @@ export async function runQuery({
 
   const queryRun = queryRunner.createQueryRun(
     databaseItem.databaseUri.fsPath,
-    { queryPath: queryFile, quickEvalPosition: undefined },
+    {
+      queryPath: queryFile,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
     false,
     getOnDiskWorkspaceFolders(),
     extensionPacks,
diff --git a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts
index 6f48d8587..d8330953e 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts
@@ -53,7 +53,11 @@ async function getModeledMethodsFromFlow(
 
   const queryRun = queryRunner.createQueryRun(
     databaseItem.databaseUri.fsPath,
-    { queryPath: query, quickEvalPosition: undefined },
+    {
+      queryPath: query,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
     false,
     getOnDiskWorkspaceFolders(),
     undefined,
diff --git a/extensions/ql-vscode/src/debugger/debug-configuration.ts b/extensions/ql-vscode/src/debugger/debug-configuration.ts
index c25e4fc6f..4d4a788a7 100644
--- a/extensions/ql-vscode/src/debugger/debug-configuration.ts
+++ b/extensions/ql-vscode/src/debugger/debug-configuration.ts
@@ -105,7 +105,7 @@ export class QLDebugConfigurationProvider
       validateQueryPath(qlConfiguration.query, quickEval);
 
       const quickEvalContext = quickEval
-        ? await getQuickEvalContext(undefined)
+        ? await getQuickEvalContext(undefined, false)
         : undefined;
 
       const resultConfiguration: QLResolvedDebugConfiguration = {
diff --git a/extensions/ql-vscode/src/debugger/debug-session.ts b/extensions/ql-vscode/src/debugger/debug-session.ts
index 9b9091092..02c183200 100644
--- a/extensions/ql-vscode/src/debugger/debug-session.ts
+++ b/extensions/ql-vscode/src/debugger/debug-session.ts
@@ -155,6 +155,7 @@ class RunningQuery extends DisposableObject {
       {
         queryPath: config.query,
         quickEvalPosition: quickEvalContext?.quickEvalPosition,
+        quickEvalCountOnly: quickEvalContext?.quickEvalCount,
       },
       true,
       config.additionalPacks,
diff --git a/extensions/ql-vscode/src/debugger/debugger-ui.ts b/extensions/ql-vscode/src/debugger/debugger-ui.ts
index e6a115a43..46168e021 100644
--- a/extensions/ql-vscode/src/debugger/debugger-ui.ts
+++ b/extensions/ql-vscode/src/debugger/debugger-ui.ts
@@ -74,7 +74,7 @@ class QLDebugAdapterTracker
 
   public async quickEval(): Promise<void> {
     const args: CodeQLProtocol.QuickEvalRequest["arguments"] = {
-      quickEvalContext: await getQuickEvalContext(undefined),
+      quickEvalContext: await getQuickEvalContext(undefined, false),
     };
     await this.session.customRequest("codeql-quickeval", args);
   }
diff --git a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts
index 3325c9df0..3262fbc8f 100644
--- a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts
+++ b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts
@@ -2,7 +2,7 @@ import { Uri, window } from "vscode";
 import { withProgress } from "../../common/vscode/progress";
 import { AstViewer } from "./ast-viewer";
 import { AstCfgCommands } from "../../common/commands";
-import { LocalQueries } from "../../local-queries";
+import { LocalQueries, QuickEvalType } from "../../local-queries";
 import {
   TemplatePrintAstProvider,
   TemplatePrintCfgProvider,
@@ -47,7 +47,7 @@ export function getAstCfgCommands({
         );
         if (res) {
           await localQueries.compileAndRunQuery(
-            false,
+            QuickEvalType.None,
             res[0],
             progress,
             token,
diff --git a/extensions/ql-vscode/src/local-queries/local-queries.ts b/extensions/ql-vscode/src/local-queries/local-queries.ts
index b7250cbb4..a5b7f3028 100644
--- a/extensions/ql-vscode/src/local-queries/local-queries.ts
+++ b/extensions/ql-vscode/src/local-queries/local-queries.ts
@@ -72,6 +72,12 @@ async function promptToSaveQueryIfNeeded(query: SelectedQuery): Promise<void> {
   }
 }
 
+export enum QuickEvalType {
+  None,
+  QuickEval,
+  QuickEvalCount,
+}
+
 export class LocalQueries extends DisposableObject {
   public constructor(
     private readonly app: App,
@@ -115,7 +121,13 @@ export class LocalQueries extends DisposableObject {
   private async runQuery(uri: Uri | undefined): Promise<void> {
     await withProgress(
       async (progress, token) => {
-        await this.compileAndRunQuery(false, uri, progress, token, undefined);
+        await this.compileAndRunQuery(
+          QuickEvalType.None,
+          uri,
+          progress,
+          token,
+          undefined,
+        );
       },
       {
         title: "Running query",
@@ -185,7 +197,7 @@ export class LocalQueries extends DisposableObject {
         await Promise.all(
           queryUris.map(async (uri) =>
             this.compileAndRunQuery(
-              false,
+              QuickEvalType.None,
               uri,
               wrappedProgress,
               token,
@@ -204,7 +216,13 @@ export class LocalQueries extends DisposableObject {
   private async quickEval(uri: Uri): Promise<void> {
     await withProgress(
       async (progress, token) => {
-        await this.compileAndRunQuery(true, uri, progress, token, undefined);
+        await this.compileAndRunQuery(
+          QuickEvalType.QuickEval,
+          uri,
+          progress,
+          token,
+          undefined,
+        );
       },
       {
         title: "Running query",
@@ -217,7 +235,7 @@ export class LocalQueries extends DisposableObject {
     await withProgress(
       async (progress, token) =>
         await this.compileAndRunQuery(
-          true,
+          QuickEvalType.QuickEval,
           uri,
           progress,
           token,
@@ -331,7 +349,7 @@ export class LocalQueries extends DisposableObject {
   }
 
   public async compileAndRunQuery(
-    quickEval: boolean,
+    quickEval: QuickEvalType,
     queryUri: Uri | undefined,
     progress: ProgressCallback,
     token: CancellationToken,
@@ -352,7 +370,7 @@ export class LocalQueries extends DisposableObject {
 
   /** Used by tests */
   public async compileAndRunQueryInternal(
-    quickEval: boolean,
+    quickEval: QuickEvalType,
     queryUri: Uri | undefined,
     progress: ProgressCallback,
     token: CancellationToken,
@@ -364,15 +382,20 @@ export class LocalQueries extends DisposableObject {
     if (queryUri !== undefined) {
       // The query URI is provided by the command, most likely because the command was run from an
       // editor context menu. Use the provided URI, but make sure it's a valid query.
-      queryPath = validateQueryUri(queryUri, quickEval);
+      queryPath = validateQueryUri(queryUri, quickEval !== QuickEvalType.None);
     } else {
       // Use the currently selected query.
-      queryPath = await this.getCurrentQuery(quickEval);
+      queryPath = await this.getCurrentQuery(quickEval !== QuickEvalType.None);
     }
 
     const selectedQuery: SelectedQuery = {
       queryPath,
-      quickEval: quickEval ? await getQuickEvalContext(range) : undefined,
+      quickEval: quickEval
+        ? await getQuickEvalContext(
+            range,
+            quickEval === QuickEvalType.QuickEvalCount,
+          )
+        : undefined,
     };
 
     // If no databaseItem is specified, use the database currently selected in the Databases UI
@@ -392,6 +415,7 @@ export class LocalQueries extends DisposableObject {
       {
         queryPath: selectedQuery.queryPath,
         quickEvalPosition: selectedQuery.quickEval?.quickEvalPosition,
+        quickEvalCountOnly: selectedQuery.quickEval?.quickEvalCount,
       },
       true,
       additionalPacks,
@@ -481,7 +505,7 @@ export class LocalQueries extends DisposableObject {
       for (const item of quickpick) {
         try {
           await this.compileAndRunQuery(
-            false,
+            QuickEvalType.None,
             uri,
             progress,
             token,
diff --git a/extensions/ql-vscode/src/pure/messages-shared.ts b/extensions/ql-vscode/src/pure/messages-shared.ts
index 23b5a6871..a07d81f82 100644
--- a/extensions/ql-vscode/src/pure/messages-shared.ts
+++ b/extensions/ql-vscode/src/pure/messages-shared.ts
@@ -68,6 +68,14 @@ export interface CompilationTarget {
  */
 export interface QuickEvalOptions {
   quickEvalPos?: Position;
+  /**
+   * Whether to only count the number of results.
+   *
+   * This is only supported by the new query server
+   * but it isn't worth having a separate type and
+   * it is fine to have an ignored optional field.
+   */
+  countOnly?: boolean;
 }
 
 /**
diff --git a/extensions/ql-vscode/src/query-server/query-runner.ts b/extensions/ql-vscode/src/query-server/query-runner.ts
index 95d7eafef..03f886ffd 100644
--- a/extensions/ql-vscode/src/query-server/query-runner.ts
+++ b/extensions/ql-vscode/src/query-server/query-runner.ts
@@ -16,6 +16,10 @@ export interface CoreQueryTarget {
    * `query`.
    */
   quickEvalPosition?: Position;
+  /**
+   * If this is quick eval, whether to only count the number of results.
+   */
+  quickEvalCountOnly?: boolean;
 }
 
 export interface CoreQueryResults {
diff --git a/extensions/ql-vscode/src/query-server/run-queries.ts b/extensions/ql-vscode/src/query-server/run-queries.ts
index 9f9e441c6..66674bab9 100644
--- a/extensions/ql-vscode/src/query-server/run-queries.ts
+++ b/extensions/ql-vscode/src/query-server/run-queries.ts
@@ -36,7 +36,10 @@ export async function compileAndRunQueryAgainstDatabaseCore(
   const target =
     query.quickEvalPosition !== undefined
       ? {
-          quickEval: { quickEvalPos: query.quickEvalPosition },
+          quickEval: {
+            quickEvalPos: query.quickEvalPosition,
+            countOnly: query.quickEvalCountOnly,
+          },
         }
       : { query: {} };
 
diff --git a/extensions/ql-vscode/src/run-queries-shared.ts b/extensions/ql-vscode/src/run-queries-shared.ts
index cdb472a0b..811fe57df 100644
--- a/extensions/ql-vscode/src/run-queries-shared.ts
+++ b/extensions/ql-vscode/src/run-queries-shared.ts
@@ -433,6 +433,7 @@ export function validateQueryPath(
 export interface QuickEvalContext {
   quickEvalPosition: messages.Position;
   quickEvalText: string;
+  quickEvalCount: boolean;
 }
 
 /**
@@ -443,6 +444,7 @@ export interface QuickEvalContext {
  */
 export async function getQuickEvalContext(
   range: Range | undefined,
+  isCountOnly: boolean,
 ): Promise<QuickEvalContext> {
   const editor = window.activeTextEditor;
   if (editor === undefined) {
@@ -465,6 +467,7 @@ export async function getQuickEvalContext(
   return {
     quickEvalPosition,
     quickEvalText,
+    quickEvalCount: isCountOnly,
   };
 }
 
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
index 491e6d037..d068a1295 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/queries.test.ts
@@ -28,7 +28,7 @@ import {
   QueryRunner,
 } from "../../../src/query-server/query-runner";
 import { SELECT_QUERY_NAME } from "../../../src/language-support";
-import { LocalQueries } from "../../../src/local-queries";
+import { LocalQueries, QuickEvalType } from "../../../src/local-queries";
 import { QueryResultType } from "../../../src/pure/new-messages";
 import { createVSCodeCommandManager } from "../../../src/common/vscode/commands";
 import {
@@ -45,7 +45,7 @@ async function compileAndRunQuery(
   mode: DebugMode,
   appCommands: AppCommandManager,
   localQueries: LocalQueries,
-  quickEval: boolean,
+  quickEval: QuickEvalType,
   queryUri: Uri,
   progress: ProgressCallback,
   token: CancellationToken,
@@ -184,7 +184,7 @@ describeWithCodeQL()("Queries", () => {
         mode,
         appCommandManager,
         localQueries,
-        false,
+        QuickEvalType.None,
         Uri.file(queryUsingExtensionPath),
         progress,
         token,
@@ -218,7 +218,7 @@ describeWithCodeQL()("Queries", () => {
         mode,
         appCommandManager,
         localQueries,
-        false,
+        QuickEvalType.None,
         Uri.file(queryPath),
         progress,
         token,
@@ -238,7 +238,7 @@ describeWithCodeQL()("Queries", () => {
         mode,
         appCommandManager,
         localQueries,
-        false,
+        QuickEvalType.None,
         Uri.file(queryPath),
         progress,
         token,
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts
index 921bf037f..61eeb724f 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/determining-selected-query-test.ts
@@ -26,7 +26,7 @@ export function run() {
 
     it("should allow ql files to be quick-evaled", async () => {
       await showQlDocument("query.ql");
-      const q = await getQuickEvalContext(undefined);
+      const q = await getQuickEvalContext(undefined, false);
       expect(
         q.quickEvalPosition.fileName.endsWith(
           join("ql-vscode", "test", "data", "query.ql"),
@@ -36,7 +36,7 @@ export function run() {
 
     it("should allow qll files to be quick-evaled", async () => {
       await showQlDocument("library.qll");
-      const q = await getQuickEvalContext(undefined);
+      const q = await getQuickEvalContext(undefined, false);
       expect(
         q.quickEvalPosition.fileName.endsWith(
           join("ql-vscode", "test", "data", "library.qll"),
@@ -55,7 +55,7 @@ export function run() {
 
     it("should reject non-ql[l] files when running a quick eval", async () => {
       await showQlDocument("textfile.txt");
-      await expect(getQuickEvalContext(undefined)).rejects.toThrow(
+      await expect(getQuickEvalContext(undefined, false)).rejects.toThrow(
         "The selected resource is not a CodeQL file",
       );
     });
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
index a4b5ec0e0..548637cc6 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
@@ -77,6 +77,7 @@ describe("runQuery", () => {
         {
           queryPath: expect.stringMatching(/FetchExternalApis\.ql/),
           quickEvalPosition: undefined,
+          quickEvalCountOnly: false,
         },
         false,
         [],

From 4087620bf5774453f990802159377b13d4f8b0fb Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Tue, 30 May 2023 14:30:07 +0000
Subject: [PATCH 093/119] Polish copy

---
 extensions/ql-vscode/package.json                 | 2 +-
 extensions/ql-vscode/src/databases/ui/db-panel.ts | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json
index 997073dd7..c4000edef 100644
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@@ -518,7 +518,7 @@
       },
       {
         "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
-        "title": "Import Repos from GitHub Code Search"
+        "title": "Add repositories with GitHub Code Search"
       },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts
index c649d8619..9172b2822 100644
--- a/extensions/ql-vscode/src/databases/ui/db-panel.ts
+++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts
@@ -363,7 +363,7 @@ export class DbPanel extends DisposableObject {
       await window.showQuickPick<CodeSearchQuickPickItem>(
         languageQuickPickItems,
         {
-          title: "Select the language you want to query",
+          title: "Select a language for your search",
           placeHolder: "Select an option",
           ignoreFocusOut: true,
         },
@@ -373,8 +373,9 @@ export class DbPanel extends DisposableObject {
     }
 
     const codeSearchQuery = await window.showInputBox({
-      title: "Code search query",
-      prompt: "Insert code search query",
+      title: "GitHub Code Search",
+      prompt:
+        "Use [GitHub's Code Search syntax](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax), including code qualifiers, regular expressions, and boolean operations, to search for repositories.",
       placeHolder: "org:github",
     });
     if (codeSearchQuery === undefined || codeSearchQuery === "") {

From 82766d1033f057043c73b263143c7d08181de7d0 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 30 May 2023 15:06:28 +0100
Subject: [PATCH 094/119] Rename: getCurrentRefreshPromise =>
 waitForCurrentRefresh

---
 extensions/ql-vscode/src/common/discovery.ts                  | 2 +-
 .../minimal-workspace/queries-panel/query-discovery.test.ts   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index dad1400a9..25398f702 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -19,7 +19,7 @@ export abstract class Discovery<T> extends DisposableObject {
    * Returns the promise of the currently running refresh operation, if one is in progress.
    * Otherwise returns a promise that resolves immediately.
    */
-  public getCurrentRefreshPromise(): Promise<void> {
+  public waitForCurrentRefresh(): Promise<void> {
     return this.currentDiscoveryPromise ?? Promise.resolve();
   }
 
diff --git a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
index 55875603c..563ee6119 100644
--- a/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/minimal-workspace/queries-panel/query-discovery.test.ts
@@ -170,7 +170,7 @@ describe("QueryDiscovery", () => {
 
       onWatcherDidChangeEvent.fire(workspace.workspaceFolders![0].uri);
 
-      await discovery.getCurrentRefreshPromise();
+      await discovery.waitForCurrentRefresh();
 
       expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
     });
@@ -200,7 +200,7 @@ describe("QueryDiscovery", () => {
 
       onDidChangeWorkspaceFoldersEvent.fire({ added: [], removed: [] });
 
-      await discovery.getCurrentRefreshPromise();
+      await discovery.waitForCurrentRefresh();
 
       expect(onDidChangeQueriesSpy).toHaveBeenCalledTimes(2);
     });

From d2bb1b844e0612c2b67dba4003d9ebe07af4e1ff Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 30 May 2023 15:11:39 +0100
Subject: [PATCH 095/119] Fix comment to be more accurate

---
 extensions/ql-vscode/src/common/discovery.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index 25398f702..d187f3368 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -8,7 +8,7 @@ import { getErrorMessage } from "../pure/helpers-pure";
  * same time.
  */
 export abstract class Discovery<T> extends DisposableObject {
-  private retry = false;
+  private restartWhenFinished = false;
   private currentDiscoveryPromise: Promise<void> | undefined;
 
   constructor(private readonly name: string) {
@@ -48,7 +48,7 @@ export abstract class Discovery<T> extends DisposableObject {
 
     if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
-      this.retry = true;
+      this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
       this.currentDiscoveryPromise = this.launchDiscovery();
@@ -72,13 +72,13 @@ export abstract class Discovery<T> extends DisposableObject {
       results = undefined;
     }
 
-    if (this.retry) {
+    if (this.restartWhenFinished) {
       // Another refresh request came in while we were still running a previous discovery
       // operation. Since the discovery results we just computed are now stale, we'll launch
       // another discovery operation instead of updating.
-      // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-      // initial discovery operation failed.
-      this.retry = false;
+      // We want to relaunch discovery regardless of if the initial discovery operation
+      // succeeded or failed.
+      this.restartWhenFinished = false;
       await this.launchDiscovery();
     } else {
       this.currentDiscoveryPromise = undefined;

From a0e63175590a0d87ef4eb8321bf322aaeaa982e1 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Tue, 30 May 2023 15:17:24 +0100
Subject: [PATCH 096/119] Keep all updates of currentDiscoveryPromise in
 refresh

---
 extensions/ql-vscode/src/common/discovery.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index d187f3368..140cc62cd 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -51,7 +51,9 @@ export abstract class Discovery<T> extends DisposableObject {
       this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
-      this.currentDiscoveryPromise = this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery().finally(() => {
+        this.currentDiscoveryPromise = undefined;
+      });
     }
     return this.currentDiscoveryPromise;
   }
@@ -81,8 +83,6 @@ export abstract class Discovery<T> extends DisposableObject {
       this.restartWhenFinished = false;
       await this.launchDiscovery();
     } else {
-      this.currentDiscoveryPromise = undefined;
-
       // If the discovery was successful, then update any listeners with the results.
       if (results !== undefined) {
         this.update(results);

From b8b378ffd4bca4fe9bee3f770772df7e7f0bf35c Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 10:59:50 +0100
Subject: [PATCH 097/119] Make assert-pure.ql ignore all /vscode/ directories

---
 .github/codeql/queries/assert-pure.ql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/codeql/queries/assert-pure.ql b/.github/codeql/queries/assert-pure.ql
index 63d9c8774..a4805b3e5 100644
--- a/.github/codeql/queries/assert-pure.ql
+++ b/.github/codeql/queries/assert-pure.ql
@@ -19,7 +19,7 @@ class PureFile extends File {
       this.getRelativePath().regexpMatch(".*/src/pure/.*") or
       this.getRelativePath().regexpMatch(".*/src/common/.*")
     ) and
-    not this.getRelativePath().regexpMatch(".*/src/common/vscode/.*")
+    not this.getRelativePath().regexpMatch(".*/vscode/.*")
   }
 }
 

From a4a67856a582c82deb74d88fb4ed02a2faf20198 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 11:26:25 +0100
Subject: [PATCH 098/119] Avoid vscode reference rfom discovery.ts

---
 extensions/ql-vscode/src/common/discovery.ts               | 6 +++---
 extensions/ql-vscode/src/queries-panel/query-discovery.ts  | 3 ++-
 extensions/ql-vscode/src/query-testing/qltest-discovery.ts | 3 ++-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/extensions/ql-vscode/src/common/discovery.ts b/extensions/ql-vscode/src/common/discovery.ts
index 8c9f970d5..1d3b39591 100644
--- a/extensions/ql-vscode/src/common/discovery.ts
+++ b/extensions/ql-vscode/src/common/discovery.ts
@@ -1,6 +1,6 @@
 import { DisposableObject } from "../pure/disposable-object";
-import { extLogger } from "./logging/vscode/loggers";
 import { getErrorMessage } from "../pure/helpers-pure";
+import { Logger } from "./logging";
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
@@ -11,7 +11,7 @@ export abstract class Discovery<T> extends DisposableObject {
   private retry = false;
   private discoveryInProgress = false;
 
-  constructor(private readonly name: string) {
+  constructor(private readonly name: string, private readonly logger: Logger) {
     super();
   }
 
@@ -63,7 +63,7 @@ export abstract class Discovery<T> extends DisposableObject {
       })
 
       .catch((err: unknown) => {
-        void extLogger.log(
+        void this.logger.log(
           `${this.name} failed. Reason: ${getErrorMessage(err)}`,
         );
       })
diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index bde646fbb..d1a077da0 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -8,6 +8,7 @@ import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
 import { getOnDiskWorkspaceFoldersObjects } from "../helpers";
 import { AppEventEmitter } from "../common/events";
 import { QueryDiscoverer } from "./query-tree-data-provider";
+import { extLogger } from "../common";
 
 /**
  * The results of discovering queries.
@@ -41,7 +42,7 @@ export class QueryDiscovery
   );
 
   constructor(app: App, private readonly cliServer: CodeQLCliServer) {
-    super("Query Discovery");
+    super("Query Discovery", extLogger);
 
     this.onDidChangeQueriesEmitter = this.push(app.createEventEmitter<void>());
     this.push(app.onDidChangeWorkspaceFolders(this.refresh.bind(this)));
diff --git a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
index b7d333ff6..db1cca7de 100644
--- a/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
+++ b/extensions/ql-vscode/src/query-testing/qltest-discovery.ts
@@ -11,6 +11,7 @@ import { MultiFileSystemWatcher } from "../common/vscode/multi-file-system-watch
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { pathExists } from "fs-extra";
 import { FileTreeDirectory, FileTreeLeaf } from "../common/file-tree-nodes";
+import { extLogger } from "../common";
 
 /**
  * The results of discovering QL tests.
@@ -42,7 +43,7 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
     private readonly workspaceFolder: WorkspaceFolder,
     private readonly cliServer: CodeQLCliServer,
   ) {
-    super("QL Test Discovery");
+    super("QL Test Discovery", extLogger);
 
     this.push(this.watcher.onDidChange(this.handleDidChange, this));
   }

From a117e09796de737868af2d078a292297db100ea1 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 31 May 2023 12:32:41 +0200
Subject: [PATCH 099/119] Use a single SARIF-compatible query instead of two
 separate queries

---
 .../auto-model-usages-query.ts                |   2 +-
 .../src/data-extensions-editor/bqrs.ts        |   6 +-
 .../data-extensions-editor-view.ts            |   2 +-
 .../external-api-usage-query.ts               |  22 +-
 .../data-extensions-editor/queries/csharp.ts  |  51 ++--
 .../data-extensions-editor/queries/java.ts    |  24 +-
 .../data-extensions-editor/queries/query.ts   |  10 +-
 .../ql-vscode/src/pure/bqrs-cli-types.ts      |   2 +-
 .../data-extensions-editor/bqrs.test.ts       | 247 +++++++++---------
 .../external-api-usage-query.test.ts          |  26 +-
 10 files changed, 185 insertions(+), 207 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
index 4f538427a..3b37def3e 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -37,7 +37,7 @@ export async function getAutoModelUsages({
 
   const cancellationTokenSource = new CancellationTokenSource();
 
-  const queryResult = await runQuery("usagesQuery", {
+  const queryResult = await runQuery({
     cliServer,
     queryRunner,
     queryStorageDir,
diff --git a/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts b/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts
index ed1f00c6a..a5c65fe13 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/bqrs.ts
@@ -7,9 +7,9 @@ export function decodeBqrsToExternalApiUsages(
   const methodsByApiName = new Map<string, ExternalApiUsage>();
 
   chunk?.tuples.forEach((tuple) => {
-    const signature = tuple[0] as string;
-    const supported = tuple[1] as boolean;
-    const usage = tuple[2] as Call;
+    const usage = tuple[0] as Call;
+    const signature = tuple[1] as string;
+    const supported = (tuple[2] as string) === "true";
 
     const [packageWithType, methodDeclaration] = signature.split("#");
 
diff --git a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
index 8432d699c..3ed055167 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts
@@ -243,7 +243,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     const cancellationTokenSource = new CancellationTokenSource();
 
     try {
-      const queryResult = await runQuery("mainQuery", {
+      const queryResult = await runQuery({
         cliServer: this.cliServer,
         queryRunner: this.queryRunner,
         databaseItem: this.databaseItem,
diff --git a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
index b8bbd88dd..9fbc7b293 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts
@@ -16,7 +16,6 @@ import { QueryResultType } from "../pure/new-messages";
 import { join } from "path";
 import { redactableError } from "../pure/errors";
 import { QueryLanguage } from "../common/query-language";
-import { Query } from "./queries/query";
 
 export type RunQueryOptions = {
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
@@ -28,17 +27,14 @@ export type RunQueryOptions = {
   token: CancellationToken;
 };
 
-export async function runQuery(
-  queryName: keyof Omit<Query, "dependencies">,
-  {
-    cliServer,
-    queryRunner,
-    databaseItem,
-    queryStorageDir,
-    progress,
-    token,
-  }: RunQueryOptions,
-): Promise<CoreCompletedQuery | undefined> {
+export async function runQuery({
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+  token,
+}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
   // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
   // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
   // This is intentionally not pretty code, as it will be removed soon.
@@ -55,7 +51,7 @@ export async function runQuery(
 
   const queryDir = (await dir({ unsafeCleanup: true })).path;
   const queryFile = join(queryDir, "FetchExternalApis.ql");
-  await writeFile(queryFile, query[queryName], "utf8");
+  await writeFile(queryFile, query.mainQuery, "utf8");
 
   if (query.dependencies) {
     for (const [filename, contents] of Object.entries(query.dependencies)) {
diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
index 7bdcfe483..ff3757fb3 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts
@@ -2,52 +2,31 @@ import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
   mainQuery: `/**
-* @name Usage of APIs coming from external libraries
-* @description A list of 3rd party APIs used in the codebase.
-* @tags telemetry
-* @id cs/telemetry/fetch-external-apis
-*/
+ * @name Usage of APIs coming from external libraries
+ * @description A list of 3rd party APIs used in the codebase.
+ * @tags telemetry
+ * @kind problem
+ * @id cs/telemetry/fetch-external-apis
+ */
 
 import csharp
 import ExternalApi
 
-private Call aUsage(ExternalApi api) {
- result.getTarget().getUnboundDeclaration() = api
-}
+private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
 
 private boolean isSupported(ExternalApi api) {
- api.isSupported() and result = true
- or
- not api.isSupported() and
- result = false
+  api.isSupported() and result = true
+  or
+  not api.isSupported() and
+  result = false
 }
 
 from ExternalApi api, string apiName, boolean supported, Call usage
 where
- apiName = api.getApiName() and
- supported = isSupported(api) and
- usage = aUsage(api)
-select apiName, supported, usage
-`,
-  usagesQuery: `/**
-* @name Usage of APIs coming from external libraries
-* @description A list of 3rd party APIs used in the codebase.
-* @kind problem
-* @id cs/telemetry/fetch-external-api-usages
-*/
-
-import csharp
-import ExternalApi
-
-private Call aUsage(ExternalApi api) {
- result.getTarget().getUnboundDeclaration() = api
-}
-
-from ExternalApi api, string apiName, Call usage
-where
- apiName = api.getApiName() and
- usage = aUsage(api)
-select usage, apiName
+  apiName = api.getApiName() and
+  supported = isSupported(api) and
+  usage = aUsage(api)
+select usage, apiName, supported.toString(), "supported"
 `,
   dependencies: {
     "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
index 4664dfaa8..0d4010721 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/java.ts
@@ -5,6 +5,7 @@ export const fetchExternalApisQuery: Query = {
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
  * @tags telemetry
+ * @kind problem
  * @id java/telemetry/fetch-external-apis
  */
 
@@ -27,28 +28,7 @@ where
   apiName = api.getApiName() and
   supported = isSupported(api) and
   usage = aUsage(api)
-select apiName, supported, usage
-`,
-  usagesQuery: `/**
- * @name Usage of APIs coming from external libraries
- * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
- * @kind problem
- * @id java/telemetry/fetch-external-api-usages
- */
-
-import java
-import ExternalApi
-
-private Call aUsage(ExternalApi api) {
-  result.getCallee().getSourceDeclaration() = api and
-  not result.getFile() instanceof GeneratedFile
-}
-
-from ExternalApi api, string apiName, Call usage
-where
-  apiName = api.getApiName() and
-  usage = aUsage(api)
-select usage, apiName
+select usage, apiName, supported.toString(), "supported"
 `,
   dependencies: {
     "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
diff --git a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
index 7c46bec49..b09b15e14 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/queries/query.ts
@@ -1,6 +1,14 @@
 export type Query = {
+  /**
+   * The main query.
+   *
+   * It should select all usages of external APIs, and return the following result pattern:
+   * - usage: the usage of the external API. This is an entity.
+   * - apiName: the name of the external API. This is a string.
+   * - supported: whether the external API is supported by the extension. This should be a string representation of a boolean to satify the result pattern for a problem query.
+   * - "supported": a string literal. This is required to make the query a valid problem query.
+   */
   mainQuery: string;
-  usagesQuery: string;
   dependencies?: {
     [filename: string]: string;
   };
diff --git a/extensions/ql-vscode/src/pure/bqrs-cli-types.ts b/extensions/ql-vscode/src/pure/bqrs-cli-types.ts
index 1a8be81be..ae8c35092 100644
--- a/extensions/ql-vscode/src/pure/bqrs-cli-types.ts
+++ b/extensions/ql-vscode/src/pure/bqrs-cli-types.ts
@@ -115,7 +115,7 @@ export type BqrsKind =
   | "Entity";
 
 interface BqrsColumn {
-  name: string;
+  name?: string;
   kind: BqrsKind;
 }
 export interface DecodedBqrsChunk {
diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts
index 99317ceda..111884de9 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts
@@ -4,126 +4,13 @@ import { DecodedBqrsChunk } from "../../../src/pure/bqrs-cli-types";
 describe("decodeBqrsToExternalApiUsages", () => {
   const chunk: DecodedBqrsChunk = {
     columns: [
-      { name: "apiName", kind: "String" },
-      { name: "supported", kind: "Boolean" },
       { name: "usage", kind: "Entity" },
+      { name: "apiName", kind: "String" },
+      { kind: "String" },
+      { kind: "String" },
     ],
     tuples: [
       [
-        "java.io.PrintStream#println(String)",
-        true,
-        {
-          label: "println(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 29,
-            startColumn: 9,
-            endLine: 29,
-            endColumn: 49,
-          },
-        },
-      ],
-      [
-        "org.springframework.boot.SpringApplication#run(Class,String[])",
-        false,
-        {
-          label: "run(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java",
-            startLine: 9,
-            startColumn: 9,
-            endLine: 9,
-            endColumn: 66,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Connection#createQuery(String)",
-        true,
-        {
-          label: "createQuery(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 15,
-            startColumn: 13,
-            endLine: 15,
-            endColumn: 56,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Connection#createQuery(String)",
-        true,
-        {
-          label: "createQuery(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 26,
-            startColumn: 13,
-            endLine: 26,
-            endColumn: 39,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Query#executeScalar(Class)",
-        true,
-        {
-          label: "executeScalar(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 15,
-            startColumn: 13,
-            endLine: 15,
-            endColumn: 85,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Query#executeScalar(Class)",
-        true,
-        {
-          label: "executeScalar(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 26,
-            startColumn: 13,
-            endLine: 26,
-            endColumn: 68,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Sql2o#open()",
-        true,
-        {
-          label: "open(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 14,
-            startColumn: 24,
-            endLine: 14,
-            endColumn: 35,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Sql2o#open()",
-        true,
-        {
-          label: "open(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 25,
-            startColumn: 24,
-            endLine: 25,
-            endColumn: 35,
-          },
-        },
-      ],
-      [
-        "org.sql2o.Sql2o#Sql2o(String,String,String)",
-        true,
         {
           label: "new Sql2o(...)",
           url: {
@@ -134,10 +21,56 @@ describe("decodeBqrsToExternalApiUsages", () => {
             endColumn: 88,
           },
         },
+        "org.sql2o.Sql2o#Sql2o(String,String,String)",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "open(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 14,
+            startColumn: 24,
+            endLine: 14,
+            endColumn: 35,
+          },
+        },
+        "org.sql2o.Sql2o#open()",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "executeScalar(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 15,
+            startColumn: 13,
+            endLine: 15,
+            endColumn: 85,
+          },
+        },
+        "org.sql2o.Query#executeScalar(Class)",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "createQuery(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 15,
+            startColumn: 13,
+            endLine: 15,
+            endColumn: 56,
+          },
+        },
+        "org.sql2o.Connection#createQuery(String)",
+        "false",
+        "supported",
       ],
       [
-        "org.sql2o.Sql2o#Sql2o(String)",
-        true,
         {
           label: "new Sql2o(...)",
           url: {
@@ -148,6 +81,84 @@ describe("decodeBqrsToExternalApiUsages", () => {
             endColumn: 36,
           },
         },
+        "org.sql2o.Sql2o#Sql2o(String)",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "open(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 25,
+            startColumn: 24,
+            endLine: 25,
+            endColumn: 35,
+          },
+        },
+        "org.sql2o.Sql2o#open()",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "executeScalar(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 26,
+            startColumn: 13,
+            endLine: 26,
+            endColumn: 68,
+          },
+        },
+        "org.sql2o.Query#executeScalar(Class)",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "createQuery(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 26,
+            startColumn: 13,
+            endLine: 26,
+            endColumn: 39,
+          },
+        },
+        "org.sql2o.Connection#createQuery(String)",
+        "false",
+        "supported",
+      ],
+      [
+        {
+          label: "println(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 29,
+            startColumn: 9,
+            endLine: 29,
+            endColumn: 49,
+          },
+        },
+        "java.io.PrintStream#println(String)",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "run(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/Sql2oExampleApplication.java",
+            startLine: 9,
+            startColumn: 9,
+            endLine: 9,
+            endColumn: 66,
+          },
+        },
+        "org.springframework.boot.SpringApplication#run(Class,String[])",
+        "false",
+        "supported",
       ],
     ],
   };
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
index 24a4f6e48..7a574b575 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/external-api-usage-query.test.ts
@@ -10,7 +10,7 @@ import { QueryResultType } from "../../../../src/pure/new-messages";
 import { readdir, readFile } from "fs-extra";
 import { load } from "js-yaml";
 import { dirname, join } from "path";
-import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries/index";
+import { fetchExternalApiQueries } from "../../../../src/data-extensions-editor/queries";
 import * as helpers from "../../../../src/helpers";
 import { RedactableError } from "../../../../src/pure/errors";
 
@@ -66,7 +66,7 @@ describe("runQuery", () => {
           onCancellationRequested: jest.fn(),
         },
       };
-      const result = await runQuery("mainQuery", options);
+      const result = await runQuery(options);
 
       expect(result?.resultType).toEqual(QueryResultType.SUCCESS);
 
@@ -161,18 +161,20 @@ describe("readQueryResults", () => {
           name: "#select",
           rows: 10,
           columns: [
-            { name: "apiName", kind: "s" },
-            { name: "supported", kind: "b" },
             { name: "usage", kind: "e" },
+            { name: "apiName", kind: "s" },
+            { kind: "s" },
+            { kind: "s" },
           ],
         },
         {
           name: "#select2",
           rows: 10,
           columns: [
-            { name: "apiName", kind: "s" },
-            { name: "supported", kind: "b" },
             { name: "usage", kind: "e" },
+            { name: "apiName", kind: "s" },
+            { kind: "s" },
+            { kind: "s" },
           ],
         },
       ],
@@ -191,9 +193,10 @@ describe("readQueryResults", () => {
           name: "#select",
           rows: 10,
           columns: [
-            { name: "apiName", kind: "s" },
-            { name: "supported", kind: "b" },
             { name: "usage", kind: "e" },
+            { name: "apiName", kind: "s" },
+            { kind: "s" },
+            { kind: "s" },
           ],
         },
       ],
@@ -201,9 +204,10 @@ describe("readQueryResults", () => {
     });
     const decodedResultSet = {
       columns: [
-        { name: "apiName", kind: "String" },
-        { name: "supported", kind: "Boolean" },
-        { name: "usage", kind: "Entity" },
+        { name: "usage", kind: "e" },
+        { name: "apiName", kind: "s" },
+        { kind: "s" },
+        { kind: "s" },
       ],
       tuples: [
         [

From b7a7329aff1abb7e877bb3fb086befca660eb053 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 11:32:45 +0100
Subject: [PATCH 100/119] Move selection-commands.ts to be in the vscode
 directory

---
 .../ql-vscode/src/common/{ => vscode}/selection-commands.ts   | 4 ++--
 extensions/ql-vscode/src/databases/local-databases-ui.ts      | 2 +-
 extensions/ql-vscode/src/local-queries/local-queries.ts       | 2 +-
 .../ql-vscode/src/query-history/query-history-manager.ts      | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)
 rename extensions/ql-vscode/src/common/{ => vscode}/selection-commands.ts (95%)

diff --git a/extensions/ql-vscode/src/common/selection-commands.ts b/extensions/ql-vscode/src/common/vscode/selection-commands.ts
similarity index 95%
rename from extensions/ql-vscode/src/common/selection-commands.ts
rename to extensions/ql-vscode/src/common/vscode/selection-commands.ts
index 186133969..0d930b8e1 100644
--- a/extensions/ql-vscode/src/common/selection-commands.ts
+++ b/extensions/ql-vscode/src/common/vscode/selection-commands.ts
@@ -1,9 +1,9 @@
-import { showAndLogErrorMessage } from "../helpers";
+import { showAndLogErrorMessage } from "../../helpers";
 import {
   ExplorerSelectionCommandFunction,
   TreeViewContextMultiSelectionCommandFunction,
   TreeViewContextSingleSelectionCommandFunction,
-} from "./commands";
+} from "../commands";
 
 // A hack to match types that are not an array, which is useful to help avoid
 // misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
diff --git a/extensions/ql-vscode/src/databases/local-databases-ui.ts b/extensions/ql-vscode/src/databases/local-databases-ui.ts
index 0839b2afc..5fbc23c56 100644
--- a/extensions/ql-vscode/src/databases/local-databases-ui.ts
+++ b/extensions/ql-vscode/src/databases/local-databases-ui.ts
@@ -49,7 +49,7 @@ import { LocalDatabasesCommands } from "../common/commands";
 import {
   createMultiSelectionCommand,
   createSingleSelectionCommand,
-} from "../common/selection-commands";
+} from "../common/vscode/selection-commands";
 
 enum SortOrder {
   NameAsc = "NameAsc",
diff --git a/extensions/ql-vscode/src/local-queries/local-queries.ts b/extensions/ql-vscode/src/local-queries/local-queries.ts
index a5b7f3028..fa69f256f 100644
--- a/extensions/ql-vscode/src/local-queries/local-queries.ts
+++ b/extensions/ql-vscode/src/local-queries/local-queries.ts
@@ -47,7 +47,7 @@ import { App } from "../common/app";
 import { DisposableObject } from "../pure/disposable-object";
 import { SkeletonQueryWizard } from "../skeleton-query-wizard";
 import { LocalQueryRun } from "./local-query-run";
-import { createMultiSelectionCommand } from "../common/selection-commands";
+import { createMultiSelectionCommand } from "../common/vscode/selection-commands";
 
 interface DatabaseQuickPickItem extends QuickPickItem {
   databaseItem: DatabaseItem;
diff --git a/extensions/ql-vscode/src/query-history/query-history-manager.ts b/extensions/ql-vscode/src/query-history/query-history-manager.ts
index ac24cd0bf..2b42c3ddf 100644
--- a/extensions/ql-vscode/src/query-history/query-history-manager.ts
+++ b/extensions/ql-vscode/src/query-history/query-history-manager.ts
@@ -59,7 +59,7 @@ import { tryOpenExternalFile } from "../common/vscode/external-files";
 import {
   createMultiSelectionCommand,
   createSingleSelectionCommand,
-} from "../common/selection-commands";
+} from "../common/vscode/selection-commands";
 
 /**
  * query-history-manager.ts

From a4d875af8d399a27d2adccc3030c8f3e0a263f48 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 31 May 2023 12:33:51 +0200
Subject: [PATCH 101/119] Fix typo

---
 .../src/data-extensions-editor/auto-model-usages-query.ts       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
index 3b37def3e..35a77ed62 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -57,7 +57,7 @@ export async function getAutoModelUsages({
   progress({
     maxStep,
     step: 1100,
-    message: "Retrieving source locatin prefix",
+    message: "Retrieving source location prefix",
   });
 
   const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(

From 56d6f19365e06a71990f706c485456b7b9a44908 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 11:57:26 +0100
Subject: [PATCH 102/119] Move CachedOperation to a new file

---
 extensions/ql-vscode/src/helpers.ts           | 71 -------------------
 .../contextual/template-provider.ts           |  2 +-
 .../ql-vscode/src/pure/cached-operation.ts    | 70 ++++++++++++++++++
 3 files changed, 71 insertions(+), 72 deletions(-)
 create mode 100644 extensions/ql-vscode/src/pure/cached-operation.ts

diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts
index d4e8a9a8c..444338cc4 100644
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@@ -584,77 +584,6 @@ export async function getPrimaryDbscheme(
   return dbscheme;
 }
 
-/**
- * A cached mapping from strings to value of type U.
- */
-export class CachedOperation<U> {
-  private readonly operation: (t: string, ...args: any[]) => Promise<U>;
-  private readonly cached: Map<string, U>;
-  private readonly lru: string[];
-  private readonly inProgressCallbacks: Map<
-    string,
-    Array<[(u: U) => void, (reason?: any) => void]>
-  >;
-
-  constructor(
-    operation: (t: string, ...args: any[]) => Promise<U>,
-    private cacheSize = 100,
-  ) {
-    this.operation = operation;
-    this.lru = [];
-    this.inProgressCallbacks = new Map<
-      string,
-      Array<[(u: U) => void, (reason?: any) => void]>
-    >();
-    this.cached = new Map<string, U>();
-  }
-
-  async get(t: string, ...args: any[]): Promise<U> {
-    // Try and retrieve from the cache
-    const fromCache = this.cached.get(t);
-    if (fromCache !== undefined) {
-      // Move to end of lru list
-      this.lru.push(
-        this.lru.splice(
-          this.lru.findIndex((v) => v === t),
-          1,
-        )[0],
-      );
-      return fromCache;
-    }
-    // Otherwise check if in progress
-    const inProgressCallback = this.inProgressCallbacks.get(t);
-    if (inProgressCallback !== undefined) {
-      // If so wait for it to resolve
-      return await new Promise((resolve, reject) => {
-        inProgressCallback.push([resolve, reject]);
-      });
-    }
-
-    // Otherwise compute the new value, but leave a callback to allow sharing work
-    const callbacks: Array<[(u: U) => void, (reason?: any) => void]> = [];
-    this.inProgressCallbacks.set(t, callbacks);
-    try {
-      const result = await this.operation(t, ...args);
-      callbacks.forEach((f) => f[0](result));
-      this.inProgressCallbacks.delete(t);
-      if (this.lru.length > this.cacheSize) {
-        const toRemove = this.lru.shift()!;
-        this.cached.delete(toRemove);
-      }
-      this.lru.push(t);
-      this.cached.set(t, result);
-      return result;
-    } catch (e) {
-      // Rethrow error on all callbacks
-      callbacks.forEach((f) => f[1](e));
-      throw e;
-    } finally {
-      this.inProgressCallbacks.delete(t);
-    }
-  }
-}
-
 /**
  * The following functions al heuristically determine metadata about databases.
  */
diff --git a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
index 26ba9d72f..8c34c29ab 100644
--- a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
+++ b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
@@ -17,7 +17,7 @@ import {
 } from "../../common/vscode/archive-filesystem-provider";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { DatabaseManager } from "../../databases/local-databases";
-import { CachedOperation } from "../../helpers";
+import { CachedOperation } from "../../pure/cached-operation";
 import { ProgressCallback, withProgress } from "../../common/vscode/progress";
 import { KeyType } from "./key-type";
 import {
diff --git a/extensions/ql-vscode/src/pure/cached-operation.ts b/extensions/ql-vscode/src/pure/cached-operation.ts
new file mode 100644
index 000000000..c51970de1
--- /dev/null
+++ b/extensions/ql-vscode/src/pure/cached-operation.ts
@@ -0,0 +1,70 @@
+/**
+ * A cached mapping from strings to value of type U.
+ */
+export class CachedOperation<U> {
+  private readonly operation: (t: string, ...args: any[]) => Promise<U>;
+  private readonly cached: Map<string, U>;
+  private readonly lru: string[];
+  private readonly inProgressCallbacks: Map<
+    string,
+    Array<[(u: U) => void, (reason?: any) => void]>
+  >;
+
+  constructor(
+    operation: (t: string, ...args: any[]) => Promise<U>,
+    private cacheSize = 100,
+  ) {
+    this.operation = operation;
+    this.lru = [];
+    this.inProgressCallbacks = new Map<
+      string,
+      Array<[(u: U) => void, (reason?: any) => void]>
+    >();
+    this.cached = new Map<string, U>();
+  }
+
+  async get(t: string, ...args: any[]): Promise<U> {
+    // Try and retrieve from the cache
+    const fromCache = this.cached.get(t);
+    if (fromCache !== undefined) {
+      // Move to end of lru list
+      this.lru.push(
+        this.lru.splice(
+          this.lru.findIndex((v) => v === t),
+          1,
+        )[0],
+      );
+      return fromCache;
+    }
+    // Otherwise check if in progress
+    const inProgressCallback = this.inProgressCallbacks.get(t);
+    if (inProgressCallback !== undefined) {
+      // If so wait for it to resolve
+      return await new Promise((resolve, reject) => {
+        inProgressCallback.push([resolve, reject]);
+      });
+    }
+
+    // Otherwise compute the new value, but leave a callback to allow sharing work
+    const callbacks: Array<[(u: U) => void, (reason?: any) => void]> = [];
+    this.inProgressCallbacks.set(t, callbacks);
+    try {
+      const result = await this.operation(t, ...args);
+      callbacks.forEach((f) => f[0](result));
+      this.inProgressCallbacks.delete(t);
+      if (this.lru.length > this.cacheSize) {
+        const toRemove = this.lru.shift()!;
+        this.cached.delete(toRemove);
+      }
+      this.lru.push(t);
+      this.cached.set(t, result);
+      return result;
+    } catch (e) {
+      // Rethrow error on all callbacks
+      callbacks.forEach((f) => f[1](e));
+      throw e;
+    } finally {
+      this.inProgressCallbacks.delete(t);
+    }
+  }
+}

From 42436e623b4686c60c8b0d3a6f83e1052321dae4 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 31 May 2023 11:30:00 +0000
Subject: [PATCH 103/119] Add JSDoc to db config store

---
 .../ql-vscode/src/databases/config/db-config-store.ts     | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/extensions/ql-vscode/src/databases/config/db-config-store.ts b/extensions/ql-vscode/src/databases/config/db-config-store.ts
index 49e5c5288..bc1f5771b 100644
--- a/extensions/ql-vscode/src/databases/config/db-config-store.ts
+++ b/extensions/ql-vscode/src/databases/config/db-config-store.ts
@@ -145,6 +145,10 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
+  /**
+   * Adds a list of remote repositories to an existing repository list and removes duplicates.
+   * @returns a list of repositories that were not added because the list reached 1000 entries.
+   */
   public async addRemoteReposToList(
     repoNwoList: string[],
     parentList: string,
@@ -173,6 +177,10 @@ export class DbConfigStore extends DisposableObject {
     return truncatedRepositories;
   }
 
+  /**
+   * Adds one remote repository
+   * @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added.
+   */
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,

From 0739c46fedeb5c96e8dafcffdbed48c7d61ded27 Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Wed, 31 May 2023 11:30:31 +0000
Subject: [PATCH 104/119] Add comments to gh api client and fix total number of
 returned repos

---
 .../src/variant-analysis/gh-api/gh-api-client.ts          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
index 2746de16c..0e8d68d1f 100644
--- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
+++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts
@@ -29,9 +29,13 @@ export async function getCodeSearchRepositories(
     },
   )) {
     nwos.push(...response.data.map((item) => item.full_name));
-    const numberOfRequests = Math.ceil(response.data.total_count / 99);
-    const increment = numberOfRequests < 10 ? 80 / numberOfRequests : 8;
+    // calculate progress bar: 80% of the progress bar is used for the code search
+    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
+    // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10
+    const increment =
+      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
     progress.report({ increment });
+
     if (token.isCancellationRequested) {
       nwos = [];
       break;

From 7c47cf0968e1d9abde6e3a258975017de853358d Mon Sep 17 00:00:00 2001
From: Arthur Baars <aibaars@github.com>
Date: Wed, 31 May 2023 14:30:08 +0200
Subject: [PATCH 105/119] View CFG: export selected line and column position

Fixes: #2456
---
 .../ast-viewer/ast-cfg-commands.ts            | 11 ++++++--
 .../contextual/location-finder.ts             |  6 ++--
 .../contextual/template-provider.ts           | 28 ++++++++++++-------
 3 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts
index 3262fbc8f..76e6eed06 100644
--- a/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts
+++ b/extensions/ql-vscode/src/language-support/ast-viewer/ast-cfg-commands.ts
@@ -42,9 +42,14 @@ export function getAstCfgCommands({
   const viewCfg = async () =>
     withProgress(
       async (progress, token) => {
-        const res = await cfgTemplateProvider.provideCfgUri(
-          window.activeTextEditor?.document,
-        );
+        const editor = window.activeTextEditor;
+        const res = !editor
+          ? undefined
+          : await cfgTemplateProvider.provideCfgUri(
+              editor.document,
+              editor.selection.active.line + 1,
+              editor.selection.active.character + 1,
+            );
         if (res) {
           await localQueries.compileAndRunQuery(
             QuickEvalType.None,
diff --git a/extensions/ql-vscode/src/language-support/contextual/location-finder.ts b/extensions/ql-vscode/src/language-support/contextual/location-finder.ts
index ef998a10a..980327475 100644
--- a/extensions/ql-vscode/src/language-support/contextual/location-finder.ts
+++ b/extensions/ql-vscode/src/language-support/contextual/location-finder.ts
@@ -24,7 +24,9 @@ import { QueryResultType } from "../../pure/new-messages";
 import { fileRangeFromURI } from "./file-range-from-uri";
 
 export const SELECT_QUERY_NAME = "#select";
-export const TEMPLATE_NAME = "selectedSourceFile";
+export const SELECTED_SOURCE_FILE = "selectedSourceFile";
+export const SELECTED_SOURCE_LINE = "selectedSourceLine";
+export const SELECTED_SOURCE_COLUMN = "selectedSourceColumn";
 
 export interface FullLocationLink extends LocationLink {
   originUri: Uri;
@@ -124,7 +126,7 @@ async function getLinksFromResults(
 
 function createTemplates(path: string): Record<string, string> {
   return {
-    [TEMPLATE_NAME]: path,
+    [SELECTED_SOURCE_FILE]: path,
   };
 }
 
diff --git a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
index 26ba9d72f..177510855 100644
--- a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
+++ b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
@@ -23,7 +23,9 @@ import { KeyType } from "./key-type";
 import {
   FullLocationLink,
   getLocationsForUriString,
-  TEMPLATE_NAME,
+  SELECTED_SOURCE_FILE,
+  SELECTED_SOURCE_LINE,
+  SELECTED_SOURCE_COLUMN,
 } from "./location-finder";
 import {
   qlpackOfDatabase,
@@ -253,7 +255,7 @@ export class TemplatePrintAstProvider {
 
     const query = queries[0];
     const templates: Record<string, string> = {
-      [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive,
+      [SELECTED_SOURCE_FILE]: zippedArchive.pathWithinSourceArchive,
     };
 
     const results = await runContextualQuery(
@@ -284,15 +286,17 @@ export class TemplatePrintCfgProvider {
   }
 
   async provideCfgUri(
-    document?: TextDocument,
+    document: TextDocument,
+    line: number,
+    character: number,
   ): Promise<[Uri, Record<string, string>] | undefined> {
-    if (!document) {
-      return;
-    }
-
     return this.shouldUseCache()
-      ? await this.cache.get(document.uri.toString())
-      : await this.getCfgUri(document.uri.toString());
+      ? await this.cache.get(
+          `${document.uri.toString()}#${line}:${character}`,
+          line,
+          character,
+        )
+      : await this.getCfgUri(document.uri.toString(), line, character);
   }
 
   private shouldUseCache() {
@@ -301,6 +305,8 @@ export class TemplatePrintCfgProvider {
 
   private async getCfgUri(
     uriString: string,
+    line: number,
+    character: number,
   ): Promise<[Uri, Record<string, string>]> {
     const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
@@ -342,7 +348,9 @@ export class TemplatePrintCfgProvider {
     const queryUri = Uri.file(queries[0]);
 
     const templates: Record<string, string> = {
-      [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive,
+      [SELECTED_SOURCE_FILE]: zippedArchive.pathWithinSourceArchive,
+      [SELECTED_SOURCE_LINE]: line.toString(),
+      [SELECTED_SOURCE_COLUMN]: character.toString(),
     };
 
     return [queryUri, templates];

From b9c0f2bc14532150c9ced829c5a9ec1a2c08b2db Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 12:11:55 +0100
Subject: [PATCH 106/119] Move InvocationRateLimiter to a separate file

---
 .../ql-vscode/src/codeql-cli/distribution.ts  |  11 +-
 extensions/ql-vscode/src/helpers.ts           | 109 +-------
 .../ql-vscode/src/invocation-rate-limiter.ts  |  91 +++++++
 .../vscode-tests/no-workspace/helpers.test.ts | 238 +-----------------
 .../invocation-rate-limiter.test.ts           | 234 +++++++++++++++++
 5 files changed, 332 insertions(+), 351 deletions(-)
 create mode 100644 extensions/ql-vscode/src/invocation-rate-limiter.ts
 create mode 100644 extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts

diff --git a/extensions/ql-vscode/src/codeql-cli/distribution.ts b/extensions/ql-vscode/src/codeql-cli/distribution.ts
index b64e73272..1fda5096a 100644
--- a/extensions/ql-vscode/src/codeql-cli/distribution.ts
+++ b/extensions/ql-vscode/src/codeql-cli/distribution.ts
@@ -6,12 +6,7 @@ import * as semver from "semver";
 import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "../config";
-import {
-  InvocationRateLimiter,
-  InvocationRateLimiterResultKind,
-  showAndLogErrorMessage,
-  showAndLogWarningMessage,
-} from "../helpers";
+import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers";
 import { extLogger } from "../common";
 import { getCodeQlCliVersion } from "./cli-version";
 import {
@@ -24,6 +19,10 @@ import {
   extractZipArchive,
   getRequiredAssetName,
 } from "../pure/distribution";
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+} from "../invocation-rate-limiter";
 
 /**
  * distribution.ts
diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts
index d4e8a9a8c..dcba07cb6 100644
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@@ -10,14 +10,7 @@ import { glob } from "glob";
 import { load } from "js-yaml";
 import { join, basename, dirname } from "path";
 import { dirSync } from "tmp-promise";
-import {
-  ExtensionContext,
-  Uri,
-  window as Window,
-  workspace,
-  env,
-  WorkspaceFolder,
-} from "vscode";
+import { Uri, window as Window, workspace, env, WorkspaceFolder } from "vscode";
 import { CodeQLCliServer, QlpacksInfo } from "./codeql-cli/cli";
 import { UserCancellationException } from "./common/vscode/progress";
 import { extLogger, OutputChannelLogger } from "./common";
@@ -363,106 +356,6 @@ export async function prepareCodeTour(
   }
 }
 
-/**
- * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
- * the last invocation of that function.
- */
-export class InvocationRateLimiter<T> {
-  constructor(
-    extensionContext: ExtensionContext,
-    funcIdentifier: string,
-    func: () => Promise<T>,
-    createDate: (dateString?: string) => Date = (s) =>
-      s ? new Date(s) : new Date(),
-  ) {
-    this._createDate = createDate;
-    this._extensionContext = extensionContext;
-    this._func = func;
-    this._funcIdentifier = funcIdentifier;
-  }
-
-  /**
-   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
-   */
-  public async invokeFunctionIfIntervalElapsed(
-    minSecondsSinceLastInvocation: number,
-  ): Promise<InvocationRateLimiterResult<T>> {
-    const updateCheckStartDate = this._createDate();
-    const lastInvocationDate = this.getLastInvocationDate();
-    if (
-      minSecondsSinceLastInvocation &&
-      lastInvocationDate &&
-      lastInvocationDate <= updateCheckStartDate &&
-      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
-        updateCheckStartDate.getTime()
-    ) {
-      return createRateLimitedResult();
-    }
-    const result = await this._func();
-    await this.setLastInvocationDate(updateCheckStartDate);
-    return createInvokedResult(result);
-  }
-
-  private getLastInvocationDate(): Date | undefined {
-    const maybeDateString: string | undefined =
-      this._extensionContext.globalState.get(
-        InvocationRateLimiter._invocationRateLimiterPrefix +
-          this._funcIdentifier,
-      );
-    return maybeDateString ? this._createDate(maybeDateString) : undefined;
-  }
-
-  private async setLastInvocationDate(date: Date): Promise<void> {
-    return await this._extensionContext.globalState.update(
-      InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier,
-      date,
-    );
-  }
-
-  private readonly _createDate: (dateString?: string) => Date;
-  private readonly _extensionContext: ExtensionContext;
-  private readonly _func: () => Promise<T>;
-  private readonly _funcIdentifier: string;
-
-  private static readonly _invocationRateLimiterPrefix =
-    "invocationRateLimiter_lastInvocationDate_";
-}
-
-export enum InvocationRateLimiterResultKind {
-  Invoked,
-  RateLimited,
-}
-
-/**
- * The function was invoked and returned the value `result`.
- */
-interface InvokedResult<T> {
-  kind: InvocationRateLimiterResultKind.Invoked;
-  result: T;
-}
-
-/**
- * The function was not invoked as the minimum interval since the last invocation had not elapsed.
- */
-interface RateLimitedResult {
-  kind: InvocationRateLimiterResultKind.RateLimited;
-}
-
-type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
-
-function createInvokedResult<T>(result: T): InvokedResult<T> {
-  return {
-    kind: InvocationRateLimiterResultKind.Invoked,
-    result,
-  };
-}
-
-function createRateLimitedResult(): RateLimitedResult {
-  return {
-    kind: InvocationRateLimiterResultKind.RateLimited,
-  };
-}
-
 export interface QlPacksForLanguage {
   /** The name of the pack containing the dbscheme. */
   dbschemePack: string;
diff --git a/extensions/ql-vscode/src/invocation-rate-limiter.ts b/extensions/ql-vscode/src/invocation-rate-limiter.ts
new file mode 100644
index 000000000..ba09d1a66
--- /dev/null
+++ b/extensions/ql-vscode/src/invocation-rate-limiter.ts
@@ -0,0 +1,91 @@
+import { ExtensionContext } from "vscode";
+
+/**
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
+ */
+export class InvocationRateLimiter<T> {
+  constructor(
+    private readonly extensionContext: ExtensionContext,
+    private readonly funcIdentifier: string,
+    private readonly func: () => Promise<T>,
+    private readonly createDate: (dateString?: string) => Date = (s) =>
+      s ? new Date(s) : new Date(),
+  ) {}
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(
+    minSecondsSinceLastInvocation: number,
+  ): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this.createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
+        updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this.func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined =
+      this.extensionContext.globalState.get(
+        InvocationRateLimiter._invocationRateLimiterPrefix +
+          this.funcIdentifier,
+      );
+    return maybeDateString ? this.createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this.extensionContext.globalState.update(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+      date,
+    );
+  }
+
+  private static readonly _invocationRateLimiterPrefix =
+    "invocationRateLimiter_lastInvocationDate_";
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited,
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result,
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited,
+  };
+}
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts
index 21be77229..d6afc9cf6 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/helpers.test.ts
@@ -1,17 +1,4 @@
-import {
-  EnvironmentVariableCollection,
-  EnvironmentVariableMutator,
-  Event,
-  ExtensionContext,
-  ExtensionMode,
-  Memento,
-  SecretStorage,
-  SecretStorageChangeEvent,
-  Uri,
-  window,
-  workspace,
-  WorkspaceFolder,
-} from "vscode";
+import { Uri, window, workspace, WorkspaceFolder } from "vscode";
 import { dump } from "js-yaml";
 import * as tmp from "tmp";
 import { join } from "path";
@@ -28,7 +15,6 @@ import { DirResult } from "tmp";
 import {
   getFirstWorkspaceFolder,
   getInitialQueryContents,
-  InvocationRateLimiter,
   isFolderAlreadyInWorkspace,
   isLikelyDatabaseRoot,
   isLikelyDbLanguageFolder,
@@ -45,118 +31,6 @@ import { Setting } from "../../../src/config";
 import { createMockCommandManager } from "../../__mocks__/commandsMock";
 
 describe("helpers", () => {
-  describe("Invocation rate limiter", () => {
-    // 1 January 2020
-    let currentUnixTime = 1577836800;
-
-    function createDate(dateString?: string): Date {
-      if (dateString) {
-        return new Date(dateString);
-      }
-      const numMillisecondsPerSecond = 1000;
-      return new Date(currentUnixTime * numMillisecondsPerSecond);
-    }
-
-    function createInvocationRateLimiter<T>(
-      funcIdentifier: string,
-      func: () => Promise<T>,
-    ): InvocationRateLimiter<T> {
-      return new InvocationRateLimiter(
-        new MockExtensionContext(),
-        funcIdentifier,
-        func,
-        (s) => createDate(s),
-      );
-    }
-
-    it("initially invokes function", async () => {
-      let numTimesFuncCalled = 0;
-      const invocationRateLimiter = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncCalled++;
-        },
-      );
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-      expect(numTimesFuncCalled).toBe(1);
-    });
-
-    it("doesn't invoke function again if no time has passed", async () => {
-      let numTimesFuncCalled = 0;
-      const invocationRateLimiter = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncCalled++;
-        },
-      );
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-      expect(numTimesFuncCalled).toBe(1);
-    });
-
-    it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => {
-      let numTimesFuncCalled = 0;
-      const invocationRateLimiter = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncCalled++;
-        },
-      );
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-      currentUnixTime += 1;
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
-      expect(numTimesFuncCalled).toBe(1);
-    });
-
-    it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => {
-      let numTimesFuncCalled = 0;
-      const invocationRateLimiter = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncCalled++;
-        },
-      );
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
-      expect(numTimesFuncCalled).toBe(2);
-    });
-
-    it("invokes function again after requested time since last invocation has elapsed", async () => {
-      let numTimesFuncCalled = 0;
-      const invocationRateLimiter = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncCalled++;
-        },
-      );
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
-      currentUnixTime += 1;
-      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
-      expect(numTimesFuncCalled).toBe(2);
-    });
-
-    it("invokes functions with different rate limiters", async () => {
-      let numTimesFuncACalled = 0;
-      const invocationRateLimiterA = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncACalled++;
-        },
-      );
-      let numTimesFuncBCalled = 0;
-      const invocationRateLimiterB = createInvocationRateLimiter(
-        "funcid",
-        async () => {
-          numTimesFuncBCalled++;
-        },
-      );
-      await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
-      await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
-      expect(numTimesFuncACalled).toBe(1);
-      expect(numTimesFuncBCalled).toBe(1);
-    });
-  });
-
   describe("codeql-database.yml tests", () => {
     let dir: tmp.DirResult;
     let language: QueryLanguage;
@@ -250,116 +124,6 @@ describe("helpers", () => {
     });
   });
 
-  class MockExtensionContext implements ExtensionContext {
-    extensionMode: ExtensionMode = 3;
-    subscriptions: Array<{ dispose(): unknown }> = [];
-    workspaceState: Memento = new MockMemento();
-    globalState = new MockGlobalStorage();
-    extensionPath = "";
-    asAbsolutePath(_relativePath: string): string {
-      throw new Error("Method not implemented.");
-    }
-    storagePath = "";
-    globalStoragePath = "";
-    logPath = "";
-    extensionUri = Uri.parse("");
-    environmentVariableCollection = new MockEnvironmentVariableCollection();
-    secrets = new MockSecretStorage();
-    storageUri = Uri.parse("");
-    globalStorageUri = Uri.parse("");
-    logUri = Uri.parse("");
-    extension: any;
-  }
-
-  class MockEnvironmentVariableCollection
-    implements EnvironmentVariableCollection
-  {
-    [Symbol.iterator](): Iterator<
-      [variable: string, mutator: EnvironmentVariableMutator],
-      any,
-      undefined
-    > {
-      throw new Error("Method not implemented.");
-    }
-    persistent = false;
-    replace(_variable: string, _value: string): void {
-      throw new Error("Method not implemented.");
-    }
-    append(_variable: string, _value: string): void {
-      throw new Error("Method not implemented.");
-    }
-    prepend(_variable: string, _value: string): void {
-      throw new Error("Method not implemented.");
-    }
-    get(_variable: string): EnvironmentVariableMutator | undefined {
-      throw new Error("Method not implemented.");
-    }
-    forEach(
-      _callback: (
-        variable: string,
-        mutator: EnvironmentVariableMutator,
-        collection: EnvironmentVariableCollection,
-      ) => any,
-      _thisArg?: any,
-    ): void {
-      throw new Error("Method not implemented.");
-    }
-    delete(_variable: string): void {
-      throw new Error("Method not implemented.");
-    }
-    clear(): void {
-      throw new Error("Method not implemented.");
-    }
-  }
-
-  class MockMemento implements Memento {
-    keys(): readonly string[] {
-      throw new Error("Method not implemented.");
-    }
-    map = new Map<any, any>();
-
-    /**
-     * Return a value.
-     *
-     * @param key A string.
-     * @param defaultValue A value that should be returned when there is no
-     * value (`undefined`) with the given key.
-     * @return The stored value or the defaultValue.
-     */
-    get<T>(key: string, defaultValue?: T): T {
-      return this.map.has(key) ? this.map.get(key) : defaultValue;
-    }
-
-    /**
-     * Store a value. The value must be JSON-stringifyable.
-     *
-     * @param key A string.
-     * @param value A value. MUST not contain cyclic references.
-     */
-    async update(key: string, value: any): Promise<void> {
-      this.map.set(key, value);
-    }
-  }
-
-  class MockGlobalStorage extends MockMemento {
-    public setKeysForSync(_keys: string[]): void {
-      return;
-    }
-  }
-
-  class MockSecretStorage implements SecretStorage {
-    get(_key: string): Thenable<string | undefined> {
-      throw new Error("Method not implemented.");
-    }
-    store(_key: string, _value: string): Thenable<void> {
-      throw new Error("Method not implemented.");
-    }
-    delete(_key: string): Thenable<void> {
-      throw new Error("Method not implemented.");
-    }
-    onDidChange!: Event<SecretStorageChangeEvent>;
-  }
-
   it("should report stream progress", () => {
     const progressSpy = jest.fn();
     const mockReadable = {
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
new file mode 100644
index 000000000..b8923c673
--- /dev/null
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
@@ -0,0 +1,234 @@
+import {
+  EnvironmentVariableCollection,
+  EnvironmentVariableMutator,
+  Event,
+  ExtensionContext,
+  ExtensionMode,
+  Memento,
+  SecretStorage,
+  SecretStorageChangeEvent,
+  Uri,
+} from "vscode";
+import { InvocationRateLimiter } from "../../../src/invocation-rate-limiter";
+
+describe("Invocation rate limiter", () => {
+  // 1 January 2020
+  let currentUnixTime = 1577836800;
+
+  function createDate(dateString?: string): Date {
+    if (dateString) {
+      return new Date(dateString);
+    }
+    const numMillisecondsPerSecond = 1000;
+    return new Date(currentUnixTime * numMillisecondsPerSecond);
+  }
+
+  function createInvocationRateLimiter<T>(
+    funcIdentifier: string,
+    func: () => Promise<T>,
+  ): InvocationRateLimiter<T> {
+    return new InvocationRateLimiter(
+      new MockExtensionContext(),
+      funcIdentifier,
+      func,
+      (s) => createDate(s),
+    );
+  }
+
+  class MockExtensionContext implements ExtensionContext {
+    extensionMode: ExtensionMode = 3;
+    subscriptions: Array<{ dispose(): unknown }> = [];
+    workspaceState: Memento = new MockMemento();
+    globalState = new MockGlobalStorage();
+    extensionPath = "";
+    asAbsolutePath(_relativePath: string): string {
+      throw new Error("Method not implemented.");
+    }
+    storagePath = "";
+    globalStoragePath = "";
+    logPath = "";
+    extensionUri = Uri.parse("");
+    environmentVariableCollection = new MockEnvironmentVariableCollection();
+    secrets = new MockSecretStorage();
+    storageUri = Uri.parse("");
+    globalStorageUri = Uri.parse("");
+    logUri = Uri.parse("");
+    extension: any;
+  }
+
+  class MockEnvironmentVariableCollection
+    implements EnvironmentVariableCollection
+  {
+    [Symbol.iterator](): Iterator<
+      [variable: string, mutator: EnvironmentVariableMutator],
+      any,
+      undefined
+    > {
+      throw new Error("Method not implemented.");
+    }
+    persistent = false;
+    replace(_variable: string, _value: string): void {
+      throw new Error("Method not implemented.");
+    }
+    append(_variable: string, _value: string): void {
+      throw new Error("Method not implemented.");
+    }
+    prepend(_variable: string, _value: string): void {
+      throw new Error("Method not implemented.");
+    }
+    get(_variable: string): EnvironmentVariableMutator | undefined {
+      throw new Error("Method not implemented.");
+    }
+    forEach(
+      _callback: (
+        variable: string,
+        mutator: EnvironmentVariableMutator,
+        collection: EnvironmentVariableCollection,
+      ) => any,
+      _thisArg?: any,
+    ): void {
+      throw new Error("Method not implemented.");
+    }
+    delete(_variable: string): void {
+      throw new Error("Method not implemented.");
+    }
+    clear(): void {
+      throw new Error("Method not implemented.");
+    }
+  }
+
+  class MockMemento implements Memento {
+    keys(): readonly string[] {
+      throw new Error("Method not implemented.");
+    }
+    map = new Map<any, any>();
+
+    /**
+     * Return a value.
+     *
+     * @param key A string.
+     * @param defaultValue A value that should be returned when there is no
+     * value (`undefined`) with the given key.
+     * @return The stored value or the defaultValue.
+     */
+    get<T>(key: string, defaultValue?: T): T {
+      return this.map.has(key) ? this.map.get(key) : defaultValue;
+    }
+
+    /**
+     * Store a value. The value must be JSON-stringifyable.
+     *
+     * @param key A string.
+     * @param value A value. MUST not contain cyclic references.
+     */
+    async update(key: string, value: any): Promise<void> {
+      this.map.set(key, value);
+    }
+  }
+
+  class MockGlobalStorage extends MockMemento {
+    public setKeysForSync(_keys: string[]): void {
+      return;
+    }
+  }
+
+  class MockSecretStorage implements SecretStorage {
+    get(_key: string): Thenable<string | undefined> {
+      throw new Error("Method not implemented.");
+    }
+    store(_key: string, _value: string): Thenable<void> {
+      throw new Error("Method not implemented.");
+    }
+    delete(_key: string): Thenable<void> {
+      throw new Error("Method not implemented.");
+    }
+    onDidChange!: Event<SecretStorageChangeEvent>;
+  }
+
+  it("initially invokes function", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncCalled++;
+      },
+    );
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    expect(numTimesFuncCalled).toBe(1);
+  });
+
+  it("doesn't invoke function again if no time has passed", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncCalled++;
+      },
+    );
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    expect(numTimesFuncCalled).toBe(1);
+  });
+
+  it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncCalled++;
+      },
+    );
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    currentUnixTime += 1;
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
+    expect(numTimesFuncCalled).toBe(1);
+  });
+
+  it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncCalled++;
+      },
+    );
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
+    expect(numTimesFuncCalled).toBe(2);
+  });
+
+  it("invokes function again after requested time since last invocation has elapsed", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncCalled++;
+      },
+    );
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
+    currentUnixTime += 1;
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
+    expect(numTimesFuncCalled).toBe(2);
+  });
+
+  it("invokes functions with different rate limiters", async () => {
+    let numTimesFuncACalled = 0;
+    const invocationRateLimiterA = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncACalled++;
+      },
+    );
+    let numTimesFuncBCalled = 0;
+    const invocationRateLimiterB = createInvocationRateLimiter(
+      "funcid",
+      async () => {
+        numTimesFuncBCalled++;
+      },
+    );
+    await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
+    await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
+    expect(numTimesFuncACalled).toBe(1);
+    expect(numTimesFuncBCalled).toBe(1);
+  });
+});

From 79dccaa12f7edae80a18990056adb405b466a2ef Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 12:18:20 +0100
Subject: [PATCH 107/119] Pass in a Memento instead of a full ExtensionContext

---
 .../ql-vscode/src/codeql-cli/distribution.ts  |  2 +-
 .../ql-vscode/src/invocation-rate-limiter.ts  | 14 ++-
 .../invocation-rate-limiter.test.ts           | 89 +------------------
 3 files changed, 9 insertions(+), 96 deletions(-)

diff --git a/extensions/ql-vscode/src/codeql-cli/distribution.ts b/extensions/ql-vscode/src/codeql-cli/distribution.ts
index 1fda5096a..c27608700 100644
--- a/extensions/ql-vscode/src/codeql-cli/distribution.ts
+++ b/extensions/ql-vscode/src/codeql-cli/distribution.ts
@@ -75,7 +75,7 @@ export class DistributionManager implements DistributionProvider {
         extensionContext,
       );
     this.updateCheckRateLimiter = new InvocationRateLimiter(
-      extensionContext,
+      extensionContext.globalState,
       "extensionSpecificDistributionUpdateCheck",
       () =>
         this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),
diff --git a/extensions/ql-vscode/src/invocation-rate-limiter.ts b/extensions/ql-vscode/src/invocation-rate-limiter.ts
index ba09d1a66..3788b9664 100644
--- a/extensions/ql-vscode/src/invocation-rate-limiter.ts
+++ b/extensions/ql-vscode/src/invocation-rate-limiter.ts
@@ -1,4 +1,4 @@
-import { ExtensionContext } from "vscode";
+import { Memento } from "./common/memento";
 
 /**
  * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
@@ -6,7 +6,7 @@ import { ExtensionContext } from "vscode";
  */
 export class InvocationRateLimiter<T> {
   constructor(
-    private readonly extensionContext: ExtensionContext,
+    private readonly globalState: Memento,
     private readonly funcIdentifier: string,
     private readonly func: () => Promise<T>,
     private readonly createDate: (dateString?: string) => Date = (s) =>
@@ -36,16 +36,14 @@ export class InvocationRateLimiter<T> {
   }
 
   private getLastInvocationDate(): Date | undefined {
-    const maybeDateString: string | undefined =
-      this.extensionContext.globalState.get(
-        InvocationRateLimiter._invocationRateLimiterPrefix +
-          this.funcIdentifier,
-      );
+    const maybeDateString: string | undefined = this.globalState.get(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+    );
     return maybeDateString ? this.createDate(maybeDateString) : undefined;
   }
 
   private async setLastInvocationDate(date: Date): Promise<void> {
-    return await this.extensionContext.globalState.update(
+    return await this.globalState.update(
       InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
       date,
     );
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
index b8923c673..ac6ba84ef 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
@@ -1,14 +1,4 @@
-import {
-  EnvironmentVariableCollection,
-  EnvironmentVariableMutator,
-  Event,
-  ExtensionContext,
-  ExtensionMode,
-  Memento,
-  SecretStorage,
-  SecretStorageChangeEvent,
-  Uri,
-} from "vscode";
+import { Memento } from "vscode";
 import { InvocationRateLimiter } from "../../../src/invocation-rate-limiter";
 
 describe("Invocation rate limiter", () => {
@@ -28,75 +18,13 @@ describe("Invocation rate limiter", () => {
     func: () => Promise<T>,
   ): InvocationRateLimiter<T> {
     return new InvocationRateLimiter(
-      new MockExtensionContext(),
+      new MockGlobalStorage(),
       funcIdentifier,
       func,
       (s) => createDate(s),
     );
   }
 
-  class MockExtensionContext implements ExtensionContext {
-    extensionMode: ExtensionMode = 3;
-    subscriptions: Array<{ dispose(): unknown }> = [];
-    workspaceState: Memento = new MockMemento();
-    globalState = new MockGlobalStorage();
-    extensionPath = "";
-    asAbsolutePath(_relativePath: string): string {
-      throw new Error("Method not implemented.");
-    }
-    storagePath = "";
-    globalStoragePath = "";
-    logPath = "";
-    extensionUri = Uri.parse("");
-    environmentVariableCollection = new MockEnvironmentVariableCollection();
-    secrets = new MockSecretStorage();
-    storageUri = Uri.parse("");
-    globalStorageUri = Uri.parse("");
-    logUri = Uri.parse("");
-    extension: any;
-  }
-
-  class MockEnvironmentVariableCollection
-    implements EnvironmentVariableCollection
-  {
-    [Symbol.iterator](): Iterator<
-      [variable: string, mutator: EnvironmentVariableMutator],
-      any,
-      undefined
-    > {
-      throw new Error("Method not implemented.");
-    }
-    persistent = false;
-    replace(_variable: string, _value: string): void {
-      throw new Error("Method not implemented.");
-    }
-    append(_variable: string, _value: string): void {
-      throw new Error("Method not implemented.");
-    }
-    prepend(_variable: string, _value: string): void {
-      throw new Error("Method not implemented.");
-    }
-    get(_variable: string): EnvironmentVariableMutator | undefined {
-      throw new Error("Method not implemented.");
-    }
-    forEach(
-      _callback: (
-        variable: string,
-        mutator: EnvironmentVariableMutator,
-        collection: EnvironmentVariableCollection,
-      ) => any,
-      _thisArg?: any,
-    ): void {
-      throw new Error("Method not implemented.");
-    }
-    delete(_variable: string): void {
-      throw new Error("Method not implemented.");
-    }
-    clear(): void {
-      throw new Error("Method not implemented.");
-    }
-  }
-
   class MockMemento implements Memento {
     keys(): readonly string[] {
       throw new Error("Method not implemented.");
@@ -132,19 +60,6 @@ describe("Invocation rate limiter", () => {
     }
   }
 
-  class MockSecretStorage implements SecretStorage {
-    get(_key: string): Thenable<string | undefined> {
-      throw new Error("Method not implemented.");
-    }
-    store(_key: string, _value: string): Thenable<void> {
-      throw new Error("Method not implemented.");
-    }
-    delete(_key: string): Thenable<void> {
-      throw new Error("Method not implemented.");
-    }
-    onDidChange!: Event<SecretStorageChangeEvent>;
-  }
-
   it("initially invokes function", async () => {
     let numTimesFuncCalled = 0;
     const invocationRateLimiter = createInvocationRateLimiter(

From 6b4726bc2baf73101780b1296b8c535babc634de Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 12:20:14 +0100
Subject: [PATCH 108/119] Move InvocationRateLimiter to common

---
 extensions/ql-vscode/src/codeql-cli/distribution.ts           | 2 +-
 .../ql-vscode/src/{ => common}/invocation-rate-limiter.ts     | 2 +-
 .../common}/invocation-rate-limiter.test.ts                   | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename extensions/ql-vscode/src/{ => common}/invocation-rate-limiter.ts (98%)
 rename extensions/ql-vscode/test/{vscode-tests/no-workspace => unit-tests/common}/invocation-rate-limiter.test.ts (97%)

diff --git a/extensions/ql-vscode/src/codeql-cli/distribution.ts b/extensions/ql-vscode/src/codeql-cli/distribution.ts
index c27608700..7c98b3aaa 100644
--- a/extensions/ql-vscode/src/codeql-cli/distribution.ts
+++ b/extensions/ql-vscode/src/codeql-cli/distribution.ts
@@ -22,7 +22,7 @@ import {
 import {
   InvocationRateLimiter,
   InvocationRateLimiterResultKind,
-} from "../invocation-rate-limiter";
+} from "../common/invocation-rate-limiter";
 
 /**
  * distribution.ts
diff --git a/extensions/ql-vscode/src/invocation-rate-limiter.ts b/extensions/ql-vscode/src/common/invocation-rate-limiter.ts
similarity index 98%
rename from extensions/ql-vscode/src/invocation-rate-limiter.ts
rename to extensions/ql-vscode/src/common/invocation-rate-limiter.ts
index 3788b9664..325a5df18 100644
--- a/extensions/ql-vscode/src/invocation-rate-limiter.ts
+++ b/extensions/ql-vscode/src/common/invocation-rate-limiter.ts
@@ -1,4 +1,4 @@
-import { Memento } from "./common/memento";
+import { Memento } from "./memento";
 
 /**
  * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts b/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts
similarity index 97%
rename from extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
rename to extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts
index ac6ba84ef..be42f7ca0 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/invocation-rate-limiter.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts
@@ -1,5 +1,5 @@
-import { Memento } from "vscode";
-import { InvocationRateLimiter } from "../../../src/invocation-rate-limiter";
+import type { Memento } from "vscode";
+import { InvocationRateLimiter } from "../../../src/common/invocation-rate-limiter";
 
 describe("Invocation rate limiter", () => {
   // 1 January 2020

From 1920a2c6b451a33f21dd26461e24b0ed7d3c8986 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 31 May 2023 13:51:51 +0200
Subject: [PATCH 109/119] Remove use of `Pick` in auto model usages query

---
 .../auto-model-usages-query.ts                   | 16 +++-------------
 .../local-databases/database-item-impl.ts        | 10 +++-------
 .../databases/local-databases/database-item.ts   |  8 ++------
 extensions/ql-vscode/src/helpers.ts              |  2 +-
 extensions/ql-vscode/src/query-results.ts        |  2 +-
 5 files changed, 10 insertions(+), 28 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
index 35a77ed62..843100650 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -8,19 +8,9 @@ import { interpretResultsSarif } from "../query-results";
 import { ProgressCallback } from "../common/vscode/progress";
 
 type Options = {
-  cliServer: Pick<
-    CodeQLCliServer,
-    "resolveDatabase" | "resolveQlpacks" | "interpretBqrsSarif"
-  >;
-  queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
-  databaseItem: Pick<
-    DatabaseItem,
-    | "contents"
-    | "databaseUri"
-    | "language"
-    | "sourceArchive"
-    | "getSourceLocationPrefix"
-  >;
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
   queryStorageDir: string;
 
   progress: ProgressCallback;
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
index efbbb9b96..16f184324 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts
@@ -121,9 +121,7 @@ export class DatabaseItemImpl implements DatabaseItem {
   /**
    * Returns information about a database.
    */
-  private async getDbInfo(
-    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
-  ): Promise<cli.DbInfo> {
+  private async getDbInfo(server: cli.CodeQLCliServer): Promise<cli.DbInfo> {
     if (this._dbinfo === undefined) {
       this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath);
     }
@@ -135,7 +133,7 @@ export class DatabaseItemImpl implements DatabaseItem {
    * has a `.dbinfo` file, which is the source of the prefix.
    */
   public async getSourceLocationPrefix(
-    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
+    server: cli.CodeQLCliServer,
   ): Promise<string> {
     const dbInfo = await this.getDbInfo(server);
     return dbInfo.sourceLocationPrefix;
@@ -144,9 +142,7 @@ export class DatabaseItemImpl implements DatabaseItem {
   /**
    * Returns path to dataset folder of database.
    */
-  public async getDatasetFolder(
-    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
-  ): Promise<string> {
+  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
     const dbInfo = await this.getDbInfo(server);
     return dbInfo.datasetFolder;
   }
diff --git a/extensions/ql-vscode/src/databases/local-databases/database-item.ts b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
index ac5ed47d9..1794d8e75 100644
--- a/extensions/ql-vscode/src/databases/local-databases/database-item.ts
+++ b/extensions/ql-vscode/src/databases/local-databases/database-item.ts
@@ -43,16 +43,12 @@ export interface DatabaseItem {
   /**
    * Returns `sourceLocationPrefix` of exported database.
    */
-  getSourceLocationPrefix(
-    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
-  ): Promise<string>;
+  getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
 
   /**
    * Returns dataset folder of exported database.
    */
-  getDatasetFolder(
-    server: Pick<cli.CodeQLCliServer, "resolveDatabase">,
-  ): Promise<string>;
+  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
 
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive,
diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts
index 374e066d7..d4e8a9a8c 100644
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@@ -786,7 +786,7 @@ export async function askForLanguage(
  * @returns A promise that resolves to the query metadata, if available.
  */
 export async function tryGetQueryMetadata(
-  cliServer: Pick<CodeQLCliServer, "resolveMetadata">,
+  cliServer: CodeQLCliServer,
   queryPath: string,
 ): Promise<QueryMetadata | undefined> {
   try {
diff --git a/extensions/ql-vscode/src/query-results.ts b/extensions/ql-vscode/src/query-results.ts
index 767729e23..77b3461ca 100644
--- a/extensions/ql-vscode/src/query-results.ts
+++ b/extensions/ql-vscode/src/query-results.ts
@@ -135,7 +135,7 @@ export class CompletedQueryInfo implements QueryWithResults {
  * Call cli command to interpret SARIF results.
  */
 export async function interpretResultsSarif(
-  cli: Pick<cli.CodeQLCliServer, "interpretBqrsSarif">,
+  cli: cli.CodeQLCliServer,
   metadata: QueryMetadata | undefined,
   resultsPaths: ResultsPaths,
   sourceInfo?: cli.SourceInfo,

From bce097d939fafe3c08ac4b6c8a6c92ecb8c7f216 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 31 May 2023 14:00:29 +0200
Subject: [PATCH 110/119] Create type for usage snippets by signature

---
 .../src/data-extensions-editor/auto-model-usages-query.ts     | 4 +++-
 extensions/ql-vscode/src/data-extensions-editor/auto-model.ts | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
index 843100650..1377161d6 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -16,13 +16,15 @@ type Options = {
   progress: ProgressCallback;
 };
 
+export type UsageSnippetsBySignature = Record<string, string[]>;
+
 export async function getAutoModelUsages({
   cliServer,
   queryRunner,
   databaseItem,
   queryStorageDir,
   progress,
-}: Options): Promise<Record<string, string[]>> {
+}: Options): Promise<UsageSnippetsBySignature> {
   const maxStep = 1500;
 
   const cancellationTokenSource = new CancellationTokenSource();
diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
index 2cfa0b334..af3a6dab1 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model.ts
@@ -6,12 +6,13 @@ import {
   Method,
   ModelRequest,
 } from "./auto-model-api";
+import type { UsageSnippetsBySignature } from "./auto-model-usages-query";
 
 export function createAutoModelRequest(
   language: string,
   externalApiUsages: ExternalApiUsage[],
   modeledMethods: Record<string, ModeledMethod>,
-  usages: Record<string, string[]>,
+  usages: UsageSnippetsBySignature,
 ): ModelRequest {
   const request: ModelRequest = {
     language,

From c654bfa4c425905ffd548ee2946bc05dd3e5aa81 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 31 May 2023 14:08:15 +0200
Subject: [PATCH 111/119] Add some comments for the auto model usages query

---
 .../auto-model-usages-query.ts                    | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
index 1377161d6..8e4a34991 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -29,6 +29,9 @@ export async function getAutoModelUsages({
 
   const cancellationTokenSource = new CancellationTokenSource();
 
+  // This will re-run the query that was already run when opening the data extensions editor. This
+  // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
+  // need to interpret the results.
   const queryResult = await runQuery({
     cliServer,
     queryRunner,
@@ -52,6 +55,9 @@ export async function getAutoModelUsages({
     message: "Retrieving source location prefix",
   });
 
+  // CodeQL needs to have access to the database to be able to retrieve the
+  // snippets from it. The source location prefix is used to determine the
+  // base path of the database.
   const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
     cliServer,
   );
@@ -70,9 +76,17 @@ export async function getAutoModelUsages({
     message: "Interpreting results",
   });
 
+  // Convert the results to SARIF so that Codeql will retrieve the snippets
+  // from the datababe. This means we don't need to do that in the extension
+  // and everything is handled by the CodeQL CLI.
   const sarif = await interpretResultsSarif(
     cliServer,
     {
+      // To interpret the results we need to provide metadata about the query. We could do this using
+      // `resolveMetadata` but that would be an extra call to the CodeQL CLI server and would require
+      // us to know the path to the query on the filesystem. Since we know what the metadata should
+      // look like and the only metadata that the CodeQL CLI requires is an ID and the kind, we can
+      // simply use constants here.
       kind: "problem",
       id: "usage",
     },
@@ -100,6 +114,7 @@ export async function getAutoModelUsages({
     throw new Error("No results");
   }
 
+  // This will group the snippets by the method signature.
   for (const result of results) {
     const signature = result.message.text;
 

From c01753041002836de2f44de9f6ab89ade96ef985 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Wed, 31 May 2023 14:53:35 +0200
Subject: [PATCH 112/119] Fix incorrect order in BQRS unit test

---
 .../data-extensions-editor/bqrs.test.ts       | 240 +++++++++---------
 1 file changed, 120 insertions(+), 120 deletions(-)

diff --git a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts
index 111884de9..6d22b178e 100644
--- a/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/data-extensions-editor/bqrs.test.ts
@@ -10,126 +10,6 @@ describe("decodeBqrsToExternalApiUsages", () => {
       { kind: "String" },
     ],
     tuples: [
-      [
-        {
-          label: "new Sql2o(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 10,
-            startColumn: 33,
-            endLine: 10,
-            endColumn: 88,
-          },
-        },
-        "org.sql2o.Sql2o#Sql2o(String,String,String)",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "open(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 14,
-            startColumn: 24,
-            endLine: 14,
-            endColumn: 35,
-          },
-        },
-        "org.sql2o.Sql2o#open()",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "executeScalar(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 15,
-            startColumn: 13,
-            endLine: 15,
-            endColumn: 85,
-          },
-        },
-        "org.sql2o.Query#executeScalar(Class)",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "createQuery(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 15,
-            startColumn: 13,
-            endLine: 15,
-            endColumn: 56,
-          },
-        },
-        "org.sql2o.Connection#createQuery(String)",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "new Sql2o(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 23,
-            startColumn: 23,
-            endLine: 23,
-            endColumn: 36,
-          },
-        },
-        "org.sql2o.Sql2o#Sql2o(String)",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "open(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 25,
-            startColumn: 24,
-            endLine: 25,
-            endColumn: 35,
-          },
-        },
-        "org.sql2o.Sql2o#open()",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "executeScalar(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 26,
-            startColumn: 13,
-            endLine: 26,
-            endColumn: 68,
-          },
-        },
-        "org.sql2o.Query#executeScalar(Class)",
-        "false",
-        "supported",
-      ],
-      [
-        {
-          label: "createQuery(...)",
-          url: {
-            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
-            startLine: 26,
-            startColumn: 13,
-            endLine: 26,
-            endColumn: 39,
-          },
-        },
-        "org.sql2o.Connection#createQuery(String)",
-        "false",
-        "supported",
-      ],
       [
         {
           label: "println(...)",
@@ -160,6 +40,126 @@ describe("decodeBqrsToExternalApiUsages", () => {
         "false",
         "supported",
       ],
+      [
+        {
+          label: "createQuery(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 15,
+            startColumn: 13,
+            endLine: 15,
+            endColumn: 56,
+          },
+        },
+        "org.sql2o.Connection#createQuery(String)",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "createQuery(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 26,
+            startColumn: 13,
+            endLine: 26,
+            endColumn: 39,
+          },
+        },
+        "org.sql2o.Connection#createQuery(String)",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "executeScalar(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 15,
+            startColumn: 13,
+            endLine: 15,
+            endColumn: 85,
+          },
+        },
+        "org.sql2o.Query#executeScalar(Class)",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "executeScalar(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 26,
+            startColumn: 13,
+            endLine: 26,
+            endColumn: 68,
+          },
+        },
+        "org.sql2o.Query#executeScalar(Class)",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "open(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 14,
+            startColumn: 24,
+            endLine: 14,
+            endColumn: 35,
+          },
+        },
+        "org.sql2o.Sql2o#open()",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "open(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 25,
+            startColumn: 24,
+            endLine: 25,
+            endColumn: 35,
+          },
+        },
+        "org.sql2o.Sql2o#open()",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "new Sql2o(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 10,
+            startColumn: 33,
+            endLine: 10,
+            endColumn: 88,
+          },
+        },
+        "org.sql2o.Sql2o#Sql2o(String,String,String)",
+        "true",
+        "supported",
+      ],
+      [
+        {
+          label: "new Sql2o(...)",
+          url: {
+            uri: "file:/home/runner/work/sql2o-example/sql2o-example/src/main/java/org/example/HelloController.java",
+            startLine: 23,
+            startColumn: 23,
+            endLine: 23,
+            endColumn: 36,
+          },
+        },
+        "org.sql2o.Sql2o#Sql2o(String)",
+        "true",
+        "supported",
+      ],
     ],
   };
 

From 9b84e0f831e0b19c3685b7a36f9d8029f702c2e6 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 14:59:23 +0100
Subject: [PATCH 113/119] "Fix" distribution tests

---
 .../no-workspace/codeql-cli/distribution.test.ts            | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts
index 0c595078d..4f3ac600d 100644
--- a/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution.test.ts
@@ -286,7 +286,7 @@ describe("Launcher path", () => {
     const manager = new DistributionManager(
       { customCodeQlPath: pathToCmd } as any,
       {} as any,
-      undefined as any,
+      {} as any,
     );
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -304,7 +304,7 @@ describe("Launcher path", () => {
     const manager = new DistributionManager(
       { customCodeQlPath: pathToCmd } as any,
       {} as any,
-      undefined as any,
+      {} as any,
     );
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -319,7 +319,7 @@ describe("Launcher path", () => {
     const manager = new DistributionManager(
       { customCodeQlPath: pathToCmd } as any,
       {} as any,
-      undefined as any,
+      {} as any,
     );
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();

From 50c46b603a05b243cf4b2b90360336aa44bda8c7 Mon Sep 17 00:00:00 2001
From: Robert <robertbrignull@github.com>
Date: Wed, 31 May 2023 16:21:28 +0100
Subject: [PATCH 114/119] Remove MockGlobalStorage

---
 .../unit-tests/common/invocation-rate-limiter.test.ts     | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts b/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts
index be42f7ca0..4623c62a5 100644
--- a/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts
+++ b/extensions/ql-vscode/test/unit-tests/common/invocation-rate-limiter.test.ts
@@ -18,7 +18,7 @@ describe("Invocation rate limiter", () => {
     func: () => Promise<T>,
   ): InvocationRateLimiter<T> {
     return new InvocationRateLimiter(
-      new MockGlobalStorage(),
+      new MockMemento(),
       funcIdentifier,
       func,
       (s) => createDate(s),
@@ -54,12 +54,6 @@ describe("Invocation rate limiter", () => {
     }
   }
 
-  class MockGlobalStorage extends MockMemento {
-    public setKeysForSync(_keys: string[]): void {
-      return;
-    }
-  }
-
   it("initially invokes function", async () => {
     let numTimesFuncCalled = 0;
     const invocationRateLimiter = createInvocationRateLimiter(

From dd268af9e972f3967a66765753f3daa2cf8622ed Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@users.noreply.github.com>
Date: Thu, 1 Jun 2023 10:47:16 +0200
Subject: [PATCH 115/119] Use type in auto model usages query file

Co-authored-by: Charis Kyriakou <charisk@users.noreply.github.com>
---
 .../src/data-extensions-editor/auto-model-usages-query.ts       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
index 8e4a34991..3a4757017 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts
@@ -107,7 +107,7 @@ export async function getAutoModelUsages({
     message: "Parsing results",
   });
 
-  const snippets: Record<string, string[]> = {};
+  const snippets: UsageSnippetsBySignature = {};
 
   const results = sarif.runs[0]?.results;
   if (!results) {

From 5387546e930ee31b6d2109c27c2ed4d72c563b0c Mon Sep 17 00:00:00 2001
From: Shati Patel <42641846+shati-patel@users.noreply.github.com>
Date: Thu, 1 Jun 2023 09:49:35 +0100
Subject: [PATCH 116/119] Don't log the `resolve queries` CLI command (#2454)

---
 extensions/ql-vscode/src/codeql-cli/cli.ts    | 43 ++++++++++++++-----
 .../src/queries-panel/query-discovery.ts      |  7 ++-
 .../cli-integration/run-cli.test.ts           | 22 ++++++++++
 3 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts
index fe9e02115..1e7a80cf8 100644
--- a/extensions/ql-vscode/src/codeql-cli/cli.ts
+++ b/extensions/ql-vscode/src/codeql-cli/cli.ts
@@ -218,7 +218,7 @@ export class CodeQLCliServer implements Disposable {
     private readonly app: App,
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    public readonly logger: Logger,
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -330,6 +330,7 @@ export class CodeQLCliServer implements Disposable {
     commandArgs: string[],
     description: string,
     onLine?: OnLineCallback,
+    silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
@@ -349,7 +350,12 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(" ");
-      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      // If we are running silently, we don't want to print anything to the console.
+      if (!silent) {
+        void this.logger.log(
+          `${description} using CodeQL CLI: ${argsString}...`,
+        );
+      }
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -395,24 +401,30 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        void this.logger.log("CLI command succeeded.");
+        if (!silent) {
+          void this.logger.log("CLI command succeeded.");
+        }
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
+        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
         const newError =
           stderrBuffers.length === 0
-            ? new Error(`${description} failed: ${err}`)
+            ? new Error(
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
+              )
             : new Error(
-                `${description} failed: ${Buffer.concat(stderrBuffers).toString(
-                  "utf8",
-                )}`,
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
+                  stderrBuffers,
+                ).toString("utf8")}`,
               );
         newError.stack += getErrorStack(err);
         throw newError;
       } finally {
-        void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent) {
+          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
         process.stderr.removeAllListeners("data");
@@ -549,9 +561,11 @@ export class CodeQLCliServer implements Disposable {
     {
       progressReporter,
       onLine,
+      silent = false,
     }: {
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<string> {
     if (progressReporter) {
@@ -567,6 +581,7 @@ export class CodeQLCliServer implements Disposable {
             commandArgs,
             description,
             onLine,
+            silent,
           ).then(resolve, reject);
         } catch (err) {
           reject(err);
@@ -600,10 +615,12 @@ export class CodeQLCliServer implements Disposable {
       addFormat = true,
       progressReporter,
       onLine,
+      silent = false,
     }: {
       addFormat?: boolean;
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
@@ -614,6 +631,7 @@ export class CodeQLCliServer implements Disposable {
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
       onLine,
+      silent,
     });
     try {
       return JSON.parse(result) as OutputType;
@@ -739,14 +757,19 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Finds all available queries in a given directory.
    * @param queryDir Root of directory tree to search for queries.
+   * @param silent If true, don't print logs to the CodeQL extension log.
    * @returns The list of queries that were found.
    */
-  public async resolveQueries(queryDir: string): Promise<ResolvedQueries> {
+  public async resolveQueries(
+    queryDir: string,
+    silent?: boolean,
+  ): Promise<ResolvedQueries> {
     const subcommandArgs = [queryDir];
     return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
       ["resolve", "queries"],
       subcommandArgs,
       "Resolving queries",
+      { silent },
     );
   }
 
diff --git a/extensions/ql-vscode/src/queries-panel/query-discovery.ts b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
index d1a077da0..f7b25ea09 100644
--- a/extensions/ql-vscode/src/queries-panel/query-discovery.ts
+++ b/extensions/ql-vscode/src/queries-panel/query-discovery.ts
@@ -114,7 +114,12 @@ export class QueryDiscovery
     const fullPath = workspaceFolder.uri.fsPath;
     const name = workspaceFolder.name;
 
-    const resolvedQueries = await this.cliServer.resolveQueries(fullPath);
+    // We don't want to log each invocation of resolveQueries, since it clutters up the log.
+    const silent = true;
+    const resolvedQueries = await this.cliServer.resolveQueries(
+      fullPath,
+      silent,
+    );
     if (resolvedQueries.length === 0) {
       return undefined;
     }
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts
index 3bc1a6b85..71908530c 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/run-cli.test.ts
@@ -15,6 +15,7 @@ import {
 import { KeyType, resolveQueries } from "../../../src/language-support";
 import { faker } from "@faker-js/faker";
 import { getActivatedExtension } from "../global.helper";
+import { BaseLogger } from "../../../src/common";
 
 /**
  * Perform proper integration tests by running the CLI
@@ -23,10 +24,14 @@ describe("Use cli", () => {
   let cli: CodeQLCliServer;
   let supportedLanguages: string[];
 
+  let logSpy: jest.SpiedFunction<BaseLogger["log"]>;
+
   beforeEach(async () => {
     const extension = await getActivatedExtension();
     cli = extension.cliServer;
     supportedLanguages = await cli.getSupportedLanguages();
+
+    logSpy = jest.spyOn(cli.logger, "log");
   });
 
   if (process.env.CLI_VERSION && process.env.CLI_VERSION !== "nightly") {
@@ -42,6 +47,23 @@ describe("Use cli", () => {
     expect(result).toEqual(["-J-Xmx4096M", "--off-heap-ram=4096"]);
   });
 
+  describe("silent logging", () => {
+    it("should log command output", async () => {
+      const queryDir = getOnDiskWorkspaceFolders()[0];
+      await cli.resolveQueries(queryDir);
+
+      expect(logSpy).toHaveBeenCalled();
+    });
+
+    it("shouldn't log command output if the `silent` flag is set", async () => {
+      const queryDir = getOnDiskWorkspaceFolders()[0];
+      const silent = true;
+      await cli.resolveQueries(queryDir, silent);
+
+      expect(logSpy).not.toHaveBeenCalled();
+    });
+  });
+
   itWithCodeQL()("should resolve query packs", async () => {
     const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
     // Depending on the version of the CLI, the qlpacks may have different names

From 2b915b82e4e532b81b72da789b2f3792af5a8522 Mon Sep 17 00:00:00 2001
From: Koen Vlaswinkel <koesie10@github.com>
Date: Thu, 1 Jun 2023 14:59:47 +0200
Subject: [PATCH 117/119] Fix neutral definition for CodeQL 2.13.3

In CodeQL 2.13.3, the definition of the neutralModel predicate has
changed to include the `kind`. This updates the definition of the
data extensions editor to match the new definition.

One caveat is that when selecting a `kind` other than `summary`, the
method will not be shown as supported. This is because a
`NeutralCallable` only calls into `neutralSummaryElement`. This matches
the previous behavior because setting the `kind` to `source` or `sink`
only says that the method is either not a source or not a sink, but not
both. Only `summary` fully models the method.

See: https://github.com/github/codeql/pull/12931
See: https://github.com/github/codeql/blob/ff78ac98d27c7b9f1adffcf235c56855f8348ad0/java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll#L338
See: https://github.com/github/codeql/blob/ff78ac98d27c7b9f1adffcf235c56855f8348ad0/java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll#L160
---
 .../ql-vscode/src/data-extensions-editor/predicates.ts      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/extensions/ql-vscode/src/data-extensions-editor/predicates.ts b/extensions/ql-vscode/src/data-extensions-editor/predicates.ts
index f52115163..65a728173 100644
--- a/extensions/ql-vscode/src/data-extensions-editor/predicates.ts
+++ b/extensions/ql-vscode/src/data-extensions-editor/predicates.ts
@@ -116,13 +116,14 @@ export const extensiblePredicateDefinitions: Record<
   neutral: {
     extensiblePredicate: "neutralModel",
     // extensible predicate neutralModel(
-    //   string package, string type, string name, string signature, string provenance
+    //   string package, string type, string name, string signature, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
       method.externalApiUsage.packageName,
       method.externalApiUsage.typeName,
       method.externalApiUsage.methodName,
       method.externalApiUsage.methodParameters,
+      method.modeledMethod.kind,
       "manual",
     ],
     readModeledMethod: (row) => ({
@@ -131,8 +132,9 @@ export const extensiblePredicateDefinitions: Record<
         type: "neutral",
         input: "",
         output: "",
-        kind: "",
+        kind: row[4] as string,
       },
     }),
+    supportedKinds: ["summary", "source", "sink"],
   },
 };

From cf343785a71aae5a8f4d5670e483e8bcc044ea1f Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 1 Jun 2023 15:57:18 +0100
Subject: [PATCH 118/119] Bump CLI version from v2.13.1 to v2.13.3 for
 integration tests (#2465)

Co-authored-by: github-actions[bot] <github-actions@github.com>
---
 extensions/ql-vscode/supported_cli_versions.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/ql-vscode/supported_cli_versions.json b/extensions/ql-vscode/supported_cli_versions.json
index 7fa2cc8c1..fe9f952c6 100644
--- a/extensions/ql-vscode/supported_cli_versions.json
+++ b/extensions/ql-vscode/supported_cli_versions.json
@@ -1,5 +1,5 @@
 [
-  "v2.13.1",
+  "v2.13.3",
   "v2.12.7",
   "v2.11.6",
   "v2.7.6",

From b2adbf63eb38e6305ecac9caf1767c66b7b2494c Mon Sep 17 00:00:00 2001
From: Nora <norascheuch@github.com>
Date: Fri, 2 Jun 2023 08:32:54 +0000
Subject: [PATCH 119/119] Move file to language-support/contextual

---
 .../{pure => language-support/contextual}/cached-operation.ts   | 0
 .../src/language-support/contextual/template-provider.ts        | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename extensions/ql-vscode/src/{pure => language-support/contextual}/cached-operation.ts (100%)

diff --git a/extensions/ql-vscode/src/pure/cached-operation.ts b/extensions/ql-vscode/src/language-support/contextual/cached-operation.ts
similarity index 100%
rename from extensions/ql-vscode/src/pure/cached-operation.ts
rename to extensions/ql-vscode/src/language-support/contextual/cached-operation.ts
diff --git a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
index 1fef79d75..7943f4513 100644
--- a/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
+++ b/extensions/ql-vscode/src/language-support/contextual/template-provider.ts
@@ -17,7 +17,7 @@ import {
 } from "../../common/vscode/archive-filesystem-provider";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { DatabaseManager } from "../../databases/local-databases";
-import { CachedOperation } from "../../pure/cached-operation";
+import { CachedOperation } from "./cached-operation";
 import { ProgressCallback, withProgress } from "../../common/vscode/progress";
 import { KeyType } from "./key-type";
 import {