Merge remote-tracking branch 'origin/main' into koesie10/yauzl-progress

docs/test-plan.md

@@ -185,6 +185,24 @@ Note that this test requires the feature flag: `codeQL.model.flowGeneration`
 2. Click "Generate".
    - Check that rows are filled out.
 
+### GitHub database download
+
+#### Test case 1: Download a database
+
+Open a clone of the [`github/codeql`](https://github.com/github/codeql) repository as a folder.
+
+1. Wait a few seconds until the CodeQL extension is fully initialized.
+   - Check that the following prompt appears:
+
+     ![database-download-prompt](images/github-database-download-prompt.png)
+
+   - If the prompt does not appear, ensure that the `codeQL.githubDatabase.download` setting is not set in workspace or user settings.
+
+2. Click "Download".
+3. Select the "C#" and "JavaScript" databases.
+   - Check that there are separate notifications for both downloads.
+   - Check that both databases are added when the downloads are complete.
+
 ### General
 
 #### Test case 1: Change to a different colour theme

extensions/ql-vscode/CHANGELOG.md

@@ -5,6 +5,7 @@
 - Add a prompt for downloading a GitHub database when opening a GitHub repository. [#3138](https://github.com/github/vscode-codeql/pull/3138)
 - Avoid showing a popup when hovering over source elements in database source files. [#3125](https://github.com/github/vscode-codeql/pull/3125)
 - Add comparison of alerts when comparing query results. This allows viewing path explanations for differences in alerts. [#3113](https://github.com/github/vscode-codeql/pull/3113)
+- Fix a bug where the CodeQL CLI and variant analysis results were corrupted after extraction in VS Code Insiders. [#3151](https://github.com/github/vscode-codeql/pull/3151) & [#3152](https://github.com/github/vscode-codeql/pull/3152)
 
 ## 1.11.0 - 13 December 2023
 

extensions/ql-vscode/package.json

@@ -424,11 +424,6 @@
         "title": "GitHub Databases",
         "order": 8,
         "properties": {
-          "codeQL.githubDatabase.enable": {
-            "type": "boolean",
-            "default": false,
-            "markdownDescription": "Enable automatic detection of GitHub databases."
-          },
           "codeQL.githubDatabase.download": {
             "type": "string",
             "default": "ask",

extensions/ql-vscode/src/databases/github-databases/github-databases-module.ts

@@ -12,10 +12,7 @@ import {
   askForGitHubDatabaseDownload,
   downloadDatabaseFromGitHub,
 } from "./download";
-import {
-  GitHubDatabaseConfig,
-  GitHubDatabaseConfigListener,
-} from "../../config";
+import { GitHubDatabaseConfig } from "../../config";
 import { DatabaseManager } from "../local-databases";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { CodeqlDatabase, listDatabases, ListDatabasesResult } from "./api";
@@ -28,17 +25,18 @@ import {
 import { Octokit } from "@octokit/rest";
 
 export class GitHubDatabasesModule extends DisposableObject {
-  private readonly config: GitHubDatabaseConfig;
-
-  private constructor(
+  /**
+   * This constructor is public only for testing purposes. Please use the `initialize` method
+   * instead.
+   */
+  constructor(
     private readonly app: App,
     private readonly databaseManager: DatabaseManager,
     private readonly databaseStoragePath: string,
     private readonly cliServer: CodeQLCliServer,
+    private readonly config: GitHubDatabaseConfig,
   ) {
     super();
-
-    this.config = this.push(new GitHubDatabaseConfigListener());
   }
 
   public static async initialize(
@@ -46,12 +44,14 @@ export class GitHubDatabasesModule extends DisposableObject {
     databaseManager: DatabaseManager,
     databaseStoragePath: string,
     cliServer: CodeQLCliServer,
+    config: GitHubDatabaseConfig,
   ): Promise<GitHubDatabasesModule> {
     const githubDatabasesModule = new GitHubDatabasesModule(
       app,
       databaseManager,
       databaseStoragePath,
       cliServer,
+      config,
     );
     app.subscriptions.push(githubDatabasesModule);
 
@@ -72,7 +72,10 @@ export class GitHubDatabasesModule extends DisposableObject {
     });
   }
 
-  private async promptGitHubRepositoryDownload(): Promise<void> {
+  /**
+   * This method is public only for testing purposes.
+   */
+  public async promptGitHubRepositoryDownload(): Promise<void> {
     if (this.config.download === "never") {
       return;
     }

extensions/ql-vscode/src/extension.ts

@@ -28,6 +28,7 @@ import { CliVersionConstraint, CodeQLCliServer } from "./codeql-cli/cli";
 import {
   CliConfigListener,
   DistributionConfigListener,
+  GitHubDatabaseConfigListener,
   isCanary,
   joinOrderWarningThreshold,
   QueryHistoryConfigListener,
@@ -867,11 +868,14 @@ async function activateWithInstalledDistribution(
     ),
   );
 
+  const githubDatabaseConfigListener = new GitHubDatabaseConfigListener();
+
   await GitHubDatabasesModule.initialize(
     app,
     dbm,
     getContextStoragePath(ctx),
     cliServer,
+    githubDatabaseConfigListener,
   );
 
   void extLogger.log("Initializing query history.");

extensions/ql-vscode/src/query-testing/qltest-discovery.ts

@@ -31,6 +31,15 @@ export class QLTestDiscovery extends Discovery {
     super("QL Test Discovery", extLogger);
 
     this.push(this.watcher.onDidChange(this.handleDidChange, this));
+
+    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
+    this.watcher.addWatch(
+      new RelativePattern(this.workspaceFolder.uri.fsPath, "**/*.{ql,qlref}"),
+    );
+    // need to explicitly watch for changes to directories themselves.
+    this.watcher.addWatch(
+      new RelativePattern(this.workspaceFolder.uri.fsPath, "**/"),
+    );
   }
 
   /**
@@ -56,15 +65,6 @@ export class QLTestDiscovery extends Discovery {
   protected async discover() {
     this._testDirectory = await this.discoverTests();
 
-    this.watcher.clear();
-    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
-    this.watcher.addWatch(
-      new RelativePattern(this.workspaceFolder.uri.fsPath, "**/*.{ql,qlref}"),
-    );
-    // need to explicitly watch for changes to directories themselves.
-    this.watcher.addWatch(
-      new RelativePattern(this.workspaceFolder.uri.fsPath, "**/"),
-    );
     this._onDidChangeTests.fire(undefined);
   }
 

extensions/ql-vscode/supported_cli_versions.json

@@ -1,5 +1,5 @@
 [
-  "v2.15.4",
+  "v2.15.5",
   "v2.14.6",
   "v2.13.5",
   "v2.12.7",

extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-databases/github-databases-module.test.ts

@@ -0,0 +1,277 @@
+import { window } from "vscode";
+import { Octokit } from "@octokit/rest";
+import { createMockApp } from "../../../../__mocks__/appMock";
+import { App } from "../../../../../src/common/app";
+import { DatabaseManager } from "../../../../../src/databases/local-databases";
+import { mockEmptyDatabaseManager } from "../../query-testing/test-runner-helpers";
+import { CodeQLCliServer } from "../../../../../src/codeql-cli/cli";
+import { mockDatabaseItem, mockedObject } from "../../../utils/mocking.helpers";
+import { GitHubDatabaseConfig } from "../../../../../src/config";
+import { GitHubDatabasesModule } from "../../../../../src/databases/github-databases";
+import { ValueResult } from "../../../../../src/common/value-result";
+import { CodeqlDatabase } from "../../../../../src/databases/github-databases/api";
+
+import * as githubRepositoryFinder from "../../../../../src/databases/github-repository-finder";
+import * as githubDatabasesApi from "../../../../../src/databases/github-databases/api";
+import * as githubDatabasesDownload from "../../../../../src/databases/github-databases/download";
+import * as githubDatabasesUpdates from "../../../../../src/databases/github-databases/updates";
+import { DatabaseUpdate } from "../../../../../src/databases/github-databases/updates";
+
+describe("GitHubDatabasesModule", () => {
+  describe("promptGitHubRepositoryDownload", () => {
+    let app: App;
+    let databaseManager: DatabaseManager;
+    let databaseStoragePath: string;
+    let cliServer: CodeQLCliServer;
+    let config: GitHubDatabaseConfig;
+    let gitHubDatabasesModule: GitHubDatabasesModule;
+
+    const owner = "github";
+    const repo = "vscode-codeql";
+
+    const databases: CodeqlDatabase[] = [
+      mockedObject<CodeqlDatabase>({}),
+      mockedObject<CodeqlDatabase>({}),
+    ];
+
+    let octokit: Octokit;
+
+    let findGitHubRepositoryForWorkspaceSpy: jest.SpiedFunction<
+      typeof githubRepositoryFinder.findGitHubRepositoryForWorkspace
+    >;
+    let listDatabasesSpy: jest.SpiedFunction<
+      typeof githubDatabasesApi.listDatabases
+    >;
+    let askForGitHubDatabaseDownloadSpy: jest.SpiedFunction<
+      typeof githubDatabasesDownload.askForGitHubDatabaseDownload
+    >;
+    let downloadDatabaseFromGitHubSpy: jest.SpiedFunction<
+      typeof githubDatabasesDownload.downloadDatabaseFromGitHub
+    >;
+    let isNewerDatabaseAvailableSpy: jest.SpiedFunction<
+      typeof githubDatabasesUpdates.isNewerDatabaseAvailable
+    >;
+    let askForGitHubDatabaseUpdateSpy: jest.SpiedFunction<
+      typeof githubDatabasesUpdates.askForGitHubDatabaseUpdate
+    >;
+    let downloadDatabaseUpdateFromGitHubSpy: jest.SpiedFunction<
+      typeof githubDatabasesUpdates.downloadDatabaseUpdateFromGitHub
+    >;
+    let showInformationMessageSpy: jest.SpiedFunction<
+      typeof window.showInformationMessage
+    >;
+
+    beforeEach(() => {
+      app = createMockApp();
+      databaseManager = mockEmptyDatabaseManager();
+      databaseStoragePath = "/a/b/some-path";
+      cliServer = mockedObject<CodeQLCliServer>({});
+      config = mockedObject<GitHubDatabaseConfig>({
+        download: "ask",
+        update: "ask",
+      });
+
+      gitHubDatabasesModule = new GitHubDatabasesModule(
+        app,
+        databaseManager,
+        databaseStoragePath,
+        cliServer,
+        config,
+      );
+
+      octokit = mockedObject<Octokit>({});
+
+      findGitHubRepositoryForWorkspaceSpy = jest
+        .spyOn(githubRepositoryFinder, "findGitHubRepositoryForWorkspace")
+        .mockResolvedValue(ValueResult.ok({ owner, name: repo }));
+
+      listDatabasesSpy = jest
+        .spyOn(githubDatabasesApi, "listDatabases")
+        .mockResolvedValue({
+          promptedForCredentials: false,
+          databases,
+          octokit,
+        });
+
+      askForGitHubDatabaseDownloadSpy = jest
+        .spyOn(githubDatabasesDownload, "askForGitHubDatabaseDownload")
+        .mockRejectedValue(new Error("Not implemented"));
+      downloadDatabaseFromGitHubSpy = jest
+        .spyOn(githubDatabasesDownload, "downloadDatabaseFromGitHub")
+        .mockRejectedValue(new Error("Not implemented"));
+      isNewerDatabaseAvailableSpy = jest
+        .spyOn(githubDatabasesUpdates, "isNewerDatabaseAvailable")
+        .mockImplementation(() => {
+          throw new Error("Not implemented");
+        });
+      askForGitHubDatabaseUpdateSpy = jest
+        .spyOn(githubDatabasesUpdates, "askForGitHubDatabaseUpdate")
+        .mockRejectedValue(new Error("Not implemented"));
+      downloadDatabaseUpdateFromGitHubSpy = jest
+        .spyOn(githubDatabasesUpdates, "downloadDatabaseUpdateFromGitHub")
+        .mockRejectedValue(new Error("Not implemented"));
+
+      showInformationMessageSpy = jest
+        .spyOn(window, "showInformationMessage")
+        .mockResolvedValue(undefined);
+    });
+
+    it("does nothing if the download config is set to never", async () => {
+      config = mockedObject<GitHubDatabaseConfig>({
+        download: "never",
+      });
+
+      gitHubDatabasesModule = new GitHubDatabasesModule(
+        app,
+        databaseManager,
+        databaseStoragePath,
+        cliServer,
+        config,
+      );
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(findGitHubRepositoryForWorkspaceSpy).not.toHaveBeenCalled();
+    });
+
+    it("does nothing if there is no GitHub repository", async () => {
+      findGitHubRepositoryForWorkspaceSpy.mockResolvedValue(
+        ValueResult.fail(["some error"]),
+      );
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+    });
+
+    it("does nothing if the user doesn't complete the download", async () => {
+      listDatabasesSpy.mockResolvedValue(undefined);
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+    });
+
+    it("does not show a prompt when there are no databases and the user was not prompted for credentials", async () => {
+      listDatabasesSpy.mockResolvedValue({
+        promptedForCredentials: false,
+        databases: [],
+        octokit,
+      });
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(showInformationMessageSpy).not.toHaveBeenCalled();
+    });
+
+    it("shows a prompt when there are no databases and the user was prompted for credentials", async () => {
+      listDatabasesSpy.mockResolvedValue({
+        promptedForCredentials: true,
+        databases: [],
+        octokit,
+      });
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(showInformationMessageSpy).toHaveBeenCalledWith(
+        "The GitHub repository does not have any CodeQL databases.",
+      );
+    });
+
+    it("shows a prompt when there are no databases and the user was prompted for credentials", async () => {
+      listDatabasesSpy.mockResolvedValue({
+        promptedForCredentials: true,
+        databases: [],
+        octokit,
+      });
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(showInformationMessageSpy).toHaveBeenCalledWith(
+        "The GitHub repository does not have any CodeQL databases.",
+      );
+    });
+
+    it("downloads the database if the user confirms the download", async () => {
+      isNewerDatabaseAvailableSpy.mockReturnValue({
+        type: "noDatabase",
+      });
+      askForGitHubDatabaseDownloadSpy.mockResolvedValue(true);
+      downloadDatabaseFromGitHubSpy.mockResolvedValue(undefined);
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(askForGitHubDatabaseDownloadSpy).toHaveBeenCalledWith(
+        databases,
+        config,
+      );
+      expect(downloadDatabaseFromGitHubSpy).toHaveBeenCalledWith(
+        octokit,
+        owner,
+        repo,
+        databases,
+        databaseManager,
+        databaseStoragePath,
+        cliServer,
+        app.commands,
+      );
+    });
+
+    it("does not perform the download if the user cancels the download", async () => {
+      isNewerDatabaseAvailableSpy.mockReturnValue({
+        type: "noDatabase",
+      });
+      askForGitHubDatabaseDownloadSpy.mockResolvedValue(false);
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(downloadDatabaseFromGitHubSpy).not.toHaveBeenCalled();
+    });
+
+    it("updates the database if the user confirms the update", async () => {
+      const databaseUpdates: DatabaseUpdate[] = [
+        {
+          database: databases[0],
+          databaseItem: mockDatabaseItem(),
+        },
+      ];
+      isNewerDatabaseAvailableSpy.mockReturnValue({
+        type: "updateAvailable",
+        databaseUpdates,
+      });
+      askForGitHubDatabaseUpdateSpy.mockResolvedValue(true);
+      downloadDatabaseUpdateFromGitHubSpy.mockResolvedValue(undefined);
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(askForGitHubDatabaseUpdateSpy).toHaveBeenCalledWith(
+        databaseUpdates,
+        config,
+      );
+      expect(downloadDatabaseUpdateFromGitHubSpy).toHaveBeenCalledWith(
+        octokit,
+        owner,
+        repo,
+        databaseUpdates,
+        databaseManager,
+        databaseStoragePath,
+        cliServer,
+        app.commands,
+      );
+    });
+
+    it("does not perform the update if the user cancels the update", async () => {
+      const databaseUpdates: DatabaseUpdate[] = [
+        {
+          database: databases[0],
+          databaseItem: mockDatabaseItem(),
+        },
+      ];
+      isNewerDatabaseAvailableSpy.mockReturnValue({
+        type: "updateAvailable",
+        databaseUpdates,
+      });
+      askForGitHubDatabaseUpdateSpy.mockResolvedValue(false);
+
+      await gitHubDatabasesModule.promptGitHubRepositoryDownload();
+
+      expect(downloadDatabaseUpdateFromGitHubSpy).not.toHaveBeenCalled();
+    });
+  });
+});