Merge pull request #3071 from github/koesie10/download-github-database

Prompt user for database download on startup

extensions/ql-vscode/package.json

@@ -421,8 +421,33 @@
       },
       {
         "type": "object",
-        "title": "Log insights",
+        "title": "GitHub Databases",
         "order": 8,
+        "properties": {
+          "codeQL.githubDatabase.enable": {
+            "type": "boolean",
+            "default": false,
+            "markdownDescription": "Enable automatic detection of GitHub databases."
+          },
+          "codeQL.githubDatabase.download": {
+            "type": "string",
+            "default": "ask",
+            "enum": [
+              "ask",
+              "never"
+            ],
+            "enumDescriptions": [
+              "Ask to download a GitHub database when a workspace is opened.",
+              "Never download a GitHub databases when a workspace is opened."
+            ],
+            "description": "Ask to download a GitHub database when a workspace is opened."
+          }
+        }
+      },
+      {
+        "type": "object",
+        "title": "Log insights",
+        "order": 9,
         "properties": {
           "codeQL.logInsights.joinOrderWarningThreshold": {
             "type": "number",
@@ -436,7 +461,7 @@
       {
         "type": "object",
         "title": "Telemetry",
-        "order": 9,
+        "order": 10,
         "properties": {
           "codeQL.telemetry.enableTelemetry": {
             "type": "boolean",

extensions/ql-vscode/src/config.ts

@@ -1,19 +1,19 @@
 import { DisposableObject } from "./common/disposable-object";
 import {
-  workspace,
+  ConfigurationChangeEvent,
+  ConfigurationScope,
+  ConfigurationTarget,
   Event,
   EventEmitter,
-  ConfigurationChangeEvent,
-  ConfigurationTarget,
-  ConfigurationScope,
+  workspace,
 } from "vscode";
 import { DistributionManager } from "./codeql-cli/distribution";
 import { extLogger } from "./common/logging/vscode";
 import { ONE_DAY_IN_MS } from "./common/time";
 import {
+  defaultFilterSortState,
   FilterKey,
   SortKey,
-  defaultFilterSortState,
 } from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
@@ -775,3 +775,52 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return !!ENABLE_RUBY.getValue<boolean>();
   }
 }
+
+const GITHUB_DATABASE_SETTING = new Setting("githubDatabase", ROOT_SETTING);
+
+// Feature flag for the GitHub database downnload.
+const GITHUB_DATABASE_ENABLE = new Setting("enable", GITHUB_DATABASE_SETTING);
+const GITHUB_DATABASE_DOWNLOAD = new Setting(
+  "download",
+  GITHUB_DATABASE_SETTING,
+);
+
+const GitHubDatabaseDownloadValues = ["ask", "never"] as const;
+type GitHubDatabaseDownload = (typeof GitHubDatabaseDownloadValues)[number];
+
+export interface GitHubDatabaseConfig {
+  enable: boolean;
+  download: GitHubDatabaseDownload;
+  setDownload(
+    value: GitHubDatabaseDownload,
+    target?: ConfigurationTarget,
+  ): Promise<void>;
+}
+
+export class GitHubDatabaseConfigListener
+  extends ConfigListener
+  implements GitHubDatabaseConfig
+{
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings(
+      [GITHUB_DATABASE_SETTING],
+      e,
+    );
+  }
+
+  public get enable() {
+    return !!GITHUB_DATABASE_ENABLE.getValue<boolean>();
+  }
+
+  public get download(): GitHubDatabaseDownload {
+    const value = GITHUB_DATABASE_DOWNLOAD.getValue<GitHubDatabaseDownload>();
+    return GitHubDatabaseDownloadValues.includes(value) ? value : "ask";
+  }
+
+  public async setDownload(
+    value: GitHubDatabaseDownload,
+    target: ConfigurationTarget = ConfigurationTarget.Workspace,
+  ): Promise<void> {
+    await GITHUB_DATABASE_DOWNLOAD.updateValue(value, target);
+  }
+}

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -207,6 +207,38 @@ export async function downloadGitHubDatabase(
   const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
     result;
 
+  return downloadGitHubDatabaseFromUrl(
+    databaseUrl,
+    databaseId,
+    databaseCreatedAt,
+    commitOid,
+    owner,
+    name,
+    octokit,
+    progress,
+    databaseManager,
+    storagePath,
+    cli,
+    makeSelected,
+    addSourceArchiveFolder,
+  );
+}
+
+export async function downloadGitHubDatabaseFromUrl(
+  databaseUrl: string,
+  databaseId: number,
+  databaseCreatedAt: string,
+  commitOid: string | null,
+  owner: string,
+  name: string,
+  octokit: Octokit.Octokit,
+  progress: ProgressCallback,
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  cli?: CodeQLCliServer,
+  makeSelected = true,
+  addSourceArchiveFolder = true,
+): Promise<DatabaseItem | undefined> {
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
    * The object is undocumented, but looks something like this:
@@ -229,7 +261,7 @@ export async function downloadGitHubDatabase(
     `${owner}/${name}`,
     {
       type: "github",
-      repository: nwo,
+      repository: `${owner}/${name}`,
       databaseId,
       databaseCreatedAt,
       commitOid,
@@ -577,7 +609,7 @@ export async function convertGithubNwoToDatabaseUrl(
     }
 
     const databaseForLanguage = response.data.find(
-      (db: any) => db.language === language,
+      (db) => db.language === language,
     );
     if (!databaseForLanguage) {
       throw new Error(`No database found for language '${language}'`);
@@ -599,9 +631,9 @@ export async function convertGithubNwoToDatabaseUrl(
 
 export async function promptForLanguage(
   languages: string[],
-  progress: ProgressCallback,
+  progress: ProgressCallback | undefined,
 ): Promise<string | undefined> {
-  progress({
+  progress?.({
     message: "Choose language",
     step: 2,
     maxStep: 2,

extensions/ql-vscode/src/databases/github-database-module.ts

@@ -2,15 +2,47 @@ import { DisposableObject } from "../common/disposable-object";
 import { App } from "../common/app";
 import { findGitHubRepositoryForWorkspace } from "./github-repository-finder";
 import { redactableError } from "../common/errors";
-import { asError } from "../common/helpers-pure";
+import { asError, getErrorMessage } from "../common/helpers-pure";
+import {
+  CodeqlDatabase,
+  findGitHubDatabasesForRepository,
+  promptAndDownloadGitHubDatabase,
+} from "./github-database-prompt";
+import {
+  GitHubDatabaseConfig,
+  GitHubDatabaseConfigListener,
+  isCanary,
+} from "../config";
+import { AppOctokit } from "../common/octokit";
+import { DatabaseManager } from "./local-databases";
+import { CodeQLCliServer } from "../codeql-cli/cli";
 
 export class GithubDatabaseModule extends DisposableObject {
-  private constructor(private readonly app: App) {
+  private readonly config: GitHubDatabaseConfig;
+
+  private constructor(
+    private readonly app: App,
+    private readonly databaseManager: DatabaseManager,
+    private readonly databaseStoragePath: string,
+    private readonly cliServer: CodeQLCliServer,
+  ) {
     super();
+
+    this.config = this.push(new GitHubDatabaseConfigListener());
   }
 
-  public static async initialize(app: App): Promise<GithubDatabaseModule> {
-    const githubDatabaseModule = new GithubDatabaseModule(app);
+  public static async initialize(
+    app: App,
+    databaseManager: DatabaseManager,
+    databaseStoragePath: string,
+    cliServer: CodeQLCliServer,
+  ): Promise<GithubDatabaseModule> {
+    const githubDatabaseModule = new GithubDatabaseModule(
+      app,
+      databaseManager,
+      databaseStoragePath,
+      cliServer,
+    );
     app.subscriptions.push(githubDatabaseModule);
 
     await githubDatabaseModule.initialize();
@@ -18,6 +50,10 @@ export class GithubDatabaseModule extends DisposableObject {
   }
 
   private async initialize(): Promise<void> {
+    if (!this.config.enable) {
+      return;
+    }
+
     // Start the check and downloading the database asynchronously. We don't want to block on this
     // in extension activation since this makes network requests and waits for user input.
     void this.promptGitHubRepositoryDownload().catch((e: unknown) => {
@@ -31,6 +67,10 @@ export class GithubDatabaseModule extends DisposableObject {
   }
 
   private async promptGitHubRepositoryDownload(): Promise<void> {
+    if (this.config.download === "never") {
+      return;
+    }
+
     const githubRepositoryResult = await findGitHubRepositoryForWorkspace();
     if (githubRepositoryResult.isFailure) {
       void this.app.logger.log(
@@ -42,8 +82,58 @@ export class GithubDatabaseModule extends DisposableObject {
     }
 
     const githubRepository = githubRepositoryResult.value;
-    void this.app.logger.log(
-      `Found GitHub repository for workspace: '${githubRepository.owner}/${githubRepository.name}'`,
+
+    const hasExistingDatabase = this.databaseManager.databaseItems.some(
+      (db) =>
+        db.origin?.type === "github" &&
+        db.origin.repository ===
+          `${githubRepository.owner}/${githubRepository.name}`,
+    );
+    if (hasExistingDatabase) {
+      return;
+    }
+
+    const credentials = isCanary() ? this.app.credentials : undefined;
+
+    const octokit = credentials
+      ? await credentials.getOctokit()
+      : new AppOctokit();
+
+    let databases: CodeqlDatabase[];
+    try {
+      databases = await findGitHubDatabasesForRepository(
+        octokit,
+        githubRepository.owner,
+        githubRepository.name,
+      );
+    } catch (e) {
+      this.app.telemetry?.sendError(
+        redactableError(
+          asError(e),
+        )`Failed to prompt for GitHub database download`,
+      );
+
+      void this.app.logger.log(
+        `Failed to find GitHub databases for repository: ${getErrorMessage(e)}`,
+      );
+
+      return;
+    }
+
+    if (databases.length === 0) {
+      return;
+    }
+
+    await promptAndDownloadGitHubDatabase(
+      octokit,
+      githubRepository.owner,
+      githubRepository.name,
+      databases,
+      this.config,
+      this.databaseManager,
+      this.databaseStoragePath,
+      this.cliServer,
+      this.app.commands,
     );
   }
 }

extensions/ql-vscode/src/databases/github-database-prompt.ts

@@ -0,0 +1,131 @@
+import { window } from "vscode";
+import { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods";
+import { Octokit } from "@octokit/rest";
+import { showNeverAskAgainDialog } from "../common/vscode/dialog";
+import { getLanguageDisplayName } from "../common/query-language";
+import {
+  downloadGitHubDatabaseFromUrl,
+  promptForLanguage,
+} from "./database-fetcher";
+import { withProgress } from "../common/vscode/progress";
+import { DatabaseManager } from "./local-databases";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { AppCommandManager } from "../common/commands";
+import { GitHubDatabaseConfig } from "../config";
+
+export type CodeqlDatabase =
+  RestEndpointMethodTypes["codeScanning"]["listCodeqlDatabases"]["response"]["data"][number];
+
+export async function findGitHubDatabasesForRepository(
+  octokit: Octokit,
+  owner: string,
+  repo: string,
+): Promise<CodeqlDatabase[]> {
+  const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+    owner,
+    repo,
+  });
+
+  return response.data;
+}
+
+/**
+ * Prompt the user to download a database from GitHub and download that database.
+ */
+export async function promptAndDownloadGitHubDatabase(
+  octokit: Octokit,
+  owner: string,
+  repo: string,
+  databases: CodeqlDatabase[],
+  config: GitHubDatabaseConfig,
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  cliServer: CodeQLCliServer,
+  commandManager: AppCommandManager,
+): Promise<void> {
+  const languages = databases.map((database) => database.language);
+
+  const message =
+    databases.length === 1
+      ? `This repository has an origin (GitHub) that has a ${getLanguageDisplayName(
+          languages[0],
+        )} CodeQL database. Connect to GitHub and download the existing database?`
+      : `This repository has an origin (GitHub) that has ${joinLanguages(
+          languages,
+        )} CodeQL databases. Connect to GitHub and download any existing databases?`;
+
+  const answer = await showNeverAskAgainDialog(
+    message,
+    false,
+    "Connect",
+    "Not now",
+    "Never",
+  );
+
+  if (answer === "Not now" || answer === undefined) {
+    return;
+  }
+
+  if (answer === "Never") {
+    await config.setDownload("never");
+    return;
+  }
+
+  const language = await promptForLanguage(languages, undefined);
+  if (!language) {
+    return;
+  }
+
+  const database = databases.find((database) => database.language === language);
+  if (!database) {
+    return;
+  }
+
+  await withProgress(async (progress) => {
+    await downloadGitHubDatabaseFromUrl(
+      database.url,
+      database.id,
+      database.created_at,
+      database.commit_oid ?? null,
+      owner,
+      repo,
+      octokit,
+      progress,
+      databaseManager,
+      storagePath,
+      cliServer,
+      true,
+      false,
+    );
+
+    await commandManager.execute("codeQLDatabases.focus");
+    void window.showInformationMessage(
+      `Downloaded ${getLanguageDisplayName(language)} database from GitHub.`,
+    );
+  });
+}
+
+/**
+ * Join languages into a string for display. Will automatically add `,` and `and` as appropriate.
+ *
+ * @param languages The languages to join. These should be language identifiers, such as `csharp`.
+ */
+function joinLanguages(languages: string[]): string {
+  const languageDisplayNames = languages
+    .map((language) => getLanguageDisplayName(language))
+    .sort();
+
+  let result = "";
+  for (let i = 0; i < languageDisplayNames.length; i++) {
+    if (i > 0) {
+      if (i === languageDisplayNames.length - 1) {
+        result += " and ";
+      } else {
+        result += ", ";
+      }
+    }
+    result += languageDisplayNames[i];
+  }
+
+  return result;
+}

extensions/ql-vscode/src/extension.ts

@@ -871,7 +871,12 @@ async function activateWithInstalledDistribution(
     ),
   );
 
-  await GithubDatabaseModule.initialize(app);
+  await GithubDatabaseModule.initialize(
+    app,
+    dbm,
+    getContextStoragePath(ctx),
+    cliServer,
+  );
 
   void extLogger.log("Initializing query history.");
   const queryHistoryDirs: QueryHistoryDirs = {

extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-database-prompt.test.ts

@@ -0,0 +1,267 @@
+import { faker } from "@faker-js/faker";
+import { Octokit } from "@octokit/rest";
+import { mockedObject } from "../../utils/mocking.helpers";
+import {
+  CodeqlDatabase,
+  promptAndDownloadGitHubDatabase,
+} from "../../../../src/databases/github-database-prompt";
+import { DatabaseManager } from "../../../../src/databases/local-databases";
+import { GitHubDatabaseConfig } from "../../../../src/config";
+import { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
+import { createMockCommandManager } from "../../../__mocks__/commandsMock";
+import * as databaseFetcher from "../../../../src/databases/database-fetcher";
+import * as dialog from "../../../../src/common/vscode/dialog";
+
+describe("promptAndDownloadGitHubDatabase", () => {
+  let octokit: Octokit;
+  const owner = "github";
+  const repo = "codeql";
+  let databaseManager: DatabaseManager;
+  const setDownload = jest.fn();
+  let config: GitHubDatabaseConfig;
+  const storagePath = "/a/b/c/d";
+  let cliServer: CodeQLCliServer;
+  const commandManager = createMockCommandManager();
+
+  let databases = [
+    mockedObject<CodeqlDatabase>({
+      id: faker.number.int(),
+      created_at: faker.date.past().toISOString(),
+      commit_oid: faker.git.commitSha(),
+      language: "swift",
+      url: faker.internet.url({
+        protocol: "https",
+      }),
+    }),
+  ];
+
+  let showNeverAskAgainDialogSpy: jest.SpiedFunction<
+    typeof dialog.showNeverAskAgainDialog
+  >;
+  let promptForLanguageSpy: jest.SpiedFunction<
+    typeof databaseFetcher.promptForLanguage
+  >;
+  let downloadGitHubDatabaseFromUrlSpy: jest.SpiedFunction<
+    typeof databaseFetcher.downloadGitHubDatabaseFromUrl
+  >;
+
+  beforeEach(() => {
+    octokit = mockedObject<Octokit>({});
+    databaseManager = mockedObject<DatabaseManager>({});
+    config = mockedObject<GitHubDatabaseConfig>({
+      setDownload,
+    });
+    cliServer = mockedObject<CodeQLCliServer>({});
+
+    showNeverAskAgainDialogSpy = jest
+      .spyOn(dialog, "showNeverAskAgainDialog")
+      .mockResolvedValue("Connect");
+    promptForLanguageSpy = jest
+      .spyOn(databaseFetcher, "promptForLanguage")
+      .mockResolvedValue(databases[0].language);
+    downloadGitHubDatabaseFromUrlSpy = jest
+      .spyOn(databaseFetcher, "downloadGitHubDatabaseFromUrl")
+      .mockResolvedValue(undefined);
+  });
+
+  it("downloads the database", async () => {
+    await promptAndDownloadGitHubDatabase(
+      octokit,
+      owner,
+      repo,
+      databases,
+      config,
+      databaseManager,
+      storagePath,
+      cliServer,
+      commandManager,
+    );
+
+    expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1);
+    expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
+      databases[0].url,
+      databases[0].id,
+      databases[0].created_at,
+      databases[0].commit_oid,
+      owner,
+      repo,
+      octokit,
+      expect.anything(),
+      databaseManager,
+      storagePath,
+      cliServer,
+      true,
+      false,
+    );
+    expect(promptForLanguageSpy).toHaveBeenCalledWith(["swift"], undefined);
+    expect(config.setDownload).not.toHaveBeenCalled();
+  });
+
+  describe("when answering not now to prompt", () => {
+    beforeEach(() => {
+      showNeverAskAgainDialogSpy.mockResolvedValue("Not now");
+    });
+
+    it("does not download the database", async () => {
+      await promptAndDownloadGitHubDatabase(
+        octokit,
+        owner,
+        repo,
+        databases,
+        config,
+        databaseManager,
+        storagePath,
+        cliServer,
+        commandManager,
+      );
+
+      expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("when cancelling prompt", () => {
+    beforeEach(() => {
+      showNeverAskAgainDialogSpy.mockResolvedValue(undefined);
+    });
+
+    it("does not download the database", async () => {
+      await promptAndDownloadGitHubDatabase(
+        octokit,
+        owner,
+        repo,
+        databases,
+        config,
+        databaseManager,
+        storagePath,
+        cliServer,
+        commandManager,
+      );
+
+      expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("when answering never to prompt", () => {
+    beforeEach(() => {
+      showNeverAskAgainDialogSpy.mockResolvedValue("Never");
+    });
+
+    it("does not download the database", async () => {
+      await promptAndDownloadGitHubDatabase(
+        octokit,
+        owner,
+        repo,
+        databases,
+        config,
+        databaseManager,
+        storagePath,
+        cliServer,
+        commandManager,
+      );
+
+      expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled();
+    });
+
+    it('sets the config to "never"', async () => {
+      await promptAndDownloadGitHubDatabase(
+        octokit,
+        owner,
+        repo,
+        databases,
+        config,
+        databaseManager,
+        storagePath,
+        cliServer,
+        commandManager,
+      );
+
+      expect(config.setDownload).toHaveBeenCalledTimes(1);
+      expect(config.setDownload).toHaveBeenCalledWith("never");
+    });
+  });
+
+  describe("when not selecting language", () => {
+    beforeEach(() => {
+      promptForLanguageSpy.mockResolvedValue(undefined);
+    });
+
+    it("does not download the database", async () => {
+      await promptAndDownloadGitHubDatabase(
+        octokit,
+        owner,
+        repo,
+        databases,
+        config,
+        databaseManager,
+        storagePath,
+        cliServer,
+        commandManager,
+      );
+
+      expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("when there are multiple languages", () => {
+    beforeEach(() => {
+      databases = [
+        mockedObject<CodeqlDatabase>({
+          id: faker.number.int(),
+          created_at: faker.date.past().toISOString(),
+          commit_oid: faker.git.commitSha(),
+          language: "swift",
+          url: faker.internet.url({
+            protocol: "https",
+          }),
+        }),
+        mockedObject<CodeqlDatabase>({
+          id: faker.number.int(),
+          created_at: faker.date.past().toISOString(),
+          commit_oid: null,
+          language: "go",
+          url: faker.internet.url({
+            protocol: "https",
+          }),
+        }),
+      ];
+
+      promptForLanguageSpy.mockResolvedValue(databases[1].language);
+    });
+
+    it("downloads the correct database", async () => {
+      await promptAndDownloadGitHubDatabase(
+        octokit,
+        owner,
+        repo,
+        databases,
+        config,
+        databaseManager,
+        storagePath,
+        cliServer,
+        commandManager,
+      );
+
+      expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1);
+      expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
+        databases[1].url,
+        databases[1].id,
+        databases[1].created_at,
+        databases[1].commit_oid,
+        owner,
+        repo,
+        octokit,
+        expect.anything(),
+        databaseManager,
+        storagePath,
+        cliServer,
+        true,
+        false,
+      );
+      expect(promptForLanguageSpy).toHaveBeenCalledWith(
+        ["swift", "go"],
+        undefined,
+      );
+      expect(config.setDownload).not.toHaveBeenCalled();
+    });
+  });
+});

extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts

@@ -53,6 +53,31 @@ export function mockedObject<T extends object>(
         return undefined;
       }
 
+      // The `$$typeof` is accessed by jest to check if the object is a React element.
+      // We don't want to throw an error when this happens.
+      if (prop === "$$typeof") {
+        return undefined;
+      }
+
+      // The `nodeType` and `tagName` are accessed by jest to check if the object is a DOM node.
+      // We don't want to throw an error when this happens.
+      if (prop === "nodeType" || prop === "tagName") {
+        return undefined;
+      }
+
+      // The `@@__IMMUTABLE_ITERABLE__@@` and variants are accessed by jest to check if the object is an
+      // immutable object (from Immutable.js).
+      // We don't want to throw an error when this happens.
+      if (prop.toString().startsWith("@@__IMMUTABLE_")) {
+        return undefined;
+      }
+
+      // The `Symbol.toStringTag` is accessed by jest.
+      // We don't want to throw an error when this happens.
+      if (prop === Symbol.toStringTag) {
+        return "MockedObject";
+      }
+
       throw new Error(`Method ${String(prop)} not mocked`);
     },
   });