diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 270c61d60..eca78aced 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -421,8 +421,33 @@ }, { "type": "object", - "title": "Log insights", + "title": "GitHub Databases", "order": 8, + "properties": { + "codeQL.githubDatabase.enable": { + "type": "boolean", + "default": false, + "markdownDescription": "Enable automatic detection of GitHub databases." + }, + "codeQL.githubDatabase.download": { + "type": "string", + "default": "ask", + "enum": [ + "ask", + "never" + ], + "enumDescriptions": [ + "Ask to download a GitHub database when a workspace is opened.", + "Never download a GitHub databases when a workspace is opened." + ], + "description": "Ask to download a GitHub database when a workspace is opened." + } + } + }, + { + "type": "object", + "title": "Log insights", + "order": 9, "properties": { "codeQL.logInsights.joinOrderWarningThreshold": { "type": "number", @@ -436,7 +461,7 @@ { "type": "object", "title": "Telemetry", - "order": 9, + "order": 10, "properties": { "codeQL.telemetry.enableTelemetry": { "type": "boolean", diff --git a/extensions/ql-vscode/src/config.ts b/extensions/ql-vscode/src/config.ts index f672f0ed2..7a9170d6d 100644 --- a/extensions/ql-vscode/src/config.ts +++ b/extensions/ql-vscode/src/config.ts @@ -1,19 +1,19 @@ import { DisposableObject } from "./common/disposable-object"; import { - workspace, + ConfigurationChangeEvent, + ConfigurationScope, + ConfigurationTarget, Event, EventEmitter, - ConfigurationChangeEvent, - ConfigurationTarget, - ConfigurationScope, + workspace, } from "vscode"; import { DistributionManager } from "./codeql-cli/distribution"; import { extLogger } from "./common/logging/vscode"; import { ONE_DAY_IN_MS } from "./common/time"; import { + defaultFilterSortState, FilterKey, SortKey, - defaultFilterSortState, } from "./variant-analysis/shared/variant-analysis-filter-sort"; export const ALL_SETTINGS: Setting[] = []; @@ -775,3 +775,52 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig { return !!ENABLE_RUBY.getValue(); } } + +const GITHUB_DATABASE_SETTING = new Setting("githubDatabase", ROOT_SETTING); + +// Feature flag for the GitHub database downnload. +const GITHUB_DATABASE_ENABLE = new Setting("enable", GITHUB_DATABASE_SETTING); +const GITHUB_DATABASE_DOWNLOAD = new Setting( + "download", + GITHUB_DATABASE_SETTING, +); + +const GitHubDatabaseDownloadValues = ["ask", "never"] as const; +type GitHubDatabaseDownload = (typeof GitHubDatabaseDownloadValues)[number]; + +export interface GitHubDatabaseConfig { + enable: boolean; + download: GitHubDatabaseDownload; + setDownload( + value: GitHubDatabaseDownload, + target?: ConfigurationTarget, + ): Promise; +} + +export class GitHubDatabaseConfigListener + extends ConfigListener + implements GitHubDatabaseConfig +{ + protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void { + this.handleDidChangeConfigurationForRelevantSettings( + [GITHUB_DATABASE_SETTING], + e, + ); + } + + public get enable() { + return !!GITHUB_DATABASE_ENABLE.getValue(); + } + + public get download(): GitHubDatabaseDownload { + const value = GITHUB_DATABASE_DOWNLOAD.getValue(); + return GitHubDatabaseDownloadValues.includes(value) ? value : "ask"; + } + + public async setDownload( + value: GitHubDatabaseDownload, + target: ConfigurationTarget = ConfigurationTarget.Workspace, + ): Promise { + await GITHUB_DATABASE_DOWNLOAD.updateValue(value, target); + } +} diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index 3a8e22903..e84d7f994 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -207,6 +207,38 @@ export async function downloadGitHubDatabase( const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } = result; + return downloadGitHubDatabaseFromUrl( + databaseUrl, + databaseId, + databaseCreatedAt, + commitOid, + owner, + name, + octokit, + progress, + databaseManager, + storagePath, + cli, + makeSelected, + addSourceArchiveFolder, + ); +} + +export async function downloadGitHubDatabaseFromUrl( + databaseUrl: string, + databaseId: number, + databaseCreatedAt: string, + commitOid: string | null, + owner: string, + name: string, + octokit: Octokit.Octokit, + progress: ProgressCallback, + databaseManager: DatabaseManager, + storagePath: string, + cli?: CodeQLCliServer, + makeSelected = true, + addSourceArchiveFolder = true, +): Promise { /** * The 'token' property of the token object returned by `octokit.auth()`. * The object is undocumented, but looks something like this: @@ -229,7 +261,7 @@ export async function downloadGitHubDatabase( `${owner}/${name}`, { type: "github", - repository: nwo, + repository: `${owner}/${name}`, databaseId, databaseCreatedAt, commitOid, @@ -577,7 +609,7 @@ export async function convertGithubNwoToDatabaseUrl( } const databaseForLanguage = response.data.find( - (db: any) => db.language === language, + (db) => db.language === language, ); if (!databaseForLanguage) { throw new Error(`No database found for language '${language}'`); @@ -599,9 +631,9 @@ export async function convertGithubNwoToDatabaseUrl( export async function promptForLanguage( languages: string[], - progress: ProgressCallback, + progress: ProgressCallback | undefined, ): Promise { - progress({ + progress?.({ message: "Choose language", step: 2, maxStep: 2, diff --git a/extensions/ql-vscode/src/databases/github-database-module.ts b/extensions/ql-vscode/src/databases/github-database-module.ts index 57852cc35..c709360fd 100644 --- a/extensions/ql-vscode/src/databases/github-database-module.ts +++ b/extensions/ql-vscode/src/databases/github-database-module.ts @@ -2,15 +2,47 @@ import { DisposableObject } from "../common/disposable-object"; import { App } from "../common/app"; import { findGitHubRepositoryForWorkspace } from "./github-repository-finder"; import { redactableError } from "../common/errors"; -import { asError } from "../common/helpers-pure"; +import { asError, getErrorMessage } from "../common/helpers-pure"; +import { + CodeqlDatabase, + findGitHubDatabasesForRepository, + promptGitHubDatabaseDownload, +} from "./github-database-prompt"; +import { + GitHubDatabaseConfig, + GitHubDatabaseConfigListener, + isCanary, +} from "../config"; +import { AppOctokit } from "../common/octokit"; +import { DatabaseManager } from "./local-databases"; +import { CodeQLCliServer } from "../codeql-cli/cli"; export class GithubDatabaseModule extends DisposableObject { - private constructor(private readonly app: App) { + private readonly config: GitHubDatabaseConfig; + + private constructor( + private readonly app: App, + private readonly databaseManager: DatabaseManager, + private readonly databaseStoragePath: string, + private readonly cliServer: CodeQLCliServer, + ) { super(); + + this.config = this.push(new GitHubDatabaseConfigListener()); } - public static async initialize(app: App): Promise { - const githubDatabaseModule = new GithubDatabaseModule(app); + public static async initialize( + app: App, + databaseManager: DatabaseManager, + databaseStoragePath: string, + cliServer: CodeQLCliServer, + ): Promise { + const githubDatabaseModule = new GithubDatabaseModule( + app, + databaseManager, + databaseStoragePath, + cliServer, + ); app.subscriptions.push(githubDatabaseModule); await githubDatabaseModule.initialize(); @@ -18,6 +50,10 @@ export class GithubDatabaseModule extends DisposableObject { } private async initialize(): Promise { + if (!this.config.enable) { + return; + } + // Start the check and downloading the database asynchronously. We don't want to block on this // in extension activation since this makes network requests and waits for user input. void this.promptGitHubRepositoryDownload().catch((e: unknown) => { @@ -31,6 +67,10 @@ export class GithubDatabaseModule extends DisposableObject { } private async promptGitHubRepositoryDownload(): Promise { + if (this.config.download === "never") { + return; + } + const githubRepositoryResult = await findGitHubRepositoryForWorkspace(); if (githubRepositoryResult.isFailure) { void this.app.logger.log( @@ -42,8 +82,58 @@ export class GithubDatabaseModule extends DisposableObject { } const githubRepository = githubRepositoryResult.value; - void this.app.logger.log( - `Found GitHub repository for workspace: '${githubRepository.owner}/${githubRepository.name}'`, + + const hasExistingDatabase = this.databaseManager.databaseItems.some( + (db) => + db.origin?.type === "github" && + db.origin.repository === + `${githubRepository.owner}/${githubRepository.name}`, + ); + if (hasExistingDatabase) { + return; + } + + const credentials = isCanary() ? this.app.credentials : undefined; + + const octokit = credentials + ? await credentials.getOctokit() + : new AppOctokit(); + + let databases: CodeqlDatabase[]; + try { + databases = await findGitHubDatabasesForRepository( + octokit, + githubRepository.owner, + githubRepository.name, + ); + } catch (e) { + this.app.telemetry?.sendError( + redactableError( + asError(e), + )`Failed to prompt for GitHub database download`, + ); + + void this.app.logger.log( + `Failed to find GitHub databases for repository: ${getErrorMessage(e)}`, + ); + + return; + } + + if (databases.length === 0) { + return; + } + + void promptGitHubDatabaseDownload( + octokit, + githubRepository.owner, + githubRepository.name, + databases, + this.config, + this.databaseManager, + this.databaseStoragePath, + this.cliServer, + this.app.commands, ); } } diff --git a/extensions/ql-vscode/src/databases/github-database-prompt.ts b/extensions/ql-vscode/src/databases/github-database-prompt.ts new file mode 100644 index 000000000..f5cd79450 --- /dev/null +++ b/extensions/ql-vscode/src/databases/github-database-prompt.ts @@ -0,0 +1,137 @@ +import { window } from "vscode"; +import { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods"; +import { Octokit } from "@octokit/rest"; +import { showNeverAskAgainDialog } from "../common/vscode/dialog"; +import { getLanguageDisplayName } from "../common/query-language"; +import { + downloadGitHubDatabaseFromUrl, + promptForLanguage, +} from "./database-fetcher"; +import { withProgress } from "../common/vscode/progress"; +import { DatabaseManager } from "./local-databases"; +import { CodeQLCliServer } from "../codeql-cli/cli"; +import { AppCommandManager } from "../common/commands"; +import { GitHubDatabaseConfig } from "../config"; + +export type CodeqlDatabase = + RestEndpointMethodTypes["codeScanning"]["listCodeqlDatabases"]["response"]["data"][number]; + +export async function findGitHubDatabasesForRepository( + octokit: Octokit, + owner: string, + repo: string, +): Promise { + const response = await octokit.rest.codeScanning.listCodeqlDatabases({ + owner, + repo, + }); + + return response.data; +} + +/** + * Prompt the user to download a database from GitHub. This is a blocking method, so this should + * almost never be called with `await`. + */ +export async function promptGitHubDatabaseDownload( + octokit: Octokit, + owner: string, + repo: string, + databases: CodeqlDatabase[], + config: GitHubDatabaseConfig, + databaseManager: DatabaseManager, + storagePath: string, + cliServer: CodeQLCliServer, + commandManager: AppCommandManager, +): Promise { + const languages = databases.map((database) => database.language); + + const databasesMessage = + databases.length === 1 + ? `This repository has an origin (GitHub) that has a ${getLanguageDisplayName( + languages[0], + )} CodeQL database.` + : `This repository has an origin (GitHub) that has ${joinLanguages( + languages, + )} CodeQL databases.`; + + const connectMessage = + databases.length === 1 + ? `Connect to GitHub and download the existing database?` + : `Connect to GitHub and download any existing databases?`; + + const answer = await showNeverAskAgainDialog( + `${databasesMessage} ${connectMessage}`, + false, + "Connect", + "Not now", + "Never", + ); + + if (answer === "Not now" || answer === undefined) { + return; + } + + if (answer === "Never") { + await config.setDownload("never"); + return; + } + + const language = await promptForLanguage(languages, undefined); + if (!language) { + return; + } + + const database = databases.find((database) => database.language === language); + if (!database) { + return; + } + + await withProgress(async (progress) => { + await downloadGitHubDatabaseFromUrl( + database.url, + database.id, + database.created_at, + database.commit_oid ?? null, + owner, + repo, + octokit, + progress, + databaseManager, + storagePath, + cliServer, + true, + false, + ); + + await commandManager.execute("codeQLDatabases.focus"); + void window.showInformationMessage( + `Downloaded ${getLanguageDisplayName(language)} database from GitHub.`, + ); + }); +} + +/** + * Join languages into a string for display. Will automatically add `,` and `and` as appropriate. + * + * @param languages The languages to join. These should be language identifiers, such as `csharp`. + */ +function joinLanguages(languages: string[]): string { + const languageDisplayNames = languages + .map((language) => getLanguageDisplayName(language)) + .sort(); + + let result = ""; + for (let i = 0; i < languageDisplayNames.length; i++) { + if (i > 0) { + if (i === languageDisplayNames.length - 1) { + result += " and "; + } else { + result += ", "; + } + } + result += languageDisplayNames[i]; + } + + return result; +} diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index da37ca0b8..5f5c5c470 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -871,7 +871,12 @@ async function activateWithInstalledDistribution( ), ); - await GithubDatabaseModule.initialize(app); + await GithubDatabaseModule.initialize( + app, + dbm, + getContextStoragePath(ctx), + cliServer, + ); void extLogger.log("Initializing query history."); const queryHistoryDirs: QueryHistoryDirs = { diff --git a/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-database-prompt.test.ts b/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-database-prompt.test.ts new file mode 100644 index 000000000..5790ef803 --- /dev/null +++ b/extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-database-prompt.test.ts @@ -0,0 +1,267 @@ +import { faker } from "@faker-js/faker"; +import { Octokit } from "@octokit/rest"; +import { mockedObject } from "../../utils/mocking.helpers"; +import { + CodeqlDatabase, + promptGitHubDatabaseDownload, +} from "../../../../src/databases/github-database-prompt"; +import { DatabaseManager } from "../../../../src/databases/local-databases"; +import { GitHubDatabaseConfig } from "../../../../src/config"; +import { CodeQLCliServer } from "../../../../src/codeql-cli/cli"; +import { createMockCommandManager } from "../../../__mocks__/commandsMock"; +import * as databaseFetcher from "../../../../src/databases/database-fetcher"; +import * as dialog from "../../../../src/common/vscode/dialog"; + +describe("promptGitHubDatabaseDownload", () => { + let octokit: Octokit; + const owner = "github"; + const repo = "codeql"; + let databaseManager: DatabaseManager; + const setDownload = jest.fn(); + let config: GitHubDatabaseConfig; + const storagePath = "/a/b/c/d"; + let cliServer: CodeQLCliServer; + const commandManager = createMockCommandManager(); + + let databases = [ + mockedObject({ + id: faker.number.int(), + created_at: faker.date.past().toISOString(), + commit_oid: faker.git.commitSha(), + language: "swift", + url: faker.internet.url({ + protocol: "https", + }), + }), + ]; + + let showNeverAskAgainDialogSpy: jest.SpiedFunction< + typeof dialog.showNeverAskAgainDialog + >; + let promptForLanguageSpy: jest.SpiedFunction< + typeof databaseFetcher.promptForLanguage + >; + let downloadGitHubDatabaseFromUrlSpy: jest.SpiedFunction< + typeof databaseFetcher.downloadGitHubDatabaseFromUrl + >; + + beforeEach(() => { + octokit = mockedObject({}); + databaseManager = mockedObject({}); + config = mockedObject({ + setDownload, + }); + cliServer = mockedObject({}); + + showNeverAskAgainDialogSpy = jest + .spyOn(dialog, "showNeverAskAgainDialog") + .mockResolvedValue("Connect"); + promptForLanguageSpy = jest + .spyOn(databaseFetcher, "promptForLanguage") + .mockResolvedValue(databases[0].language); + downloadGitHubDatabaseFromUrlSpy = jest + .spyOn(databaseFetcher, "downloadGitHubDatabaseFromUrl") + .mockResolvedValue(undefined); + }); + + it("downloads the database", async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1); + expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith( + databases[0].url, + databases[0].id, + databases[0].created_at, + databases[0].commit_oid, + owner, + repo, + octokit, + expect.anything(), + databaseManager, + storagePath, + cliServer, + true, + false, + ); + expect(promptForLanguageSpy).toHaveBeenCalledWith(["swift"], undefined); + expect(config.setDownload).not.toHaveBeenCalled(); + }); + + describe("when answering not now to prompt", () => { + beforeEach(() => { + showNeverAskAgainDialogSpy.mockResolvedValue("Not now"); + }); + + it("does not download the database", async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled(); + }); + }); + + describe("when cancelling prompt", () => { + beforeEach(() => { + showNeverAskAgainDialogSpy.mockResolvedValue(undefined); + }); + + it("does not download the database", async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled(); + }); + }); + + describe("when answering never to prompt", () => { + beforeEach(() => { + showNeverAskAgainDialogSpy.mockResolvedValue("Never"); + }); + + it("does not download the database", async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled(); + }); + + it('sets the config to "never"', async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(config.setDownload).toHaveBeenCalledTimes(1); + expect(config.setDownload).toHaveBeenCalledWith("never"); + }); + }); + + describe("when not selecting language", () => { + beforeEach(() => { + promptForLanguageSpy.mockResolvedValue(undefined); + }); + + it("does not download the database", async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled(); + }); + }); + + describe("when there are multiple languages", () => { + beforeEach(() => { + databases = [ + mockedObject({ + id: faker.number.int(), + created_at: faker.date.past().toISOString(), + commit_oid: faker.git.commitSha(), + language: "swift", + url: faker.internet.url({ + protocol: "https", + }), + }), + mockedObject({ + id: faker.number.int(), + created_at: faker.date.past().toISOString(), + commit_oid: null, + language: "go", + url: faker.internet.url({ + protocol: "https", + }), + }), + ]; + + promptForLanguageSpy.mockResolvedValue(databases[1].language); + }); + + it("downloads the correct database", async () => { + await promptGitHubDatabaseDownload( + octokit, + owner, + repo, + databases, + config, + databaseManager, + storagePath, + cliServer, + commandManager, + ); + + expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1); + expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith( + databases[1].url, + databases[1].id, + databases[1].created_at, + databases[1].commit_oid, + owner, + repo, + octokit, + expect.anything(), + databaseManager, + storagePath, + cliServer, + true, + false, + ); + expect(promptForLanguageSpy).toHaveBeenCalledWith( + ["swift", "go"], + undefined, + ); + expect(config.setDownload).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts b/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts index bb826293d..2b10a0bcc 100644 --- a/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts +++ b/extensions/ql-vscode/test/vscode-tests/utils/mocking.helpers.ts @@ -53,6 +53,31 @@ export function mockedObject( return undefined; } + // The `$$typeof` is accessed by jest to check if the object is a React element. + // We don't want to throw an error when this happens. + if (prop === "$$typeof") { + return undefined; + } + + // The `nodeType` and `tagName` are accessed by jest to check if the object is a DOM node. + // We don't want to throw an error when this happens. + if (prop === "nodeType" || prop === "tagName") { + return undefined; + } + + // The `@@__IMMUTABLE_ITERABLE__@@` and variants are accessed by jest to check if the object is an + // immutable object (from Immutable.js). + // We don't want to throw an error when this happens. + if (prop.toString().startsWith("@@__IMMUTABLE_")) { + return undefined; + } + + // The `Symbol.toStringTag` is accessed by jest. + // We don't want to throw an error when this happens. + if (prop === Symbol.toStringTag) { + return "MockedObject"; + } + throw new Error(`Method ${String(prop)} not mocked`); }, });