mrva/vscode-codeql
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #3458 from github/robertbrignull/database-prompting

Browse Source 
Refactor database-fetcher.ts into a class and simplify
This commit is contained in:
Robert
2024-03-28 18:05:54 +00:00
committed by GitHub
parent 6f7b7cf84f 7d259a9322
commit a5239fbede
20 changed files with 753 additions and 912 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
extensions/ql-vscode/src/databases/database-fetcher.ts
View File

@@ -1,13 +1,11 @@
import type { Response } from "node-fetch";
import fetch, { AbortError } from "node-fetch";
import { zip } from "zip-a-folder";
import type { InputBoxOptions } from "vscode";
import { Uri, window } from "vscode";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import {
ensureDir,
realpath as fs_realpath,
pathExists,
createWriteStream,
remove,
readdir,
@@ -27,7 +25,6 @@ import {
getNwoFromGitHubUrl,
isValidGitHubNwo,
} from "../common/github-url-identifier-helper";
import type { AppCommandManager } from "../common/commands";
import {
addDatabaseSourceToWorkspace,
allowHttp,
@@ -42,19 +39,31 @@ import type { App } from "../common/app";
import { createFilenameFromString } from "../common/filenames";
import { findDirWithFile } from "../common/files";
import { convertGithubNwoToDatabaseUrl } from "./github-databases/api";
import { ensureZippedSourceLocation } from "./local-databases/database-contents";
// The number of tries to use when generating a unique filename before
// giving up and using a nanoid.
const DUPLICATE_FILENAMES_TRIES = 10_000;
export class DatabaseFetcher {
/**
* @param app the App
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param cli the CodeQL CLI server
**/
constructor(
private readonly app: App,
private readonly databaseManager: DatabaseManager,
private readonly storagePath: string,
private readonly cli: CodeQLCliServer,
) {}
/**
* Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
*
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
*/
export async function promptImportInternetDatabase(
commandManager: AppCommandManager,
databaseManager: DatabaseManager,
storagePath: string,
public async promptImportInternetDatabase(
progress: ProgressCallback,
cli: CodeQLCliServer,
): Promise<DatabaseItem | undefined> {
const databaseUrl = await window.showInputBox({
prompt: "Enter URL of zipfile of database to download",
@@ -63,24 +72,21 @@ export async function promptImportInternetDatabase(
return;
}
validateUrl(databaseUrl);
this.validateUrl(databaseUrl);
const item = await fetchDatabaseToWorkspaceStorage(
const item = await this.fetchDatabaseToWorkspaceStorage(
databaseUrl,
{},
databaseManager,
storagePath,
undefined,
{
type: "url",
url: databaseUrl,
},
progress,
cli,
);
if (item) {
await commandManager.execute("codeQLDatabases.focus");
await this.app.commands.execute("codeQLDatabases.focus");
void showAndLogInformationMessage(
extLogger,
"Database downloaded and imported successfully.",
@@ -94,37 +100,27 @@ export async function promptImportInternetDatabase(
* User enters a GitHub repository and then the user is asked which language
* to download (if there is more than one)
*
* @param app the App
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param progress the progress callback
* @param cli the CodeQL CLI server
* @param language the language to download. If undefined, the user will be prompted to choose a language.
* @param suggestedRepoNwo the suggested value to use when prompting for a github repo
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
*/
export async function promptImportGithubDatabase(
app: App,
databaseManager: DatabaseManager,
storagePath: string,
public async promptImportGithubDatabase(
progress: ProgressCallback,
cli: CodeQLCliServer,
language?: string,
suggestedRepoNwo?: string,
makeSelected = true,
addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
): Promise<DatabaseItem | undefined> {
const githubRepo = await askForGitHubRepo(progress);
const githubRepo = await this.askForGitHubRepo(progress, suggestedRepoNwo);
if (!githubRepo) {
return;
}
const databaseItem = await downloadGitHubDatabase(
const databaseItem = await this.downloadGitHubDatabase(
githubRepo,
app,
databaseManager,
storagePath,
progress,
cli,
language,
makeSelected,
addSourceArchiveFolder,
@@ -132,7 +128,7 @@ export async function promptImportGithubDatabase(
if (databaseItem) {
if (makeSelected) {
await app.commands.execute("codeQLDatabases.focus");
await this.app.commands.execute("codeQLDatabases.focus");
}
void showAndLogInformationMessage(
extLogger,
@@ -144,7 +140,7 @@ export async function promptImportGithubDatabase(
return;
}
export async function askForGitHubRepo(
private async askForGitHubRepo(
progress?: ProgressCallback,
suggestedValue?: string,
): Promise<string | undefined> {
@@ -172,22 +168,14 @@ export async function askForGitHubRepo(
* Downloads a database from GitHub
*
* @param githubRepo the GitHub repository to download the database from
* @param app the App
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param progress the progress callback
* @param cli the CodeQL CLI server
* @param language the language to download. If undefined, the user will be prompted to choose a language.
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
**/
export async function downloadGitHubDatabase(
private async downloadGitHubDatabase(
githubRepo: string,
app: App,
databaseManager: DatabaseManager,
storagePath: string,
progress: ProgressCallback,
cli: CodeQLCliServer,
language?: string,
makeSelected = true,
addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
@@ -197,7 +185,7 @@ export async function downloadGitHubDatabase(
throw new Error(`Invalid GitHub repository: ${githubRepo}`);
}
const credentials = isCanary() ? app.credentials : undefined;
const credentials = isCanary() ? this.app.credentials : undefined;
const octokit = credentials
? await credentials.getOctokit()
@@ -213,10 +201,16 @@ export async function downloadGitHubDatabase(
return;
}
const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
result;
const {
databaseUrl,
name,
owner,
databaseId,
databaseCreatedAt,
commitOid,
} = result;
return downloadGitHubDatabaseFromUrl(
return this.downloadGitHubDatabaseFromUrl(
databaseUrl,
databaseId,
databaseCreatedAt,
@@ -225,15 +219,12 @@ export async function downloadGitHubDatabase(
name,
octokit,
progress,
databaseManager,
storagePath,
cli,
makeSelected,
addSourceArchiveFolder,
);
}
export async function downloadGitHubDatabaseFromUrl(
public async downloadGitHubDatabaseFromUrl(
databaseUrl: string,
databaseId: number,
databaseCreatedAt: string,
@@ -242,9 +233,6 @@ export async function downloadGitHubDatabaseFromUrl(
name: string,
octokit: Octokit,
progress: ProgressCallback,
databaseManager: DatabaseManager,
storagePath: string,
cli: CodeQLCliServer,
makeSelected = true,
addSourceArchiveFolder = true,
): Promise<DatabaseItem | undefined> {
@@ -259,14 +247,12 @@ export async function downloadGitHubDatabaseFromUrl(
* We only need the actual token string.
*/
const octokitToken = ((await octokit.auth()) as { token: string })?.token;
return await fetchDatabaseToWorkspaceStorage(
return await this.fetchDatabaseToWorkspaceStorage(
databaseUrl,
{
Accept: "application/zip",
Authorization: octokitToken ? `Bearer ${octokitToken}` : "",
},
databaseManager,
storagePath,
`${owner}/${name}`,
{
type: "github",
@@ -276,7 +262,6 @@ export async function downloadGitHubDatabaseFromUrl(
commitOid,
},
progress,
cli,
makeSelected,
addSourceArchiveFolder,
);
@@ -287,35 +272,26 @@ export async function downloadGitHubDatabaseFromUrl(
* ending with `.testproj`.
*
* @param databaseUrl the file url of the archive or directory to import
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param cli the CodeQL CLI server
* @param progress the progress callback
*/
export async function importLocalDatabase(
commandManager: AppCommandManager,
public async importLocalDatabase(
databaseUrl: string,
databaseManager: DatabaseManager,
storagePath: string,
progress: ProgressCallback,
cli: CodeQLCliServer,
): Promise<DatabaseItem | undefined> {
try {
const origin: DatabaseOrigin = {
type: databaseUrl.endsWith(".testproj") ? "testproj" : "archive",
path: Uri.parse(databaseUrl).fsPath,
};
const item = await fetchDatabaseToWorkspaceStorage(
const item = await this.fetchDatabaseToWorkspaceStorage(
databaseUrl,
{},
databaseManager,
storagePath,
undefined,
origin,
progress,
cli,
);
if (item) {
await commandManager.execute("codeQLDatabases.focus");
await this.app.commands.execute("codeQLDatabases.focus");
void showAndLogInformationMessage(
extLogger,
origin.type === "testproj"
@@ -342,24 +318,18 @@ export async function importLocalDatabase(
*
* @param databaseUrl URL from which to grab the database. This could be a local archive file, a local directory, or a remote URL.
* @param requestHeaders Headers to send with the request
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param nameOverride a name for the database that overrides the default
* @param origin the origin of the database
* @param progress callback to send progress messages to
* @param cli the CodeQL CLI server
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
*/
async function fetchDatabaseToWorkspaceStorage(
private async fetchDatabaseToWorkspaceStorage(
databaseUrl: string,
requestHeaders: { [key: string]: string },
databaseManager: DatabaseManager,
storagePath: string,
nameOverride: string | undefined,
origin: DatabaseOrigin,
progress: ProgressCallback,
cli: CodeQLCliServer,
makeSelected = true,
addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
): Promise<DatabaseItem> {
@@ -368,24 +338,25 @@ async function fetchDatabaseToWorkspaceStorage(
step: 1,
maxStep: 4,
});
if (!storagePath) {
if (!this.storagePath) {
throw new Error("No storage path specified.");
}
await ensureDir(storagePath);
const unzipPath = await getStorageFolder(
storagePath,
databaseUrl,
nameOverride,
);
await ensureDir(this.storagePath);
const unzipPath = await this.getStorageFolder(databaseUrl, nameOverride);
if (isFile(databaseUrl)) {
if (Uri.parse(databaseUrl).scheme === "file") {
if (origin.type === "testproj") {
await copyDatabase(databaseUrl, unzipPath, progress);
await this.copyDatabase(databaseUrl, unzipPath, progress);
} else {
await readAndUnzip(databaseUrl, unzipPath, cli, progress);
await this.readAndUnzip(databaseUrl, unzipPath, progress);
}
} else {
await fetchAndUnzip(databaseUrl, requestHeaders, unzipPath, cli, progress);
await this.fetchAndUnzip(
databaseUrl,
requestHeaders,
unzipPath,
progress,
);
}
progress({
@@ -408,7 +379,7 @@ async function fetchDatabaseToWorkspaceStorage(
});
await ensureZippedSourceLocation(dbPath);
const item = await databaseManager.openDatabase(
const item = await this.databaseManager.openDatabase(
Uri.file(dbPath),
origin,
makeSelected,
@@ -424,15 +395,7 @@ async function fetchDatabaseToWorkspaceStorage(
}
}
// The number of tries to use when generating a unique filename before
// giving up and using a nanoid.
const DUPLICATE_FILENAMES_TRIES = 10_000;
async function getStorageFolder(
storagePath: string,
urlStr: string,
nameOverrride?: string,
) {
private async getStorageFolder(urlStr: string, nameOverrride?: string) {
let lastName: string;
if (nameOverrride) {
@@ -452,7 +415,7 @@ async function getStorageFolder(
}
}
const realpath = await fs_realpath(storagePath);
const realpath = await fs_realpath(this.storagePath);
let folderName = lastName;
// get all existing files instead of calling pathExists on every
@@ -482,7 +445,7 @@ async function getStorageFolder(
return join(realpath, folderName);
}
function validateUrl(databaseUrl: string) {
private validateUrl(databaseUrl: string) {
let uri;
try {
uri = Uri.parse(databaseUrl, true);
@@ -501,7 +464,7 @@ function validateUrl(databaseUrl: string) {
* @param destDir the location to copy the database to. This should be a folder in the workspace storage.
* @param progress callback to send progress messages to
*/
async function copyDatabase(
private async copyDatabase(
srcDirURL: string,
destDir: string,
progress?: ProgressCallback,
@@ -515,10 +478,9 @@ async function copyDatabase(
await copy(Uri.parse(srcDirURL).fsPath, destDir);
}
async function readAndUnzip(
private async readAndUnzip(
zipUrl: string,
unzipPath: string,
cli: CodeQLCliServer,
progress?: ProgressCallback,
) {
const zipFile = Uri.parse(zipUrl).fsPath;
@@ -528,14 +490,13 @@ async function readAndUnzip(
message: `Unzipping into ${basename(unzipPath)}`,
});
await cli.databaseUnbundle(zipFile, unzipPath);
await this.cli.databaseUnbundle(zipFile, unzipPath);
}
async function fetchAndUnzip(
private async fetchAndUnzip(
databaseUrl: string,
requestHeaders: { [key: string]: string },
unzipPath: string,
cli: CodeQLCliServer,
progress?: ProgressCallback,
) {
// Although it is possible to download and stream directly to an unzipped directory,
@@ -560,7 +521,7 @@ async function fetchAndUnzip(
let response: Response;
try {
response = await checkForFailingResponse(
response = await this.checkForFailingResponse(
await fetch(databaseUrl, {
headers: requestHeaders,
signal,
@@ -582,7 +543,9 @@ async function fetchAndUnzip(
const archiveFileStream = createWriteStream(archivePath);
const contentLength = response.headers.get("content-length");
const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
const totalNumBytes = contentLength
? parseInt(contentLength, 10)
: undefined;
reportStreamProgress(
response.body,
"Downloading database",
@@ -619,10 +582,9 @@ async function fetchAndUnzip(
disposeTimeout();
}
await readAndUnzip(
await this.readAndUnzip(
Uri.file(archivePath).toString(true),
unzipPath,
cli,
progress,
);
@@ -630,7 +592,7 @@ async function fetchAndUnzip(
await remove(archivePath);
}
async function checkForFailingResponse(
private async checkForFailingResponse(
response: Response,
errorMessage: string,
): Promise<Response | never> {
@@ -650,30 +612,4 @@ async function checkForFailingResponse(
}
throw new Error(`${errorMessage}.\n\nReason: ${msg}`);
}
function isFile(databaseUrl: string) {
return Uri.parse(databaseUrl).scheme === "file";
}
/**
* Databases created by the old odasa tool will not have a zipped
* source location. However, this extension works better if sources
* are zipped.
*
* This function ensures that the source location is zipped. If the
* `src` folder exists and the `src.zip` file does not, the `src`
* folder will be zipped and then deleted.
*
* @param databasePath The full path to the unzipped database
*/
export async function ensureZippedSourceLocation(
databasePath: string,
): Promise<void> {
const srcFolderPath = join(databasePath, "src");
const srcZipPath = `${srcFolderPath}.zip`;
if ((await pathExists(srcFolderPath)) && !(await pathExists(srcZipPath))) {
await zip(srcFolderPath, srcZipPath);
await remove(srcFolderPath);
}
}

13
extensions/ql-vscode/src/databases/github-databases/download.ts
View File

@@ -2,10 +2,8 @@ import { window } from "vscode";
import type { Octokit } from "@octokit/rest";
import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
import { getLanguageDisplayName } from "../../common/query-language";
import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
import type { DatabaseFetcher } from "../database-fetcher";
import { withProgress } from "../../common/vscode/progress";
import type { DatabaseManager } from "../local-databases";
import type { CodeQLCliServer } from "../../codeql-cli/cli";
import type { AppCommandManager } from "../../common/commands";
import type { GitHubDatabaseConfig } from "../../config";
import type { CodeqlDatabase } from "./api";
@@ -58,9 +56,7 @@ export async function downloadDatabaseFromGitHub(
owner: string,
repo: string,
databases: CodeqlDatabase[],
databaseManager: DatabaseManager,
storagePath: string,
cliServer: CodeQLCliServer,
databaseFetcher: DatabaseFetcher,
commandManager: AppCommandManager,
): Promise<void> {
const selectedDatabases = await promptForDatabases(databases);
@@ -72,7 +68,7 @@ export async function downloadDatabaseFromGitHub(
selectedDatabases.map((database) =>
withProgress(
async (progress) => {
await downloadGitHubDatabaseFromUrl(
await databaseFetcher.downloadGitHubDatabaseFromUrl(
database.url,
database.id,
database.created_at,
@@ -81,9 +77,6 @@ export async function downloadDatabaseFromGitHub(
repo,
octokit,
progress,
databaseManager,
storagePath,
cliServer,
true,
false,
);

18
extensions/ql-vscode/src/databases/github-databases/github-databases-module.ts
View File

@@ -14,7 +14,6 @@ import {
} from "./download";
import type { GitHubDatabaseConfig } from "../../config";
import type { DatabaseManager } from "../local-databases";
import type { CodeQLCliServer } from "../../codeql-cli/cli";
import type { CodeqlDatabase, ListDatabasesResult } from "./api";
import { listDatabases } from "./api";
import type { DatabaseUpdate } from "./updates";
@@ -24,6 +23,7 @@ import {
isNewerDatabaseAvailable,
} from "./updates";
import type { Octokit } from "@octokit/rest";
import type { DatabaseFetcher } from "../database-fetcher";
export class GitHubDatabasesModule extends DisposableObject {
/**
@@ -33,8 +33,7 @@ export class GitHubDatabasesModule extends DisposableObject {
constructor(
private readonly app: App,
private readonly databaseManager: DatabaseManager,
private readonly databaseStoragePath: string,
private readonly cliServer: CodeQLCliServer,
private readonly databaseFetcher: DatabaseFetcher,
private readonly config: GitHubDatabaseConfig,
) {
super();
@@ -43,15 +42,13 @@ export class GitHubDatabasesModule extends DisposableObject {
public static async initialize(
app: App,
databaseManager: DatabaseManager,
databaseStoragePath: string,
cliServer: CodeQLCliServer,
databaseFetcher: DatabaseFetcher,
config: GitHubDatabaseConfig,
): Promise<GitHubDatabasesModule> {
const githubDatabasesModule = new GitHubDatabasesModule(
app,
databaseManager,
databaseStoragePath,
cliServer,
databaseFetcher,
config,
);
app.subscriptions.push(githubDatabasesModule);
@@ -185,9 +182,7 @@ export class GitHubDatabasesModule extends DisposableObject {
owner,
repo,
databases,
this.databaseManager,
this.databaseStoragePath,
this.cliServer,
this.databaseFetcher,
this.app.commands,
);
}
@@ -212,8 +207,7 @@ export class GitHubDatabasesModule extends DisposableObject {
repo,
databaseUpdates,
this.databaseManager,
this.databaseStoragePath,
this.cliServer,
this.databaseFetcher,
this.app.commands,
);
}

12
extensions/ql-vscode/src/databases/github-databases/updates.ts
View File

@@ -1,11 +1,10 @@
import type { CodeqlDatabase } from "./api";
import type { DatabaseItem, DatabaseManager } from "../local-databases";
import type { Octokit } from "@octokit/rest";
import type { CodeQLCliServer } from "../../codeql-cli/cli";
import type { AppCommandManager } from "../../common/commands";
import { getLanguageDisplayName } from "../../common/query-language";
import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
import type { DatabaseFetcher } from "../database-fetcher";
import { withProgress } from "../../common/vscode/progress";
import { window } from "vscode";
import type { GitHubDatabaseConfig } from "../../config";
@@ -156,8 +155,7 @@ export async function downloadDatabaseUpdateFromGitHub(
repo: string,
updates: DatabaseUpdate[],
databaseManager: DatabaseManager,
storagePath: string,
cliServer: CodeQLCliServer,
databaseFetcher: DatabaseFetcher,
commandManager: AppCommandManager,
): Promise<void> {
const selectedDatabases = await promptForDatabases(
@@ -179,7 +177,8 @@ export async function downloadDatabaseUpdateFromGitHub(
return withProgress(
async (progress) => {
const newDatabase = await downloadGitHubDatabaseFromUrl(
const newDatabase =
await databaseFetcher.downloadGitHubDatabaseFromUrl(
database.url,
database.id,
database.created_at,
@@ -188,9 +187,6 @@ export async function downloadDatabaseUpdateFromGitHub(
repo,
octokit,
progress,
databaseManager,
storagePath,
cliServer,
databaseManager.currentDatabaseItem === update.databaseItem,
update.databaseItem.hasSourceArchiveInExplorer(),
);

41
extensions/ql-vscode/src/databases/local-databases-ui.ts
View File

@@ -37,11 +37,7 @@ import {
showAndLogExceptionWithTelemetry,
showAndLogErrorMessage,
} from "../common/logging";
import {
importLocalDatabase,
promptImportGithubDatabase,
promptImportInternetDatabase,
} from "./database-fetcher";
import type { DatabaseFetcher } from "./database-fetcher";
import { asError, asyncFilter, getErrorMessage } from "../common/helpers-pure";
import type { QueryRunner } from "../query-server";
import type { App } from "../common/app";
@@ -248,6 +244,7 @@ export class DatabaseUI extends DisposableObject {
public constructor(
private app: App,
private databaseManager: DatabaseManager,
private readonly databaseFetcher: DatabaseFetcher,
languageContext: LanguageContextStore,
private readonly queryServer: QueryRunner,
private readonly storagePath: string,
@@ -535,13 +532,7 @@ export class DatabaseUI extends DisposableObject {
private async handleChooseDatabaseInternet(): Promise<void> {
return withProgress(
async (progress) => {
await promptImportInternetDatabase(
this.app.commands,
this.databaseManager,
this.storagePath,
progress,
this.queryServer.cliServer,
);
await this.databaseFetcher.promptImportInternetDatabase(progress);
},
{
title: "Adding database from URL",
@@ -552,13 +543,7 @@ export class DatabaseUI extends DisposableObject {
private async handleChooseDatabaseGithub(): Promise<void> {
return withProgress(
async (progress) => {
await promptImportGithubDatabase(
this.app,
this.databaseManager,
this.storagePath,
progress,
this.queryServer.cliServer,
);
await this.databaseFetcher.promptImportGithubDatabase(progress);
},
{
title: "Adding database from GitHub",
@@ -707,13 +692,9 @@ export class DatabaseUI extends DisposableObject {
try {
// Assume user has selected an archive if the file has a .zip extension
if (uri.path.endsWith(".zip")) {
await importLocalDatabase(
this.app.commands,
await this.databaseFetcher.importLocalDatabase(
uri.toString(true),
this.databaseManager,
this.storagePath,
progress,
this.queryServer.cliServer,
);
} else {
await this.databaseManager.openDatabase(uri, {
@@ -758,13 +739,9 @@ export class DatabaseUI extends DisposableObject {
await this.databaseManager.removeDatabaseItem(existingItem);
}
await importLocalDatabase(
this.app.commands,
await this.databaseFetcher.importLocalDatabase(
uri.toString(true),
this.databaseManager,
this.storagePath,
progress,
this.queryServer.cliServer,
);
if (existingItem !== undefined) {
@@ -1005,13 +982,9 @@ export class DatabaseUI extends DisposableObject {
// we are selecting a database archive or a testproj.
// Unzip archives (if an archive) and copy into a workspace-controlled area
// before importing.
return await importLocalDatabase(
this.app.commands,
return await this.databaseFetcher.importLocalDatabase(
uri.toString(true),
this.databaseManager,
this.storagePath,
progress,
this.queryServer.cliServer,
);
}
}

26
extensions/ql-vscode/src/databases/local-databases/database-contents.ts
View File

@@ -1,4 +1,7 @@
import { pathExists, remove } from "fs-extra";
import { join } from "path";
import type { Uri } from "vscode";
import { zip } from "zip-a-folder";
/**
* The layout of the database.
@@ -28,3 +31,26 @@ export interface DatabaseContents {
export interface DatabaseContentsWithDbScheme extends DatabaseContents {
dbSchemeUri: Uri; // Always present
}
/**
* Databases created by the old odasa tool will not have a zipped
* source location. However, this extension works better if sources
* are zipped.
*
* This function ensures that the source location is zipped. If the
* `src` folder exists and the `src.zip` file does not, the `src`
* folder will be zipped and then deleted.
*
* @param databasePath The full path to the unzipped database
*/
export async function ensureZippedSourceLocation(
databasePath: string,
): Promise<void> {
const srcFolderPath = join(databasePath, "src");
const srcZipPath = `${srcFolderPath}.zip`;
if ((await pathExists(srcFolderPath)) && !(await pathExists(srcZipPath))) {
await zip(srcFolderPath, srcZipPath);
await remove(srcFolderPath);
}
}

2
extensions/ql-vscode/src/databases/local-databases/database-manager.ts
View File

@@ -43,7 +43,7 @@ import { DatabaseResolver } from "./database-resolver";
import { telemetryListener } from "../../common/vscode/telemetry";
import type { LanguageContextStore } from "../../language-context-store";
import type { DatabaseOrigin } from "./database-origin";
import { ensureZippedSourceLocation } from "../database-fetcher";
import { ensureZippedSourceLocation } from "./database-contents";
/**
* The name of the key in the workspaceState dictionary in which we

14
extensions/ql-vscode/src/extension.ts
View File

@@ -133,6 +133,7 @@ import { OpenReferencedFileCodeLensProvider } from "./local-queries/open-referen
import { LanguageContextStore } from "./language-context-store";
import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
import { GitHubDatabasesModule } from "./databases/github-databases";
import { DatabaseFetcher } from "./databases/database-fetcher";
/**
* extension.ts
@@ -799,12 +800,20 @@ async function activateWithInstalledDistribution(
// Let this run async.
void dbm.loadPersistedState();
const databaseFetcher = new DatabaseFetcher(
app,
dbm,
getContextStoragePath(ctx),
cliServer,
);
ctx.subscriptions.push(dbm);
void extLogger.log("Initializing database panel.");
const databaseUI = new DatabaseUI(
app,
dbm,
databaseFetcher,
languageContext,
qs,
getContextStoragePath(ctx),
@@ -881,8 +890,7 @@ async function activateWithInstalledDistribution(
await GitHubDatabasesModule.initialize(
app,
dbm,
getContextStoragePath(ctx),
cliServer,
databaseFetcher,
githubDatabaseConfigListener,
);
@@ -953,6 +961,7 @@ async function activateWithInstalledDistribution(
qs,
qhm,
dbm,
databaseFetcher,
cliServer,
databaseUI,
localQueryResultsView,
@@ -977,6 +986,7 @@ async function activateWithInstalledDistribution(
const modelEditorModule = await ModelEditorModule.initialize(
app,
dbm,
databaseFetcher,
variantAnalysisManager,
cliServer,
qs,

6
extensions/ql-vscode/src/local-queries/local-queries.ts
View File

@@ -54,6 +54,7 @@ import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
import { tryGetQueryLanguage } from "../common/query-language";
import type { LanguageContextStore } from "../language-context-store";
import type { ExtensionApp } from "../common/vscode/vscode-app";
import type { DatabaseFetcher } from "../databases/database-fetcher";
export enum QuickEvalType {
None,
@@ -69,6 +70,7 @@ export class LocalQueries extends DisposableObject {
private readonly queryRunner: QueryRunner,
private readonly queryHistoryManager: QueryHistoryManager,
private readonly databaseManager: DatabaseManager,
private readonly databaseFetcher: DatabaseFetcher,
private readonly cliServer: CodeQLCliServer,
private readonly databaseUI: DatabaseUI,
private readonly localQueryResultsView: ResultsView,
@@ -319,15 +321,13 @@ export class LocalQueries extends DisposableObject {
private async createSkeletonQuery(): Promise<void> {
await withProgress(
async (progress: ProgressCallback) => {
const contextStoragePath =
this.app.workspaceStoragePath || this.app.globalStoragePath;
const language = this.languageContextStore.selectedLanguage;
const skeletonQueryWizard = new SkeletonQueryWizard(
this.cliServer,
progress,
this.app,
this.databaseManager,
contextStoragePath,
this.databaseFetcher,
this.selectedQueryTreeViewItems,
language,
);

25
extensions/ql-vscode/src/local-queries/skeleton-query-wizard.ts
View File

@@ -19,10 +19,7 @@ import {
UserCancellationException,
withProgress,
} from "../common/vscode/progress";
import {
askForGitHubRepo,
downloadGitHubDatabase,
} from "../databases/database-fetcher";
import type { DatabaseFetcher } from "../databases/database-fetcher";
import {
getQlPackLocation,
isCodespacesTemplate,
@@ -62,7 +59,7 @@ export class SkeletonQueryWizard {
private readonly progress: ProgressCallback,
private readonly app: App,
private readonly databaseManager: DatabaseManager,
private readonly databaseStoragePath: string | undefined,
private readonly databaseFetcher: DatabaseFetcher,
private readonly selectedItems: readonly QueryTreeViewItem[],
private language: QueryLanguage | undefined = undefined,
) {}
@@ -363,10 +360,6 @@ export class SkeletonQueryWizard {
}
private async downloadDatabase(progress: ProgressCallback) {
if (this.databaseStoragePath === undefined) {
throw new Error("Database storage path is undefined");
}
if (this.language === undefined) {
throw new Error("Language is undefined");
}
@@ -378,20 +371,10 @@ export class SkeletonQueryWizard {
});
const githubRepoNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[this.language];
const chosenRepo = await askForGitHubRepo(undefined, githubRepoNwo);
if (!chosenRepo) {
throw new UserCancellationException("No GitHub repository provided");
}
await downloadGitHubDatabase(
chosenRepo,
this.app,
this.databaseManager,
this.databaseStoragePath,
await this.databaseFetcher.promptImportGithubDatabase(
progress,
this.cliServer,
this.language,
githubRepoNwo,
);
}

5
extensions/ql-vscode/src/model-editor/model-editor-module.ts
View File

@@ -32,6 +32,7 @@ import { INITIAL_MODE } from "./shared/mode";
import { isSupportedLanguage } from "./supported-languages";
import { DefaultNotifier, checkConsistency } from "./consistency-check";
import type { VariantAnalysisManager } from "../variant-analysis/variant-analysis-manager";
import type { DatabaseFetcher } from "../databases/database-fetcher";
export class ModelEditorModule extends DisposableObject {
private readonly queryStorageDir: string;
@@ -42,6 +43,7 @@ export class ModelEditorModule extends DisposableObject {
private constructor(
private readonly app: App,
private readonly databaseManager: DatabaseManager,
private readonly databaseFetcher: DatabaseFetcher,
private readonly variantAnalysisManager: VariantAnalysisManager,
private readonly cliServer: CodeQLCliServer,
private readonly queryRunner: QueryRunner,
@@ -65,6 +67,7 @@ export class ModelEditorModule extends DisposableObject {
public static async initialize(
app: App,
databaseManager: DatabaseManager,
databaseFetcher: DatabaseFetcher,
variantAnalysisManager: VariantAnalysisManager,
cliServer: CodeQLCliServer,
queryRunner: QueryRunner,
@@ -73,6 +76,7 @@ export class ModelEditorModule extends DisposableObject {
const modelEditorModule = new ModelEditorModule(
app,
databaseManager,
databaseFetcher,
variantAnalysisManager,
cliServer,
queryRunner,
@@ -236,6 +240,7 @@ export class ModelEditorModule extends DisposableObject {
this.modelingEvents,
this.modelConfig,
this.databaseManager,
this.databaseFetcher,
this.variantAnalysisManager,
this.cliServer,
this.queryRunner,

11
extensions/ql-vscode/src/model-editor/model-editor-view.ts
View File

@@ -29,7 +29,7 @@ import type {
} from "../databases/local-databases";
import type { CodeQLCliServer } from "../codeql-cli/cli";
import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
import { promptImportGithubDatabase } from "../databases/database-fetcher";
import type { DatabaseFetcher } from "../databases/database-fetcher";
import type { App } from "../common/app";
import { redactableError } from "../common/errors";
import {
@@ -86,6 +86,7 @@ export class ModelEditorView extends AbstractWebview<
private readonly modelingEvents: ModelingEvents,
private readonly modelConfig: ModelConfigListener,
private readonly databaseManager: DatabaseManager,
private readonly databaseFetcher: DatabaseFetcher,
private readonly variantAnalysisManager: VariantAnalysisManager,
private readonly cliServer: CodeQLCliServer,
private readonly queryRunner: QueryRunner,
@@ -852,6 +853,7 @@ export class ModelEditorView extends AbstractWebview<
this.modelingEvents,
this.modelConfig,
this.databaseManager,
this.databaseFetcher,
this.variantAnalysisManager,
this.cliServer,
this.queryRunner,
@@ -920,13 +922,10 @@ export class ModelEditorView extends AbstractWebview<
// the user to import the library database. We need to have the database
// imported to the query server, so we need to register it to our workspace.
const makeSelected = false;
const addedDatabase = await promptImportGithubDatabase(
this.app,
this.databaseManager,
this.app.workspaceStoragePath ?? this.app.globalStoragePath,
const addedDatabase = await this.databaseFetcher.promptImportGithubDatabase(
progress,
this.cliServer,
this.databaseItem.language,
undefined,
makeSelected,
false,
);

37
extensions/ql-vscode/test/vscode-tests/cli-integration/databases/database-fetcher.test.ts
View File

@@ -3,10 +3,7 @@ import { Uri, window } from "vscode";
import type { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import type { DatabaseManager } from "../../../../src/databases/local-databases";
import {
importLocalDatabase,
promptImportInternetDatabase,
} from "../../../../src/databases/database-fetcher";
import { DatabaseFetcher } from "../../../../src/databases/database-fetcher";
import {
cleanDatabases,
dbLoc,
@@ -15,9 +12,8 @@ import {
storagePath,
testprojLoc,
} from "../../global.helper";
import { createMockCommandManager } from "../../../__mocks__/commandsMock";
import { utimesSync } from "fs";
import { remove, existsSync } from "fs-extra";
import { existsSync, remove, utimesSync } from "fs-extra";
import { createMockApp } from "../../../__mocks__/appMock";
/**
* Run various integration tests for databases
@@ -51,14 +47,16 @@ describe("database-fetcher", () => {
describe("importLocalDatabase", () => {
it("should add a database from an archive", async () => {
const uri = Uri.file(dbLoc);
let dbItem = await importLocalDatabase(
createMockCommandManager(),
uri.toString(true),
const databaseFetcher = new DatabaseFetcher(
createMockApp(),
databaseManager,
storagePath,
progressCallback,
cli,
);
let dbItem = await databaseFetcher.importLocalDatabase(
uri.toString(true),
progressCallback,
);
expect(dbItem).toBe(databaseManager.currentDatabaseItem);
expect(dbItem).toBe(databaseManager.databaseItems[0]);
expect(dbItem).toBeDefined();
@@ -68,14 +66,16 @@ describe("database-fetcher", () => {
});
it("should import a testproj database", async () => {
let dbItem = await importLocalDatabase(
createMockCommandManager(),
Uri.file(testprojLoc).toString(true),
const databaseFetcher = new DatabaseFetcher(
createMockApp(),
databaseManager,
storagePath,
progressCallback,
cli,
);
let dbItem = await databaseFetcher.importLocalDatabase(
Uri.file(testprojLoc).toString(true),
progressCallback,
);
expect(dbItem).toBe(databaseManager.currentDatabaseItem);
expect(dbItem).toBe(databaseManager.databaseItems[0]);
expect(dbItem).toBeDefined();
@@ -109,13 +109,14 @@ describe("database-fetcher", () => {
// Provide a database URL when prompted
inputBoxStub.mockResolvedValue(DB_URL);
let dbItem = await promptImportInternetDatabase(
createMockCommandManager(),
const databaseFetcher = new DatabaseFetcher(
createMockApp(),
databaseManager,
storagePath,
progressCallback,
cli,
);
let dbItem =
await databaseFetcher.promptImportInternetDatabase(progressCallback);
expect(dbItem).toBeDefined();
dbItem = dbItem!;
expect(dbItem.name).toBe("db");

81
extensions/ql-vscode/test/vscode-tests/cli-integration/local-queries/skeleton-query-wizard.test.ts
View File

@@ -23,7 +23,7 @@ import type {
DatabaseManager,
FullDatabaseOptions,
} from "../../../../src/databases/local-databases";
import * as databaseFetcher from "../../../../src/databases/database-fetcher";
import type { DatabaseFetcher } from "../../../../src/databases/database-fetcher";
import { createMockDB } from "../../../factories/databases/databases";
import { asError } from "../../../../src/common/helpers-pure";
import { Setting } from "../../../../src/config";
@@ -42,6 +42,7 @@ describe("SkeletonQueryWizard", () => {
let mockApp: App;
let wizard: SkeletonQueryWizard;
let mockDatabaseManager: DatabaseManager;
let databaseFetcher: DatabaseFetcher;
let dir: DirResult;
let storagePath: string;
let quickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
@@ -55,11 +56,8 @@ describe("SkeletonQueryWizard", () => {
let createExampleQlFileSpy: jest.SpiedFunction<
typeof QlPackGenerator.prototype.createExampleQlFile
>;
let downloadGitHubDatabaseSpy: jest.SpiedFunction<
typeof databaseFetcher.downloadGitHubDatabase
>;
let askForGitHubRepoSpy: jest.SpiedFunction<
typeof databaseFetcher.askForGitHubRepo
let promptImportGithubDatabaseMock: jest.MockedFunction<
DatabaseFetcher["promptImportGithubDatabase"]
>;
let openTextDocumentSpy: jest.SpiedFunction<
typeof workspace.openTextDocument
@@ -115,6 +113,11 @@ describe("SkeletonQueryWizard", () => {
},
] as WorkspaceFolder[]);
promptImportGithubDatabaseMock = jest.fn().mockReturnValue(undefined);
databaseFetcher = mockedObject<DatabaseFetcher>({
promptImportGithubDatabase: promptImportGithubDatabaseMock,
});
quickPickSpy = jest.spyOn(window, "showQuickPick").mockResolvedValueOnce(
mockedQuickPickItem({
label: chosenLanguage,
@@ -133,9 +136,6 @@ describe("SkeletonQueryWizard", () => {
createExampleQlFileSpy = jest
.spyOn(QlPackGenerator.prototype, "createExampleQlFile")
.mockResolvedValue(undefined);
downloadGitHubDatabaseSpy = jest
.spyOn(databaseFetcher, "downloadGitHubDatabase")
.mockResolvedValue(undefined);
openTextDocumentSpy = jest
.spyOn(workspace, "openTextDocument")
.mockResolvedValue({} as TextDocument);
@@ -145,13 +145,9 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
);
askForGitHubRepoSpy = jest
.spyOn(databaseFetcher, "askForGitHubRepo")
.mockResolvedValue(QUERY_LANGUAGE_TO_DATABASE_REPO[chosenLanguage]);
});
afterEach(async () => {
@@ -172,7 +168,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
QueryLanguage.Swift,
);
@@ -202,7 +198,7 @@ describe("SkeletonQueryWizard", () => {
title: "Download database",
}),
);
expect(downloadGitHubDatabaseSpy).not.toHaveBeenCalled();
expect(promptImportGithubDatabaseMock).not.toHaveBeenCalled();
});
it("should download database for selected language when selecting download in prompt", async () => {
@@ -219,7 +215,7 @@ describe("SkeletonQueryWizard", () => {
await wizard.execute();
await wizard.waitForDownload();
expect(downloadGitHubDatabaseSpy).toHaveBeenCalled();
expect(promptImportGithubDatabaseMock).toHaveBeenCalled();
});
it("should open the query file", async () => {
@@ -320,7 +316,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManagerWithItems,
storagePath,
databaseFetcher,
selectedItems,
);
});
@@ -328,7 +324,7 @@ describe("SkeletonQueryWizard", () => {
it("should not download a new database for language", async () => {
await wizard.execute();
expect(downloadGitHubDatabaseSpy).not.toHaveBeenCalled();
expect(promptImportGithubDatabaseMock).not.toHaveBeenCalled();
});
it("should not select the database", async () => {
@@ -369,7 +365,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManagerWithItems,
storagePath,
databaseFetcher,
selectedItems,
);
});
@@ -377,7 +373,7 @@ describe("SkeletonQueryWizard", () => {
it("should not download a new database for language", async () => {
await wizard.execute();
expect(downloadGitHubDatabaseSpy).not.toHaveBeenCalled();
expect(promptImportGithubDatabaseMock).not.toHaveBeenCalled();
});
it("should select an existing database", async () => {
@@ -409,7 +405,6 @@ describe("SkeletonQueryWizard", () => {
});
describe("if database is missing", () => {
describe("if the user chooses to downloaded the suggested database from GitHub", () => {
beforeEach(() => {
showInformationMessageSpy.mockImplementation(
async (_message, options, item) => {
@@ -426,37 +421,7 @@ describe("SkeletonQueryWizard", () => {
await wizard.execute();
await wizard.waitForDownload();
expect(askForGitHubRepoSpy).toHaveBeenCalled();
expect(downloadGitHubDatabaseSpy).toHaveBeenCalled();
});
});
describe("if the user choses to download a different database from GitHub than the one suggested", () => {
beforeEach(() => {
showInformationMessageSpy.mockImplementation(
async (_message, options, item) => {
if (item === undefined) {
return options as MessageItem;
}
return item;
},
);
const chosenGitHubRepo = "pickles-owner/pickles-repo";
askForGitHubRepoSpy = jest
.spyOn(databaseFetcher, "askForGitHubRepo")
.mockResolvedValue(chosenGitHubRepo);
});
it("should download the newly chosen database", async () => {
await wizard.execute();
await wizard.waitForDownload();
expect(askForGitHubRepoSpy).toHaveBeenCalled();
expect(downloadGitHubDatabaseSpy).toHaveBeenCalled();
});
expect(promptImportGithubDatabaseMock).toHaveBeenCalled();
});
});
});
@@ -504,7 +469,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
QueryLanguage.Javascript,
);
@@ -725,7 +690,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
);
});
@@ -754,7 +719,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
);
});
@@ -787,7 +752,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
QueryLanguage.Swift,
);
@@ -830,7 +795,7 @@ describe("SkeletonQueryWizard", () => {
jest.fn(),
mockApp,
mockDatabaseManager,
storagePath,
databaseFetcher,
selectedItems,
);
});

14
extensions/ql-vscode/test/vscode-tests/global.helper.ts
View File

@@ -7,8 +7,8 @@ import type {
} from "../../src/databases/local-databases";
import type { CodeQLCliServer } from "../../src/codeql-cli/cli";
import type { CodeQLExtensionInterface } from "../../src/extension";
import { importLocalDatabase } from "../../src/databases/database-fetcher";
import { createMockCommandManager } from "../__mocks__/commandsMock";
import { DatabaseFetcher } from "../../src/databases/database-fetcher";
import { createMockApp } from "../__mocks__/appMock";
// This file contains helpers shared between tests that work with an activated extension.
@@ -40,15 +40,17 @@ export async function ensureTestDatabase(
// Add a database, but make sure the database manager is empty first
await cleanDatabases(databaseManager);
const uri = Uri.file(dbLoc);
const maybeDbItem = await importLocalDatabase(
createMockCommandManager(),
uri.toString(true),
const databaseFetcher = new DatabaseFetcher(
createMockApp(),
databaseManager,
storagePath,
cli,
);
const maybeDbItem = await databaseFetcher.importLocalDatabase(
uri.toString(true),
(_p) => {
/**/
},
cli,
);
if (!maybeDbItem) {

78
extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-databases/download.test.ts
View File

@@ -10,13 +10,11 @@ import {
askForGitHubDatabaseDownload,
downloadDatabaseFromGitHub,
} from "../../../../../src/databases/github-databases/download";
import type { DatabaseManager } from "../../../../../src/databases/local-databases";
import type { GitHubDatabaseConfig } from "../../../../../src/config";
import type { CodeQLCliServer } from "../../../../../src/codeql-cli/cli";
import { createMockCommandManager } from "../../../../__mocks__/commandsMock";
import * as databaseFetcher from "../../../../../src/databases/database-fetcher";
import type { DatabaseFetcher } from "../../../../../src/databases/database-fetcher";
import * as dialog from "../../../../../src/common/vscode/dialog";
import type { CodeqlDatabase } from "../../../../../src/databases/github-databases/api";
import { createMockApp } from "../../../../__mocks__/appMock";
describe("askForGitHubDatabaseDownload", () => {
const setDownload = jest.fn();
@@ -96,11 +94,9 @@ describe("downloadDatabaseFromGitHub", () => {
let octokit: Octokit;
const owner = "github";
const repo = "codeql";
let databaseManager: DatabaseManager;
let databaseFetcher: DatabaseFetcher;
const storagePath = "/a/b/c/d";
let cliServer: CodeQLCliServer;
const commandManager = createMockCommandManager();
const app = createMockApp();
let databases = [
mockedObject<CodeqlDatabase>({
@@ -116,14 +112,17 @@ describe("downloadDatabaseFromGitHub", () => {
];
let showQuickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
let downloadGitHubDatabaseFromUrlSpy: jest.SpiedFunction<
typeof databaseFetcher.downloadGitHubDatabaseFromUrl
let downloadGitHubDatabaseFromUrlMock: jest.MockedFunction<
DatabaseFetcher["downloadGitHubDatabaseFromUrl"]
>;
beforeEach(() => {
octokit = mockedObject<Octokit>({});
databaseManager = mockedObject<DatabaseManager>({});
cliServer = mockedObject<CodeQLCliServer>({});
downloadGitHubDatabaseFromUrlMock = jest.fn().mockReturnValue(undefined);
databaseFetcher = mockedObject<DatabaseFetcher>({
downloadGitHubDatabaseFromUrl: downloadGitHubDatabaseFromUrlMock,
});
showQuickPickSpy = jest.spyOn(window, "showQuickPick").mockResolvedValue(
mockedQuickPickItem([
@@ -132,9 +131,6 @@ describe("downloadDatabaseFromGitHub", () => {
}),
]),
);
downloadGitHubDatabaseFromUrlSpy = jest
.spyOn(databaseFetcher, "downloadGitHubDatabaseFromUrl")
.mockResolvedValue(undefined);
});
it("downloads the database", async () => {
@@ -143,14 +139,12 @@ describe("downloadDatabaseFromGitHub", () => {
owner,
repo,
databases,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
databases[0].url,
databases[0].id,
databases[0].created_at,
@@ -159,9 +153,6 @@ describe("downloadDatabaseFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
true,
false,
);
@@ -207,14 +198,12 @@ describe("downloadDatabaseFromGitHub", () => {
owner,
repo,
databases,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
databases[1].url,
databases[1].id,
databases[1].created_at,
@@ -223,9 +212,6 @@ describe("downloadDatabaseFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
true,
false,
);
@@ -263,14 +249,12 @@ describe("downloadDatabaseFromGitHub", () => {
owner,
repo,
databases,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(2);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledTimes(2);
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
databases[0].url,
databases[0].id,
databases[0].created_at,
@@ -279,13 +263,10 @@ describe("downloadDatabaseFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
true,
false,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
databases[1].url,
databases[1].id,
databases[1].created_at,
@@ -294,9 +275,6 @@ describe("downloadDatabaseFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
true,
false,
);
@@ -328,13 +306,11 @@ describe("downloadDatabaseFromGitHub", () => {
owner,
repo,
databases,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled();
expect(downloadGitHubDatabaseFromUrlMock).not.toHaveBeenCalled();
});
});
});

20
extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-databases/github-databases-module.test.ts
View File

@@ -4,7 +4,6 @@ import { createMockApp } from "../../../../__mocks__/appMock";
import type { App } from "../../../../../src/common/app";
import type { DatabaseManager } from "../../../../../src/databases/local-databases";
import { mockEmptyDatabaseManager } from "../../query-testing/test-runner-helpers";
import type { CodeQLCliServer } from "../../../../../src/codeql-cli/cli";
import { mockDatabaseItem, mockedObject } from "../../../utils/mocking.helpers";
import type { GitHubDatabaseConfig } from "../../../../../src/config";
import { GitHubDatabasesModule } from "../../../../../src/databases/github-databases";
@@ -16,13 +15,13 @@ import * as githubDatabasesApi from "../../../../../src/databases/github-databas
import * as githubDatabasesDownload from "../../../../../src/databases/github-databases/download";
import * as githubDatabasesUpdates from "../../../../../src/databases/github-databases/updates";
import type { DatabaseUpdate } from "../../../../../src/databases/github-databases/updates";
import type { DatabaseFetcher } from "../../../../../src/databases/database-fetcher";
describe("GitHubDatabasesModule", () => {
describe("promptGitHubRepositoryDownload", () => {
let app: App;
let databaseManager: DatabaseManager;
let databaseStoragePath: string;
let cliServer: CodeQLCliServer;
const databaseFetcher = mockedObject<DatabaseFetcher>({});
let config: GitHubDatabaseConfig;
let gitHubDatabasesModule: GitHubDatabasesModule;
@@ -64,8 +63,6 @@ describe("GitHubDatabasesModule", () => {
beforeEach(() => {
app = createMockApp();
databaseManager = mockEmptyDatabaseManager();
databaseStoragePath = "/a/b/some-path";
cliServer = mockedObject<CodeQLCliServer>({});
config = mockedObject<GitHubDatabaseConfig>({
download: "ask",
update: "ask",
@@ -74,8 +71,7 @@ describe("GitHubDatabasesModule", () => {
gitHubDatabasesModule = new GitHubDatabasesModule(
app,
databaseManager,
databaseStoragePath,
cliServer,
databaseFetcher,
config,
);
@@ -124,8 +120,7 @@ describe("GitHubDatabasesModule", () => {
gitHubDatabasesModule = new GitHubDatabasesModule(
app,
databaseManager,
databaseStoragePath,
cliServer,
databaseFetcher,
config,
);
@@ -206,9 +201,7 @@ describe("GitHubDatabasesModule", () => {
owner,
repo,
databases,
databaseManager,
databaseStoragePath,
cliServer,
databaseFetcher,
app.commands,
);
});
@@ -250,8 +243,7 @@ describe("GitHubDatabasesModule", () => {
repo,
databaseUpdates,
databaseManager,
databaseStoragePath,
cliServer,
databaseFetcher,
app.commands,
);
});

71
extensions/ql-vscode/test/vscode-tests/no-workspace/databases/github-databases/updates.test.ts
View File

@@ -10,9 +10,7 @@ import {
import type { CodeqlDatabase } from "../../../../../src/databases/github-databases/api";
import type { DatabaseManager } from "../../../../../src/databases/local-databases";
import type { GitHubDatabaseConfig } from "../../../../../src/config";
import type { CodeQLCliServer } from "../../../../../src/codeql-cli/cli";
import { createMockCommandManager } from "../../../../__mocks__/commandsMock";
import * as databaseFetcher from "../../../../../src/databases/database-fetcher";
import type { DatabaseFetcher } from "../../../../../src/databases/database-fetcher";
import * as dialog from "../../../../../src/common/vscode/dialog";
import type { DatabaseUpdate } from "../../../../../src/databases/github-databases/updates";
import {
@@ -20,6 +18,7 @@ import {
downloadDatabaseUpdateFromGitHub,
isNewerDatabaseAvailable,
} from "../../../../../src/databases/github-databases/updates";
import { createMockApp } from "../../../../__mocks__/appMock";
describe("isNewerDatabaseAvailable", () => {
const owner = "github";
@@ -344,9 +343,8 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
const owner = "github";
const repo = "codeql";
let databaseManager: DatabaseManager;
const storagePath = "/a/b/c/d";
let cliServer: CodeQLCliServer;
const commandManager = createMockCommandManager();
let databaseFetcher: DatabaseFetcher;
const app = createMockApp();
let updates: DatabaseUpdate[] = [
{
@@ -367,8 +365,8 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
];
let showQuickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
let downloadGitHubDatabaseFromUrlSpy: jest.SpiedFunction<
typeof databaseFetcher.downloadGitHubDatabaseFromUrl
let downloadGitHubDatabaseFromUrlMock: jest.MockedFunction<
DatabaseFetcher["downloadGitHubDatabaseFromUrl"]
>;
beforeEach(() => {
@@ -376,7 +374,11 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
databaseManager = mockedObject<DatabaseManager>({
currentDatabaseItem: mockDatabaseItem(),
});
cliServer = mockedObject<CodeQLCliServer>({});
downloadGitHubDatabaseFromUrlMock = jest.fn().mockReturnValue(undefined);
databaseFetcher = mockedObject<DatabaseFetcher>({
downloadGitHubDatabaseFromUrl: downloadGitHubDatabaseFromUrlMock,
});
showQuickPickSpy = jest.spyOn(window, "showQuickPick").mockResolvedValue(
mockedQuickPickItem([
@@ -385,9 +387,6 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
}),
]),
);
downloadGitHubDatabaseFromUrlSpy = jest
.spyOn(databaseFetcher, "downloadGitHubDatabaseFromUrl")
.mockResolvedValue(undefined);
});
it("downloads the database", async () => {
@@ -397,13 +396,12 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
updates,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
updates[0].database.url,
updates[0].database.id,
updates[0].database.created_at,
@@ -412,9 +410,6 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
false,
false,
);
@@ -476,13 +471,12 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
updates,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledTimes(1);
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
updates[1].database.url,
updates[1].database.id,
updates[1].database.created_at,
@@ -491,9 +485,6 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
true,
true,
);
@@ -532,13 +523,12 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
updates,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledTimes(2);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledTimes(2);
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
updates[0].database.url,
updates[0].database.id,
updates[0].database.created_at,
@@ -547,13 +537,10 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
false,
false,
);
expect(downloadGitHubDatabaseFromUrlSpy).toHaveBeenCalledWith(
expect(downloadGitHubDatabaseFromUrlMock).toHaveBeenCalledWith(
updates[1].database.url,
updates[1].database.id,
updates[1].database.created_at,
@@ -562,9 +549,6 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
octokit,
expect.anything(),
databaseManager,
storagePath,
cliServer,
true,
true,
);
@@ -597,12 +581,11 @@ describe("downloadDatabaseUpdateFromGitHub", () => {
repo,
updates,
databaseManager,
storagePath,
cliServer,
commandManager,
databaseFetcher,
app.commands,
);
expect(downloadGitHubDatabaseFromUrlSpy).not.toHaveBeenCalled();
expect(downloadGitHubDatabaseFromUrlMock).not.toHaveBeenCalled();
});
});
});

4
extensions/ql-vscode/test/vscode-tests/no-workspace/databases/local-databases-ui.test.ts
View File

@@ -20,6 +20,7 @@ import { testDisposeHandler } from "../../test-dispose-handler";
import { createMockApp } from "../../../__mocks__/appMock";
import { QueryLanguage } from "../../../../src/common/query-language";
import { mockedQuickPickItem, mockedObject } from "../../utils/mocking.helpers";
import type { DatabaseFetcher } from "../../../../src/databases/database-fetcher";
describe("local-databases-ui", () => {
const storageDir = dirSync({ unsafeCleanup: true }).name;
@@ -104,6 +105,7 @@ describe("local-databases-ui", () => {
},
setCurrentDatabaseItem: () => {},
} as any,
mockedObject<DatabaseFetcher>({}),
{
onLanguageContextChanged: () => {
/**/
@@ -141,6 +143,7 @@ describe("local-databases-ui", () => {
setCurrentDatabaseItem: () => {},
currentDatabaseItem: { databaseUri: Uri.file(db1) },
} as any,
mockedObject<DatabaseFetcher>({}),
{
onLanguageContextChanged: () => {
/**/
@@ -177,6 +180,7 @@ describe("local-databases-ui", () => {
const databaseUI = new DatabaseUI(
app,
databaseManager,
mockedObject<DatabaseFetcher>({}),
{
onLanguageContextChanged: () => {
/**/

3
extensions/ql-vscode/test/vscode-tests/no-workspace/model-editor/model-editor-view.test.ts
View File

@@ -13,6 +13,7 @@ import type { ModelConfigListener } from "../../../../src/config";
import { createMockModelingEvents } from "../../../__mocks__/model-editor/modelingEventsMock";
import { QueryLanguage } from "../../../../src/common/query-language";
import type { VariantAnalysisManager } from "../../../../src/variant-analysis/variant-analysis-manager";
import type { DatabaseFetcher } from "../../../../src/databases/database-fetcher";
describe("ModelEditorView", () => {
const app = createMockApp({});
@@ -22,6 +23,7 @@ describe("ModelEditorView", () => {
onDidChangeConfiguration: jest.fn(),
});
const databaseManager = mockEmptyDatabaseManager();
const databaseFetcher = mockedObject<DatabaseFetcher>({});
const variantAnalysisManager = mockedObject<VariantAnalysisManager>({});
const cliServer = mockedObject<CodeQLCliServer>({});
const queryRunner = mockedObject<QueryRunner>({});
@@ -50,6 +52,7 @@ describe("ModelEditorView", () => {
modelingEvents,
modelConfig,
databaseManager,
databaseFetcher,
variantAnalysisManager,
cliServer,
queryRunner,