Merge pull request #3797 from reitowo/main-multiple-folder

Allow import all database subfolders by selecting a folder

extensions/ql-vscode/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [UNRELEASED]
 
+- Add a palette command that allows importing all databases directly inside of a parent folder. [3797](https://github.com/github/vscode-codeql/pull/3797)
+
 ## 1.16.1 - 6 November 2024
 
 - Support result columns of type `QlBuiltins::BigInt` in quick evaluations. [#3647](https://github.com/github/vscode-codeql/pull/3647)

extensions/ql-vscode/package.json

@@ -839,6 +839,10 @@
         "command": "codeQL.chooseDatabaseFolder",
         "title": "CodeQL: Choose Database from Folder"
       },
+      {
+        "command": "codeQL.chooseDatabaseFoldersParent",
+        "title": "CodeQL: Import All Databases Directly Contained in a Parent Folder"
+      },
       {
         "command": "codeQL.chooseDatabaseArchive",
         "title": "CodeQL: Choose Database from Archive"

extensions/ql-vscode/src/common/commands.ts

@@ -211,6 +211,7 @@ export type LanguageSelectionCommands = {
 export type LocalDatabasesCommands = {
   // Command palette commands
   "codeQL.chooseDatabaseFolder": () => Promise<void>;
+  "codeQL.chooseDatabaseFoldersParent": () => Promise<void>;
   "codeQL.chooseDatabaseArchive": () => Promise<void>;
   "codeQL.chooseDatabaseInternet": () => Promise<void>;
   "codeQL.chooseDatabaseGithub": () => Promise<void>;

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -16,6 +16,7 @@ import {
   ThemeIcon,
   ThemeColor,
   workspace,
+  FileType,
 } from "vscode";
 import { pathExists, stat, readdir, remove } from "fs-extra";
 
@@ -36,6 +37,7 @@ import {
 import {
   showAndLogExceptionWithTelemetry,
   showAndLogErrorMessage,
+  showAndLogInformationMessage,
 } from "../common/logging";
 import type { DatabaseFetcher } from "./database-fetcher";
 import { asError, asyncFilter, getErrorMessage } from "../common/helpers-pure";
@@ -267,6 +269,8 @@ export class DatabaseUI extends DisposableObject {
       "codeQL.getCurrentDatabase": this.handleGetCurrentDatabase.bind(this),
       "codeQL.chooseDatabaseFolder":
         this.handleChooseDatabaseFolderFromPalette.bind(this),
+      "codeQL.chooseDatabaseFoldersParent":
+        this.handleChooseDatabaseFoldersParentFromPalette.bind(this),
       "codeQL.chooseDatabaseArchive":
         this.handleChooseDatabaseArchiveFromPalette.bind(this),
       "codeQL.chooseDatabaseInternet":
@@ -359,6 +363,12 @@ export class DatabaseUI extends DisposableObject {
     );
   }
 
+  private async handleChooseDatabaseFoldersParentFromPalette(): Promise<void> {
+    return withProgress(async (progress) => {
+      await this.chooseDatabasesParentFolder(progress);
+    });
+  }
+
   private async handleSetDefaultTourDatabase(): Promise<void> {
     return withProgress(
       async () => {
@@ -956,6 +966,32 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
+  /**
+   * Import database from uri. Returns the imported database, or `undefined` if the
+   * operation was unsuccessful or canceled.
+   */
+  private async importDatabase(
+    uri: Uri,
+    byFolder: boolean,
+    progress: ProgressCallback,
+  ): Promise<DatabaseItem | undefined> {
+    if (byFolder && !uri.fsPath.endsWith(".testproj")) {
+      const fixedUri = await this.fixDbUri(uri);
+      // we are selecting a database folder
+      return await this.databaseManager.openDatabase(fixedUri, {
+        type: "folder",
+      });
+    } else {
+      // we are selecting a database archive or a .testproj.
+      // Unzip archives (if an archive) and copy into a workspace-controlled area
+      // before importing.
+      return await this.databaseFetcher.importLocalDatabase(
+        uri.toString(true),
+        progress,
+      );
+    }
+  }
+
   /**
    * Ask the user for a database directory. Returns the chosen database, or `undefined` if the
    * operation was canceled.
@@ -969,21 +1005,89 @@ export class DatabaseUI extends DisposableObject {
       return undefined;
     }
 
-    if (byFolder && !uri.fsPath.endsWith("testproj")) {
+    return await this.importDatabase(uri, byFolder, progress);
-      const fixedUri = await this.fixDbUri(uri);
+  }
-      // we are selecting a database folder
+
-      return await this.databaseManager.openDatabase(fixedUri, {
+  /**
-        type: "folder",
+   * Ask the user for a parent directory that contains all databases.
+   * Returns all valid databases, or `undefined` if the operation was canceled.
+   */
+  private async chooseDatabasesParentFolder(
+    progress: ProgressCallback,
+  ): Promise<DatabaseItem[] | undefined> {
+    const uri = await chooseDatabaseDir(true);
+    if (!uri) {
+      return undefined;
+    }
+
+    const databases: DatabaseItem[] = [];
+    const failures: string[] = [];
+    const entries = await workspace.fs.readDirectory(uri);
+    const validFileTypes = [FileType.File, FileType.Directory];
+
+    for (const [index, entry] of entries.entries()) {
+      progress({
+        step: index + 1,
+        maxStep: entries.length,
+        message: `Importing '${entry[0]}'`,
       });
+
+      const subProgress: ProgressCallback = (p) => {
+        progress({
+          step: index + 1,
+          maxStep: entries.length,
+          message: `Importing '${entry[0]}': (${p.step}/${p.maxStep}) ${p.message}`,
+        });
+      };
+
+      if (!validFileTypes.includes(entry[1])) {
+        void this.app.logger.log(
+          `Skipping import for '${entry}', invalid file type: ${entry[1]}`,
+        );
+        continue;
+      }
+
+      try {
+        const databaseUri = Uri.joinPath(uri, entry[0]);
+        void this.app.logger.log(`Importing from ${databaseUri}`);
+
+        const database = await this.importDatabase(
+          databaseUri,
+          entry[1] === FileType.Directory,
+          subProgress,
+        );
+        if (database) {
+          databases.push(database);
+        } else {
+          failures.push(entry[0]);
+        }
+      } catch (e) {
+        failures.push(`${entry[0]}: ${getErrorMessage(e)}`.trim());
+      }
+    }
+
+    if (failures.length) {
+      void showAndLogErrorMessage(
+        this.app.logger,
+        `Failed to import ${failures.length} database(s), successfully imported ${databases.length} database(s).`,
+        {
+          fullMessage: `Failed to import ${failures.length} database(s), successfully imported ${databases.length} database(s).\nFailed databases:\n  - ${failures.join("\n  - ")}`,
+        },
+      );
+    } else if (databases.length === 0) {
+      void showAndLogErrorMessage(
+        this.app.logger,
+        `No database folder to import.`,
+      );
+      return undefined;
     } else {
-      // we are selecting a database archive or a testproj.
+      void showAndLogInformationMessage(
-      // Unzip archives (if an archive) and copy into a workspace-controlled area
+        this.app.logger,
-      // before importing.
+        `Successfully imported ${databases.length} database(s).`,
-      return await this.databaseFetcher.importLocalDatabase(
-        uri.toString(true),
-        progress,
       );
     }
+
+    return databases;
   }
 
   /**