Clean databases folder on startup (#675)

Cleans orphan databases on startup. This commit also bumps the fs-extra
dependency to get readdir with dirent objects.

Adds the `asyncFilter` to filter arrays asynchronously.

extensions/ql-vscode/CHANGELOG.md

@@ -7,6 +7,7 @@
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
 - Add more structured output for tests. [#626](https://github.com/github/vscode-codeql/pull/626)
+- Whenever the extension restarts, orphaned databases will be cleaned up. These are databases whose files are located inside of the extension's storage area, but are not imported into the workspace.
 
 ## 1.3.6 - 4 November 2020
 

extensions/ql-vscode/package-lock.json

@@ -217,9 +217,9 @@
       "dev": true
     },
     "@types/fs-extra": {
-      "version": "8.1.1",
+      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.3.tgz",
-      "integrity": "sha512-TcUlBem321DFQzBNuz8p0CLLKp0VvF/XH9E4KHNmgwyp4E3AfgI5cjiIVZWlbfThBop2qxFIh4+LeY6hVWWZ2w==",
+      "integrity": "sha512-NKdGoXLTFTRED3ENcfCsH8+ekV4gbsysanx2OPbstXVV6fZMgUCqTxubs6I9r7pbOJbFgVq1rpFtLURjKCZWUw==",
       "dev": true,
       "requires": {
         "@types/node": "*"
@@ -310,9 +310,9 @@
       "dev": true
     },
     "@types/node": {
-      "version": "12.12.50",
+      "version": "12.19.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.50.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.19.4.tgz",
-      "integrity": "sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w==",
+      "integrity": "sha512-o3oj1bETk8kBwzz1WlO6JWL/AfAA3Vm6J1B3C9CsdxHYp7XgPiH7OEXPUbZTndHlRaIElrANkQfe6ZmfJb3H2w==",
       "dev": true
     },
     "@types/node-fetch": {
@@ -1251,6 +1251,11 @@
       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
       "dev": true
     },
+    "at-least-node": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",
+      "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg=="
+    },
     "atob": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
@@ -3748,13 +3753,14 @@
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "fs-extra": {
-      "version": "8.1.0",
+      "version": "9.0.1",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz",
-      "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
+      "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==",
       "requires": {
+        "at-least-node": "^1.0.0",
         "graceful-fs": "^4.2.0",
-        "jsonfile": "^4.0.0",
+        "jsonfile": "^6.0.1",
-        "universalify": "^0.1.0"
+        "universalify": "^1.0.0"
       }
     },
     "fs-mkdirp-stream": {
@@ -5026,11 +5032,19 @@
       "dev": true
     },
     "jsonfile": {
-      "version": "4.0.0",
+      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
-      "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
       "requires": {
-        "graceful-fs": "^4.1.6"
+        "graceful-fs": "^4.1.6",
+        "universalify": "^2.0.0"
+      },
+      "dependencies": {
+        "universalify": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
+          "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
+        }
       }
     },
     "jsx-ast-utils": {
@@ -9001,9 +9015,9 @@
       }
     },
     "universalify": {
-      "version": "0.1.2",
+      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz",
-      "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
+      "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug=="
     },
     "unset-value": {
       "version": "1.0.0",

extensions/ql-vscode/package.json

@@ -197,6 +197,10 @@
           "dark": "media/dark/folder-opened-plus.svg"
         }
       },
+      {
+        "command": "codeQLDatabases.removeOrphanedDatabases",
+        "title": "Delete unused databases"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseArchive",
         "title": "Choose Database from Archive",
@@ -573,6 +577,10 @@
           "command": "codeQLDatabases.chooseDatabaseArchive",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.removeOrphanedDatabases",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.chooseDatabaseInternet",
           "when": "false"
@@ -704,7 +712,7 @@
   "dependencies": {
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.0.1",
     "glob-promise": "^3.4.0",
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.5",
@@ -727,7 +735,7 @@
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^8.0.0",
+    "@types/fs-extra": "^9.0.3",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
@@ -735,7 +743,7 @@
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
     "@types/mocha": "~8.0.3",
-    "@types/node": "^12.0.8",
+    "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
     "@types/react": "^16.8.17",

extensions/ql-vscode/src/databases-ui.ts

@@ -8,7 +8,7 @@ import {
   TreeItem,
   Uri,
   window,
-  env
+  env,
 } from 'vscode';
 import * as fs from 'fs-extra';
 
@@ -18,6 +18,8 @@ import {
   DatabaseItem,
   DatabaseManager,
   getUpgradesDirectories,
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder,
 } from './databases';
 import {
   commandRunner,
@@ -36,6 +38,7 @@ import {
   promptImportLgtmDatabase,
 } from './databaseFetcher';
 import { CancellationToken } from 'vscode-jsonrpc';
+import { asyncFilter } from './pure/helpers-pure';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -229,7 +232,9 @@ export class DatabaseUI extends DisposableObject {
         canSelectMany: true,
       })
     );
+  }
 
+  init() {
     logger.log('Registering database panel commands.');
     this.push(
       commandRunnerWithProgress(
@@ -340,6 +345,12 @@ export class DatabaseUI extends DisposableObject {
         this.handleOpenFolder
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.removeOrphanedDatabases',
+        this.handleRemoveOrphanedDatabases
+      )
+    );
   }
 
   private handleMakeCurrentDatabase = async (
@@ -360,6 +371,53 @@ export class DatabaseUI extends DisposableObject {
     }
   };
 
+  handleRemoveOrphanedDatabases = async (): Promise<void> => {
+    logger.log('Removing orphaned databases from workspace storage.');
+    let dbDirs =
+      // read directory
+      (await fs.readdir(this.storagePath, { withFileTypes: true }))
+        // remove non-directories
+        .filter(dirent => dirent.isDirectory())
+        // get the full path
+        .map(dirent => path.join(this.storagePath, dirent.name))
+        // remove databases still in workspace
+        .filter(dbDir => {
+          const dbUri = Uri.file(dbDir);
+          return this.databaseManager.databaseItems.every(item => item.databaseUri.fsPath !== dbUri.fsPath);
+        });
+
+    // remove non-databases
+    dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
+
+    if (!dbDirs.length) {
+      logger.log('No orphaned databases found.');
+      return;
+    }
+
+    // delete
+    const failures = [] as string[];
+    await Promise.all(
+      dbDirs.map(async dbDir => {
+        try {
+          logger.log(`Deleting orphaned database '${dbDir}'.`);
+          await fs.rmdir(dbDir, { recursive: true } as any);  // typings doesn't recognize the options argument
+        } catch (e) {
+          failures.push(`${path.basename(dbDir)}`);
+        }
+      })
+    );
+
+    if (failures.length) {
+      const dirname = path.dirname(failures[0]);
+      showAndLogErrorMessage(
+        `Failed to delete unused databases:\n  ${
+        failures.join('\n  ')
+        }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.`
+      );
+    }
+  };
+
+
   handleChooseDatabaseArchive = async (
     progress: ProgressCallback,
     token: CancellationToken
@@ -653,7 +711,7 @@ export class DatabaseUI extends DisposableObject {
       dbPath = path.dirname(dbPath);
     }
 
-    if (isLikelyDbFolder(dbPath)) {
+    if (isLikelyDbLanguageFolder(dbPath)) {
       dbPath = path.dirname(dbPath);
     }
     return Uri.file(dbPath);
@@ -668,9 +726,3 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 }
-
-// TODO: Get the list of supported languages from a list that will be auto-updated.
-const dbRegeEx = /^db-(javascript|go|cpp|java|python|csharp)$/;
-function isLikelyDbFolder(dbPath: string) {
-  return path.basename(dbPath).match(dbRegeEx);
-}

extensions/ql-vscode/src/databases.ts

@@ -397,10 +397,7 @@ export class DatabaseItemImpl implements DatabaseItem {
    * Holds if the database item refers to an exported snapshot
    */
   public async hasMetadataFile(): Promise<boolean> {
-    return (await Promise.all([
+    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
-      fs.pathExists(path.join(this.databaseUri.fsPath, '.dbinfo')),
-      fs.pathExists(path.join(this.databaseUri.fsPath, 'codeql-database.yml'))
-    ])).some(x => x);
   }
 
   /**
@@ -730,3 +727,23 @@ export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] {
   const uniqueParentDirs = new Set(parentDirs);
   return Array.from(uniqueParentDirs).map(filePath => vscode.Uri.file(filePath));
 }
+
+
+// TODO: Get the list of supported languages from a list that will be auto-updated.
+
+export async function isLikelyDatabaseRoot(fsPath: string) {
+  const [a, b, c] = (await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    fs.pathExists(path.join(fsPath, '.dbinfo')),
+    fs.pathExists(path.join(fsPath, 'codeql-database.yml')),
+
+    // they *must* have a db-language folder
+    (await fs.readdir(fsPath)).some(isLikelyDbLanguageFolder)
+  ]));
+
+  return (a || b) && c;
+}
+
+export function isLikelyDbLanguageFolder(dbPath: string) {
+  return !!path.basename(dbPath).startsWith('db-');
+}

extensions/ql-vscode/src/extension.ts

@@ -349,6 +349,7 @@ async function activateWithInstalledDistribution(
     getContextStoragePath(ctx),
     ctx.extensionPath
   );
+  databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
   logger.log('Initializing query history manager.');
@@ -643,6 +644,8 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
+  commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
+
   logger.log('Successfully finished extension initialization.');
 }
 

extensions/ql-vscode/src/pure/helpers-pure.ts

@@ -21,3 +21,11 @@ class ExhaustivityCheckingError extends Error {
 export function assertNever(value: never): never {
   throw new ExhaustivityCheckingError(value);
 }
+
+/**
+ * Use to perform array filters where the predicate is asynchronous.
+ */
+export const asyncFilter = async function <T>(arr: T[], predicate: (arg0: T) => Promise<boolean>) {
+  const results = await Promise.all(arr.map(predicate));
+  return arr.filter((_, index) => results[index]);
+};

extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts

@@ -49,11 +49,59 @@ describe('databases-ui', () => {
       const parentDir = path.join(dir, 'db-hucairz');
       const dbDir = path.join(parentDir, 'db-javascript');
       const file = path.join(dbDir, 'nested');
-      await fs.mkdirs(dbDir);
+      fs.mkdirsSync(dbDir);
-      await fs.createFile(file);
+      fs.createFileSync(file);
 
       const uri = await fixDbUri(Uri.file(file));
       expect(uri.toString()).to.eq(Uri.file(parentDir).toString());
     });
   });
+
+  it('should delete orphaned databases', async () => {
+    const storageDir = tmp.dirSync().name;
+    const db1 = createDatabase(storageDir, 'db1-imported', 'cpp');
+    const db2 = createDatabase(storageDir, 'db2-notimported', 'cpp');
+    const db3 = createDatabase(storageDir, 'db3-invalidlanguage', 'hucairz');
+
+    // these two should be deleted
+    const db4 = createDatabase(storageDir, 'db2-notimported-with-db-info', 'cpp', '.dbinfo');
+    const db5 = createDatabase(storageDir, 'db2-notimported-with-codeql-database.yml', 'cpp', 'codeql-database.yml');
+
+    const databaseUI = new DatabaseUI(
+      {} as any,
+      {
+        databaseItems: [
+          { databaseUri: Uri.file(db1) }
+        ],
+        onDidChangeDatabaseItem: () => { /**/ },
+        onDidChangeCurrentDatabaseItem: () => { /**/ },
+      } as any,
+      {} as any,
+      storageDir,
+      storageDir
+    );
+
+    await databaseUI.handleRemoveOrphanedDatabases();
+
+    expect(fs.pathExistsSync(db1)).to.be.true;
+    expect(fs.pathExistsSync(db2)).to.be.true;
+    expect(fs.pathExistsSync(db3)).to.be.true;
+
+    expect(fs.pathExistsSync(db4)).to.be.false;
+    expect(fs.pathExistsSync(db5)).to.be.false;
+
+    databaseUI.dispose();
+  });
+
+  function createDatabase(storageDir: string, dbName: string, language: string, extraFile?: string) {
+    const parentDir = path.join(storageDir, dbName);
+    const dbDir = path.join(parentDir, `db-${language}`);
+    fs.mkdirsSync(dbDir);
+
+    if (extraFile) {
+      fs.createFileSync(path.join(parentDir, extraFile));
+    }
+
+    return parentDir;
+  }
 });

extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts

@@ -9,7 +9,8 @@ import {
   DatabaseItem,
   DatabaseManager,
   DatabaseItemImpl,
-  DatabaseContents
+  DatabaseContents,
+  isLikelyDbLanguageFolder
 } from '../../databases';
 import { QueryServerConfig } from '../../config';
 import { Logger } from '../../logging';
@@ -179,4 +180,9 @@ describe('databases', () => {
       );
     }
   });
+
+  it('should find likely db language folders', () => {
+    expect(isLikelyDbLanguageFolder('db-javascript')).to.be.true;
+    expect(isLikelyDbLanguageFolder('dbnot-a-db')).to.be.false;
+  });
 });

extensions/ql-vscode/test/pure-tests/helpers-pure.test.ts

@@ -0,0 +1,22 @@
+import { fail } from 'assert';
+import { expect } from 'chai';
+import { asyncFilter } from '../../src/pure/helpers-pure';
+
+describe('helpers-pure', () => {
+  it('should filter asynchronously', async () => {
+    expect(await asyncFilter([1, 2, 3], x => Promise.resolve(x > 2))).to.deep.eq([3]);
+  });
+
+  it('should throw on error when filtering', async () => {
+    const rejects = (x: number) => x === 3
+      ? Promise.reject(new Error('opps'))
+      : Promise.resolve(true);
+
+    try {
+      await asyncFilter([1, 2, 3], rejects);
+      fail('Should have thrown');
+    } catch (e) {
+      expect(e.message).to.eq('opps');
+    }
+  });
+});

extensions/ql-vscode/tsconfig.json

@@ -6,9 +6,7 @@
     "module": "commonjs",
     "target": "es2017",
     "outDir": "out",
-    "lib": [
+    "lib": ["ES2020"],
-      "es6"
-    ],
     "moduleResolution": "node",
     "sourceMap": true,
     "rootDir": "src",
@@ -21,12 +19,6 @@
     "noUnusedLocals": true,
     "noUnusedParameters": true
   },
-  "include": [
+  "include": ["src/**/*.ts"],
-    "src/**/*.ts"
+  "exclude": ["node_modules", "test", "**/view"]
-  ],
-  "exclude": [
-    "node_modules",
-    "test",
-    "**/view"
-  ]
 }