diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index ec64803ab..41d1879da 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -7,6 +7,7 @@ - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624) - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624) - Add more structured output for tests. [#626](https://github.com/github/vscode-codeql/pull/626) +- Whenever the extension restarts, orphaned databases will be cleaned up. These are databases whose files are located inside of the extension's storage area, but are not imported into the workspace. ## 1.3.6 - 4 November 2020 diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index a337d38ab..ee75e1947 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -217,9 +217,9 @@ "dev": true }, "@types/fs-extra": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz", - "integrity": "sha512-TcUlBem321DFQzBNuz8p0CLLKp0VvF/XH9E4KHNmgwyp4E3AfgI5cjiIVZWlbfThBop2qxFIh4+LeY6hVWWZ2w==", + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.3.tgz", + "integrity": "sha512-NKdGoXLTFTRED3ENcfCsH8+ekV4gbsysanx2OPbstXVV6fZMgUCqTxubs6I9r7pbOJbFgVq1rpFtLURjKCZWUw==", "dev": true, "requires": { "@types/node": "*" @@ -310,9 +310,9 @@ "dev": true }, "@types/node": { - "version": "12.12.50", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.50.tgz", - "integrity": "sha512-5ImO01Fb8YsEOYpV+aeyGYztcYcjGsBvN4D7G5r1ef2cuQOpymjWNQi5V0rKHE6PC2ru3HkoUr/Br2/8GUA84w==", + "version": "12.19.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.19.4.tgz", + "integrity": "sha512-o3oj1bETk8kBwzz1WlO6JWL/AfAA3Vm6J1B3C9CsdxHYp7XgPiH7OEXPUbZTndHlRaIElrANkQfe6ZmfJb3H2w==", "dev": true }, "@types/node-fetch": { @@ -1251,6 +1251,11 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "dev": true }, + "at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==" + }, "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", @@ -3748,13 +3753,14 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", - "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz", + "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==", "requires": { + "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" + "jsonfile": "^6.0.1", + "universalify": "^1.0.0" } }, "fs-mkdirp-stream": { @@ -5026,11 +5032,19 @@ "dev": true }, "jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", "requires": { - "graceful-fs": "^4.1.6" + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + }, + "dependencies": { + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" + } } }, "jsx-ast-utils": { @@ -9001,9 +9015,9 @@ } }, "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz", + "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==" }, "unset-value": { "version": "1.0.0", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 598e99f10..e4183741c 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -197,6 +197,10 @@ "dark": "media/dark/folder-opened-plus.svg" } }, + { + "command": "codeQLDatabases.removeOrphanedDatabases", + "title": "Delete unused databases" + }, { "command": "codeQLDatabases.chooseDatabaseArchive", "title": "Choose Database from Archive", @@ -573,6 +577,10 @@ "command": "codeQLDatabases.chooseDatabaseArchive", "when": "false" }, + { + "command": "codeQLDatabases.removeOrphanedDatabases", + "when": "false" + }, { "command": "codeQLDatabases.chooseDatabaseInternet", "when": "false" @@ -704,7 +712,7 @@ "dependencies": { "child-process-promise": "^2.2.1", "classnames": "~2.2.6", - "fs-extra": "^8.1.0", + "fs-extra": "^9.0.1", "glob-promise": "^3.4.0", "js-yaml": "^3.14.0", "minimist": "~1.2.5", @@ -727,7 +735,7 @@ "@types/chai-as-promised": "~7.1.2", "@types/child-process-promise": "^2.2.1", "@types/classnames": "~2.2.9", - "@types/fs-extra": "^8.0.0", + "@types/fs-extra": "^9.0.3", "@types/glob": "^7.1.1", "@types/google-protobuf": "^3.2.7", "@types/gulp": "^4.0.6", @@ -735,7 +743,7 @@ "@types/js-yaml": "^3.12.5", "@types/jszip": "~3.1.6", "@types/mocha": "~8.0.3", - "@types/node": "^12.0.8", + "@types/node": "^12.14.1", "@types/node-fetch": "~2.5.2", "@types/proxyquire": "~1.3.28", "@types/react": "^16.8.17", diff --git a/extensions/ql-vscode/src/databases-ui.ts b/extensions/ql-vscode/src/databases-ui.ts index aeaa6263b..442c88c7b 100644 --- a/extensions/ql-vscode/src/databases-ui.ts +++ b/extensions/ql-vscode/src/databases-ui.ts @@ -8,7 +8,7 @@ import { TreeItem, Uri, window, - env + env, } from 'vscode'; import * as fs from 'fs-extra'; @@ -18,6 +18,8 @@ import { DatabaseItem, DatabaseManager, getUpgradesDirectories, + isLikelyDatabaseRoot, + isLikelyDbLanguageFolder, } from './databases'; import { commandRunner, @@ -36,6 +38,7 @@ import { promptImportLgtmDatabase, } from './databaseFetcher'; import { CancellationToken } from 'vscode-jsonrpc'; +import { asyncFilter } from './pure/helpers-pure'; type ThemableIconPath = { light: string; dark: string } | string; @@ -229,7 +232,9 @@ export class DatabaseUI extends DisposableObject { canSelectMany: true, }) ); + } + init() { logger.log('Registering database panel commands.'); this.push( commandRunnerWithProgress( @@ -340,6 +345,12 @@ export class DatabaseUI extends DisposableObject { this.handleOpenFolder ) ); + this.push( + commandRunner( + 'codeQLDatabases.removeOrphanedDatabases', + this.handleRemoveOrphanedDatabases + ) + ); } private handleMakeCurrentDatabase = async ( @@ -360,6 +371,53 @@ export class DatabaseUI extends DisposableObject { } }; + handleRemoveOrphanedDatabases = async (): Promise => { + logger.log('Removing orphaned databases from workspace storage.'); + let dbDirs = + // read directory + (await fs.readdir(this.storagePath, { withFileTypes: true })) + // remove non-directories + .filter(dirent => dirent.isDirectory()) + // get the full path + .map(dirent => path.join(this.storagePath, dirent.name)) + // remove databases still in workspace + .filter(dbDir => { + const dbUri = Uri.file(dbDir); + return this.databaseManager.databaseItems.every(item => item.databaseUri.fsPath !== dbUri.fsPath); + }); + + // remove non-databases + dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot); + + if (!dbDirs.length) { + logger.log('No orphaned databases found.'); + return; + } + + // delete + const failures = [] as string[]; + await Promise.all( + dbDirs.map(async dbDir => { + try { + logger.log(`Deleting orphaned database '${dbDir}'.`); + await fs.rmdir(dbDir, { recursive: true } as any); // typings doesn't recognize the options argument + } catch (e) { + failures.push(`${path.basename(dbDir)}`); + } + }) + ); + + if (failures.length) { + const dirname = path.dirname(failures[0]); + showAndLogErrorMessage( + `Failed to delete unused databases:\n ${ + failures.join('\n ') + }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.` + ); + } + }; + + handleChooseDatabaseArchive = async ( progress: ProgressCallback, token: CancellationToken @@ -653,7 +711,7 @@ export class DatabaseUI extends DisposableObject { dbPath = path.dirname(dbPath); } - if (isLikelyDbFolder(dbPath)) { + if (isLikelyDbLanguageFolder(dbPath)) { dbPath = path.dirname(dbPath); } return Uri.file(dbPath); @@ -668,9 +726,3 @@ export class DatabaseUI extends DisposableObject { } } } - -// TODO: Get the list of supported languages from a list that will be auto-updated. -const dbRegeEx = /^db-(javascript|go|cpp|java|python|csharp)$/; -function isLikelyDbFolder(dbPath: string) { - return path.basename(dbPath).match(dbRegeEx); -} diff --git a/extensions/ql-vscode/src/databases.ts b/extensions/ql-vscode/src/databases.ts index d9e6fbc2a..aeee04448 100644 --- a/extensions/ql-vscode/src/databases.ts +++ b/extensions/ql-vscode/src/databases.ts @@ -397,10 +397,7 @@ export class DatabaseItemImpl implements DatabaseItem { * Holds if the database item refers to an exported snapshot */ public async hasMetadataFile(): Promise { - return (await Promise.all([ - fs.pathExists(path.join(this.databaseUri.fsPath, '.dbinfo')), - fs.pathExists(path.join(this.databaseUri.fsPath, 'codeql-database.yml')) - ])).some(x => x); + return await isLikelyDatabaseRoot(this.databaseUri.fsPath); } /** @@ -730,3 +727,23 @@ export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] { const uniqueParentDirs = new Set(parentDirs); return Array.from(uniqueParentDirs).map(filePath => vscode.Uri.file(filePath)); } + + +// TODO: Get the list of supported languages from a list that will be auto-updated. + +export async function isLikelyDatabaseRoot(fsPath: string) { + const [a, b, c] = (await Promise.all([ + // databases can have either .dbinfo or codeql-database.yml. + fs.pathExists(path.join(fsPath, '.dbinfo')), + fs.pathExists(path.join(fsPath, 'codeql-database.yml')), + + // they *must* have a db-language folder + (await fs.readdir(fsPath)).some(isLikelyDbLanguageFolder) + ])); + + return (a || b) && c; +} + +export function isLikelyDbLanguageFolder(dbPath: string) { + return !!path.basename(dbPath).startsWith('db-'); +} diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index a5c00ba20..d61d565c4 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -349,6 +349,7 @@ async function activateWithInstalledDistribution( getContextStoragePath(ctx), ctx.extensionPath ); + databaseUI.init(); ctx.subscriptions.push(databaseUI); logger.log('Initializing query history manager.'); @@ -643,6 +644,8 @@ async function activateWithInstalledDistribution( title: 'Calculate AST' })); + commands.executeCommand('codeQLDatabases.removeOrphanedDatabases'); + logger.log('Successfully finished extension initialization.'); } diff --git a/extensions/ql-vscode/src/pure/helpers-pure.ts b/extensions/ql-vscode/src/pure/helpers-pure.ts index 8de911936..d370b1b62 100644 --- a/extensions/ql-vscode/src/pure/helpers-pure.ts +++ b/extensions/ql-vscode/src/pure/helpers-pure.ts @@ -21,3 +21,11 @@ class ExhaustivityCheckingError extends Error { export function assertNever(value: never): never { throw new ExhaustivityCheckingError(value); } + +/** + * Use to perform array filters where the predicate is asynchronous. + */ +export const asyncFilter = async function (arr: T[], predicate: (arg0: T) => Promise) { + const results = await Promise.all(arr.map(predicate)); + return arr.filter((_, index) => results[index]); +}; diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts index bbca853ef..db0fba55a 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts @@ -49,11 +49,59 @@ describe('databases-ui', () => { const parentDir = path.join(dir, 'db-hucairz'); const dbDir = path.join(parentDir, 'db-javascript'); const file = path.join(dbDir, 'nested'); - await fs.mkdirs(dbDir); - await fs.createFile(file); + fs.mkdirsSync(dbDir); + fs.createFileSync(file); const uri = await fixDbUri(Uri.file(file)); expect(uri.toString()).to.eq(Uri.file(parentDir).toString()); }); }); + + it('should delete orphaned databases', async () => { + const storageDir = tmp.dirSync().name; + const db1 = createDatabase(storageDir, 'db1-imported', 'cpp'); + const db2 = createDatabase(storageDir, 'db2-notimported', 'cpp'); + const db3 = createDatabase(storageDir, 'db3-invalidlanguage', 'hucairz'); + + // these two should be deleted + const db4 = createDatabase(storageDir, 'db2-notimported-with-db-info', 'cpp', '.dbinfo'); + const db5 = createDatabase(storageDir, 'db2-notimported-with-codeql-database.yml', 'cpp', 'codeql-database.yml'); + + const databaseUI = new DatabaseUI( + {} as any, + { + databaseItems: [ + { databaseUri: Uri.file(db1) } + ], + onDidChangeDatabaseItem: () => { /**/ }, + onDidChangeCurrentDatabaseItem: () => { /**/ }, + } as any, + {} as any, + storageDir, + storageDir + ); + + await databaseUI.handleRemoveOrphanedDatabases(); + + expect(fs.pathExistsSync(db1)).to.be.true; + expect(fs.pathExistsSync(db2)).to.be.true; + expect(fs.pathExistsSync(db3)).to.be.true; + + expect(fs.pathExistsSync(db4)).to.be.false; + expect(fs.pathExistsSync(db5)).to.be.false; + + databaseUI.dispose(); + }); + + function createDatabase(storageDir: string, dbName: string, language: string, extraFile?: string) { + const parentDir = path.join(storageDir, dbName); + const dbDir = path.join(parentDir, `db-${language}`); + fs.mkdirsSync(dbDir); + + if (extraFile) { + fs.createFileSync(path.join(parentDir, extraFile)); + } + + return parentDir; + } }); diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts index 07b8f49ab..70dda6173 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts @@ -9,7 +9,8 @@ import { DatabaseItem, DatabaseManager, DatabaseItemImpl, - DatabaseContents + DatabaseContents, + isLikelyDbLanguageFolder } from '../../databases'; import { QueryServerConfig } from '../../config'; import { Logger } from '../../logging'; @@ -179,4 +180,9 @@ describe('databases', () => { ); } }); + + it('should find likely db language folders', () => { + expect(isLikelyDbLanguageFolder('db-javascript')).to.be.true; + expect(isLikelyDbLanguageFolder('dbnot-a-db')).to.be.false; + }); }); diff --git a/extensions/ql-vscode/test/pure-tests/helpers-pure.test.ts b/extensions/ql-vscode/test/pure-tests/helpers-pure.test.ts new file mode 100644 index 000000000..bc6935629 --- /dev/null +++ b/extensions/ql-vscode/test/pure-tests/helpers-pure.test.ts @@ -0,0 +1,22 @@ +import { fail } from 'assert'; +import { expect } from 'chai'; +import { asyncFilter } from '../../src/pure/helpers-pure'; + +describe('helpers-pure', () => { + it('should filter asynchronously', async () => { + expect(await asyncFilter([1, 2, 3], x => Promise.resolve(x > 2))).to.deep.eq([3]); + }); + + it('should throw on error when filtering', async () => { + const rejects = (x: number) => x === 3 + ? Promise.reject(new Error('opps')) + : Promise.resolve(true); + + try { + await asyncFilter([1, 2, 3], rejects); + fail('Should have thrown'); + } catch (e) { + expect(e.message).to.eq('opps'); + } + }); +}); diff --git a/extensions/ql-vscode/tsconfig.json b/extensions/ql-vscode/tsconfig.json index b1de71d49..518af2b84 100644 --- a/extensions/ql-vscode/tsconfig.json +++ b/extensions/ql-vscode/tsconfig.json @@ -6,9 +6,7 @@ "module": "commonjs", "target": "es2017", "outDir": "out", - "lib": [ - "es6" - ], + "lib": ["ES2020"], "moduleResolution": "node", "sourceMap": true, "rootDir": "src", @@ -21,12 +19,6 @@ "noUnusedLocals": true, "noUnusedParameters": true }, - "include": [ - "src/**/*.ts" - ], - "exclude": [ - "node_modules", - "test", - "**/view" - ] + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "test", "**/view"] }