diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index c678396d8..e1c089a6c 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -5,6 +5,7 @@ - Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting VS Code. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681) - Fix bug when removing databases where sometimes the source folder would not be removed from the workspace or the database files would not be removed from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692) - Query results with no string representation will now be displayed with placeholder text in query results. Previously, they were omitted. [#694](https://github.com/github/vscode-codeql/pull/694) +- Add a label for the language of a database in the databases view. This will only take effect for new databases created with the CodeQL CLI v2.4.1 or later. [#697](https://github.com/github/vscode-codeql/pull/697) ## 1.3.7 - 24 November 2020 diff --git a/extensions/ql-vscode/src/cli.ts b/extensions/ql-vscode/src/cli.ts index f6a892dda..2df416259 100644 --- a/extensions/ql-vscode/src/cli.ts +++ b/extensions/ql-vscode/src/cli.ts @@ -50,6 +50,7 @@ export interface DbInfo { sourceArchiveRoot: string; datasetFolder: string; logsFolder: string; + primaryLanguage: string; } /** diff --git a/extensions/ql-vscode/src/contextual/queryResolver.ts b/extensions/ql-vscode/src/contextual/queryResolver.ts index b415ba7f7..c57039cbe 100644 --- a/extensions/ql-vscode/src/contextual/queryResolver.ts +++ b/extensions/ql-vscode/src/contextual/queryResolver.ts @@ -16,8 +16,7 @@ export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): if (db.contents === undefined) return undefined; const datasetPath = db.contents.datasetUri.fsPath; - const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath); - return qlpack; + return await helpers.getQlPackForDbscheme(cli, datasetPath); } diff --git a/extensions/ql-vscode/src/databases-ui.ts b/extensions/ql-vscode/src/databases-ui.ts index a4a9b01a8..58419eb82 100644 --- a/extensions/ql-vscode/src/databases-ui.ts +++ b/extensions/ql-vscode/src/databases-ui.ts @@ -18,15 +18,15 @@ import { DatabaseItem, DatabaseManager, getUpgradesDirectories, - isLikelyDatabaseRoot, - isLikelyDbLanguageFolder, } from './databases'; import { commandRunner, commandRunnerWithProgress, getOnDiskWorkspaceFolders, ProgressCallback, - showAndLogErrorMessage + showAndLogErrorMessage, + isLikelyDatabaseRoot, + isLikelyDbLanguageFolder } from './helpers'; import { logger } from './logging'; import { clearCacheInDatabase } from './run-queries'; @@ -143,6 +143,7 @@ class DatabaseTreeDataProvider extends DisposableObject ); } item.tooltip = element.databaseUri.fsPath; + item.description = element.language; return item; } diff --git a/extensions/ql-vscode/src/databases.ts b/extensions/ql-vscode/src/databases.ts index e9655fa3e..af52dd7e6 100644 --- a/extensions/ql-vscode/src/databases.ts +++ b/extensions/ql-vscode/src/databases.ts @@ -4,7 +4,15 @@ import * as path from 'path'; import * as vscode from 'vscode'; import * as cli from './cli'; import { ExtensionContext } from 'vscode'; -import { showAndLogErrorMessage, showAndLogWarningMessage, showAndLogInformationMessage, ProgressCallback, withProgress } from './helpers'; +import { + showAndLogErrorMessage, + showAndLogWarningMessage, + showAndLogInformationMessage, + getPrimaryLanguage, + isLikelyDatabaseRoot, + ProgressCallback, + withProgress +} from './helpers'; import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider'; import { DisposableObject } from './vscode-utils/disposable-object'; import { Logger, logger } from './logging'; @@ -37,11 +45,13 @@ export interface DatabaseOptions { displayName?: string; ignoreSourceArchive?: boolean; dateAdded?: number | undefined; + language?: string; } export interface FullDatabaseOptions extends DatabaseOptions { ignoreSourceArchive: boolean; dateAdded: number | undefined; + language: string; } interface PersistedDatabaseItem { @@ -194,6 +204,9 @@ export interface DatabaseItem { readonly databaseUri: vscode.Uri; /** The name of the database to be displayed in the UI */ name: string; + + /** The primary language of the database or empty string if unknown */ + readonly language: string; /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */ readonly sourceArchive: vscode.Uri | undefined; /** @@ -433,6 +446,10 @@ export class DatabaseItemImpl implements DatabaseItem { return dbInfo.datasetFolder; } + public get language() { + return this.options.language || ''; + } + /** * Returns the root uri of the virtual filesystem for this database's source archive. */ @@ -502,18 +519,16 @@ export class DatabaseManager extends DisposableObject { progress: ProgressCallback, token: vscode.CancellationToken, uri: vscode.Uri, - options?: DatabaseOptions ): Promise { - const contents = await resolveDatabaseContents(uri); - const realOptions = options || {}; // Ignore the source archive for QLTest databases by default. const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj'; const fullOptions: FullDatabaseOptions = { - ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ? - realOptions.ignoreSourceArchive : isQLTestDatabase, - displayName: realOptions.displayName, - dateAdded: realOptions.dateAdded || Date.now() + ignoreSourceArchive: isQLTestDatabase, + // displayName is only set if a user explicitly renames a database + displayName: undefined, + dateAdded: Date.now(), + language: await getPrimaryLanguage(uri.fsPath) }; const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => { this._onDidChangeDatabaseItem.fire(event); @@ -573,6 +588,7 @@ export class DatabaseManager extends DisposableObject { let displayName: string | undefined = undefined; let ignoreSourceArchive = false; let dateAdded = undefined; + let language = ''; if (state.options) { if (typeof state.options.displayName === 'string') { displayName = state.options.displayName; @@ -583,11 +599,16 @@ export class DatabaseManager extends DisposableObject { if (typeof state.options.dateAdded === 'number') { dateAdded = state.options.dateAdded; } + if (state.options.language) { + language = state.options.language; + } } + const fullOptions: FullDatabaseOptions = { ignoreSourceArchive, displayName, - dateAdded + dateAdded, + language }; const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri, true), undefined, fullOptions, (event) => { @@ -815,23 +836,3 @@ export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] { const uniqueParentDirs = new Set(parentDirs); return Array.from(uniqueParentDirs).map(filePath => vscode.Uri.file(filePath)); } - - -// TODO: Get the list of supported languages from a list that will be auto-updated. - -export async function isLikelyDatabaseRoot(fsPath: string) { - const [a, b, c] = (await Promise.all([ - // databases can have either .dbinfo or codeql-database.yml. - fs.pathExists(path.join(fsPath, '.dbinfo')), - fs.pathExists(path.join(fsPath, 'codeql-database.yml')), - - // they *must* have a db-language folder - (await fs.readdir(fsPath)).some(isLikelyDbLanguageFolder) - ])); - - return (a || b) && c; -} - -export function isLikelyDbLanguageFolder(dbPath: string) { - return !!path.basename(dbPath).startsWith('db-'); -} diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts index e8ac27ba1..4e58d0cbb 100644 --- a/extensions/ql-vscode/src/helpers.ts +++ b/extensions/ql-vscode/src/helpers.ts @@ -358,12 +358,6 @@ function createRateLimitedResult(): RateLimitedResult { }; } - -export type DatasetFolderInfo = { - dbscheme: string; - qlpack: string; -} - export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise { const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders()); const packs: { packDir: string | undefined; packName: string }[] = @@ -391,7 +385,7 @@ export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemeP throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`); } -export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise { +export async function getPrimaryDbscheme(datasetFolder: string): Promise { const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme')); if (dbschemes.length < 1) { @@ -400,12 +394,11 @@ export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFo dbschemes.sort(); const dbscheme = dbschemes[0]; + if (dbschemes.length > 1) { Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`); } - - const qlpack = await getQlPackForDbscheme(cliServer, dbscheme); - return { dbscheme, qlpack }; + return dbscheme; } /** @@ -464,3 +457,71 @@ export class CachedOperation { } } } + + + +/** + * The following functions al heuristically determine metadata about databases. + */ + +const dbSchemeToLanguage = { + 'semmlecode.javascript.dbscheme': 'javascript', + 'semmlecode.cpp.dbscheme': 'cpp', + 'semmlecode.dbscheme': 'java', + 'semmlecode.python.dbscheme': 'python', + 'semmlecode.csharp.dbscheme': 'csharp', + 'go.dbscheme': 'go' +}; + +/** + * Returns the initial contents for an empty query, based on the language of the selected + * databse. + * + * First try to get the contents text based on language. if that fails, try to get based on + * dbscheme. Otherwise return no import statement. + * + * @param language the database language or empty string if unknown + * @param dbscheme path to the dbscheme file + * + * @returns an import and empty select statement appropriate for the selected language + */ +export function getInitialQueryContents(language: string, dbscheme: string) { + if (!language) { + const dbschemeBase = path.basename(dbscheme) as keyof typeof dbSchemeToLanguage; + language = dbSchemeToLanguage[dbschemeBase]; + } + + return language + ? `import ${language}\n\nselect ""` + : 'select ""'; +} + +export async function isLikelyDatabaseRoot(maybeRoot: string) { + const [a, b, c] = (await Promise.all([ + // databases can have either .dbinfo or codeql-database.yml. + fs.pathExists(path.join(maybeRoot, '.dbinfo')), + fs.pathExists(path.join(maybeRoot, 'codeql-database.yml')), + + // they *must* have a db-{language} folder + glob('db-*/', { cwd: maybeRoot }) + ])); + + return !!((a || b) && c); +} + +export function isLikelyDbLanguageFolder(dbPath: string) { + return !!path.basename(dbPath).startsWith('db-'); +} + +export async function getPrimaryLanguage(root: string) { + try { + const metadataFile = path.join(root, 'codeql-database.yml'); + if (await fs.pathExists(metadataFile)) { + const metadata = yaml.safeLoad(await fs.readFile(metadataFile, 'utf8')) as { primaryLanguage: string | undefined }; + return metadata.primaryLanguage || ''; + } + } catch (e) { + // could not determine language + } + return ''; +} diff --git a/extensions/ql-vscode/src/pure/helpers-pure.ts b/extensions/ql-vscode/src/pure/helpers-pure.ts index d370b1b62..dd811b00f 100644 --- a/extensions/ql-vscode/src/pure/helpers-pure.ts +++ b/extensions/ql-vscode/src/pure/helpers-pure.ts @@ -2,7 +2,7 @@ * helpers-pure.ts * ------------ * - * Helper functions that don't depend on vscode and therefore can be used by the front-end and pure unit tests. + * Helper functions that don't depend on vscode or the CLI and therefore can be used by the front-end and pure unit tests. */ /** diff --git a/extensions/ql-vscode/src/quick-query.ts b/extensions/ql-vscode/src/quick-query.ts index ac65e391a..af9932da5 100644 --- a/extensions/ql-vscode/src/quick-query.ts +++ b/extensions/ql-vscode/src/quick-query.ts @@ -5,8 +5,16 @@ import { CancellationToken, ExtensionContext, window as Window, workspace, Uri } import { ErrorCodes, ResponseError } from 'vscode-languageclient'; import { CodeQLCliServer } from './cli'; import { DatabaseUI } from './databases-ui'; -import * as helpers from './helpers'; import { logger } from './logging'; +import { + getInitialQueryContents, + getPrimaryDbscheme, + getQlPackForDbscheme, + ProgressCallback, + showAndLogErrorMessage, + showBinaryChoiceDialog, + UserCancellationException +} from './helpers'; const QUICK_QUERIES_DIR_NAME = 'quick-queries'; const QUICK_QUERY_QUERY_NAME = 'quick-query.ql'; @@ -16,21 +24,6 @@ export function isQuickQueryPath(queryPath: string): boolean { return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME; } -/** - * `getBaseText` heuristically returns an appropriate import statement - * prelude based on the filename of the dbscheme file given. TODO: add - * a 'default import' field to the qlpack itself, and use that. - */ -function getBaseText(dbschemeBase: string) { - if (dbschemeBase == 'semmlecode.javascript.dbscheme') return 'import javascript\n\nselect ""'; - if (dbschemeBase == 'semmlecode.cpp.dbscheme') return 'import cpp\n\nselect ""'; - if (dbschemeBase == 'semmlecode.dbscheme') return 'import java\n\nselect ""'; - if (dbschemeBase == 'semmlecode.python.dbscheme') return 'import python\n\nselect ""'; - if (dbschemeBase == 'semmlecode.csharp.dbscheme') return 'import csharp\n\nselect ""'; - if (dbschemeBase == 'go.dbscheme') return 'import go\n\nselect ""'; - return 'select ""'; -} - function getQuickQueriesDir(ctx: ExtensionContext): string { const storagePath = ctx.storagePath; if (storagePath === undefined) { @@ -51,7 +44,7 @@ export async function displayQuickQuery( ctx: ExtensionContext, cliServer: CodeQLCliServer, databaseUI: DatabaseUI, - progress: helpers.ProgressCallback, + progress: ProgressCallback, token: CancellationToken ) { @@ -85,7 +78,7 @@ export async function displayQuickQuery( // being undefined) just let the user know that they're in for a // restart. if (workspace.workspaceFile === undefined) { - const makeMultiRoot = await helpers.showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?'); + const makeMultiRoot = await showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?'); if (makeMultiRoot) { updateQuickQueryDir(queriesDir, workspaceFolders.length, 0); } @@ -105,7 +98,9 @@ export async function displayQuickQuery( } const datasetFolder = await dbItem.getDatasetFolder(cliServer); - const { qlpack, dbscheme } = await helpers.resolveDatasetFolder(cliServer, datasetFolder); + const dbscheme = await getPrimaryDbscheme(datasetFolder); + const qlpack = await getQlPackForDbscheme(cliServer, dbscheme); + const quickQueryQlpackYaml: any = { name: 'quick-query', version: '1.0.0', @@ -114,21 +109,21 @@ export async function displayQuickQuery( const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME); const qlPackFile = path.join(queriesDir, 'qlpack.yml'); - await fs.writeFile(qlFile, getBaseText(path.basename(dbscheme)), 'utf8'); + await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8'); await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8'); Window.showTextDocument(await workspace.openTextDocument(qlFile)); } // TODO: clean up error handling for top-level commands like this catch (e) { - if (e instanceof helpers.UserCancellationException) { + if (e instanceof UserCancellationException) { logger.log(e.message); } else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) { logger.log(e.message); } else if (e instanceof Error) - helpers.showAndLogErrorMessage(e.message); + showAndLogErrorMessage(e.message); else throw e; } diff --git a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts index 8c32c3dd4..b36ace7d1 100644 --- a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts @@ -12,20 +12,20 @@ import { DatabaseManager, DatabaseItemImpl, DatabaseContents, - isLikelyDbLanguageFolder, FullDatabaseOptions } from '../../databases'; import { Logger } from '../../logging'; import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider'; import { QueryServerClient } from '../../queryserver-client'; -import { ProgressCallback } from '../../helpers'; import { registerDatabases } from '../../pure/messages'; +import { isLikelyDbLanguageFolder, ProgressCallback } from '../../helpers'; describe('databases', () => { const MOCK_DB_OPTIONS: FullDatabaseOptions = { dateAdded: 123, - ignoreSourceArchive: false + ignoreSourceArchive: false, + language: '' }; let databaseManager: DatabaseManager; diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts index fd719b583..d3df8d9ff 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts @@ -16,7 +16,7 @@ const expect = chai.expect; describe('queryResolver', () => { let module: Record; let writeFileSpy: sinon.SinonSpy; - let resolveDatasetFolderSpy: sinon.SinonStub; + let getQlPackForDbschemeSpy: sinon.SinonStub; let mockCli: Record; beforeEach(() => { mockCli = { @@ -60,7 +60,7 @@ describe('queryResolver', () => { describe('qlpackOfDatabase', () => { it('should get the qlpack of a database', async () => { - resolveDatasetFolderSpy.returns({ qlpack: 'my-qlpack' }); + getQlPackForDbschemeSpy.resolves('my-qlpack'); const db = { contents: { datasetUri: { @@ -70,20 +70,20 @@ describe('queryResolver', () => { }; const result = await module.qlpackOfDatabase(mockCli, db); expect(result).to.eq('my-qlpack'); - expect(resolveDatasetFolderSpy).to.have.been.calledWith(mockCli, '/path/to/database'); + expect(getQlPackForDbschemeSpy).to.have.been.calledWith(mockCli, '/path/to/database'); }); }); function createModule() { writeFileSpy = sinon.spy(); - resolveDatasetFolderSpy = sinon.stub(); + getQlPackForDbschemeSpy = sinon.stub(); return proxyquire('../../../contextual/queryResolver', { 'fs-extra': { writeFile: writeFileSpy }, '../helpers': { - resolveDatasetFolder: resolveDatasetFolderSpy, + getQlPackForDbscheme: getQlPackForDbschemeSpy, getOnDiskWorkspaceFolders: () => ({}), showAndLogErrorMessage: () => ({}) } diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts index 1afa9aa53..586470632 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts @@ -1,7 +1,12 @@ import { expect } from 'chai'; import 'mocha'; import { ExtensionContext, Memento } from 'vscode'; -import { InvocationRateLimiter } from '../../helpers'; +import * as yaml from 'js-yaml'; +import * as tmp from 'tmp'; +import * as path from 'path'; +import * as fs from 'fs-extra'; + +import { getInitialQueryContents, getPrimaryLanguage, InvocationRateLimiter } from '../../helpers'; describe('Invocation rate limiter', () => { // 1 January 2020 @@ -86,6 +91,41 @@ describe('Invocation rate limiter', () => { }); }); +describe('codeql-database.yml tests', () => { + let dir: tmp.DirResult; + beforeEach(() => { + dir = tmp.dirSync(); + const contents = yaml.safeDump({ + primaryLanguage: 'cpp' + }); + fs.writeFileSync(path.join(dir.name, 'codeql-database.yml'), contents, 'utf8'); + }); + + afterEach(() => { + dir.removeCallback(); + }); + + it('should get the language of a database', async () => { + expect(await getPrimaryLanguage(dir.name)).to.eq('cpp'); + }); + + it('should get the language of a database when langauge is not known', async () => { + expect(await getPrimaryLanguage('xxx')).to.eq(''); + }); + + it('should get initial query contents when language is known', () => { + expect(getInitialQueryContents('cpp', 'hucairz')).to.eq('import cpp\n\nselect ""'); + }); + + it('should get initial query contents when dbscheme is known', () => { + expect(getInitialQueryContents('', 'semmlecode.cpp.dbscheme')).to.eq('import cpp\n\nselect ""'); + }); + + it('should get initial query contents when nothing is known', () => { + expect(getInitialQueryContents('', 'hucairz')).to.eq('select ""'); + }); +}); + class MockExtensionContext implements ExtensionContext { subscriptions: { dispose(): unknown }[] = []; workspaceState: Memento = new MockMemento();