Add a language label next to databases in the UI

This change will only work on databases created by cli >= 2.4.1. In that
version, a new `primaryLanguage` field in the `codeql-database.yml`
file. We use this property as the language.

This change also includes a refactoring of the logic around extracting
database information heuristically based on file location. As much
as possible, it is extracted to the `helpers` module. Also, the
initial quick query text is generated based on the language (if known)
otherwise it falls back to the old style of generation.

extensions/ql-vscode/CHANGELOG.md

@@ -5,6 +5,7 @@
 - Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting VS Code. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)
 - Fix bug when removing databases where sometimes the source folder would not be removed from the workspace or the database files would not be removed from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692)
 - Query results with no string representation will now be displayed with placeholder text in query results. Previously, they were omitted. [#694](https://github.com/github/vscode-codeql/pull/694)
+- Add a label for the language of a database in the databases view. This will only take effect for new databases created with the CodeQL CLI v2.4.1 or later. [#697](https://github.com/github/vscode-codeql/pull/697)
 
 ## 1.3.7 - 24 November 2020
 

extensions/ql-vscode/src/cli.ts

@@ -50,6 +50,7 @@ export interface DbInfo {
   sourceArchiveRoot: string;
   datasetFolder: string;
   logsFolder: string;
+  primaryLanguage: string;
 }
 
 /**

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -16,8 +16,7 @@ export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem):
   if (db.contents === undefined)
     return undefined;
   const datasetPath = db.contents.datasetUri.fsPath;
-  const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath);
-  return qlpack;
+  return await helpers.getQlPackForDbscheme(cli, datasetPath);
 }
 
 

extensions/ql-vscode/src/databases-ui.ts

@@ -18,15 +18,15 @@ import {
   DatabaseItem,
   DatabaseManager,
   getUpgradesDirectories,
-  isLikelyDatabaseRoot,
-  isLikelyDbLanguageFolder,
 } from './databases';
 import {
   commandRunner,
   commandRunnerWithProgress,
   getOnDiskWorkspaceFolders,
   ProgressCallback,
-  showAndLogErrorMessage
+  showAndLogErrorMessage,
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder
 } from './helpers';
 import { logger } from './logging';
 import { clearCacheInDatabase } from './run-queries';
@@ -143,6 +143,7 @@ class DatabaseTreeDataProvider extends DisposableObject
       );
     }
     item.tooltip = element.databaseUri.fsPath;
+    item.description = element.language;
     return item;
   }
 

extensions/ql-vscode/src/databases.ts

@@ -4,7 +4,15 @@ import * as path from 'path';
 import * as vscode from 'vscode';
 import * as cli from './cli';
 import { ExtensionContext } from 'vscode';
-import { showAndLogErrorMessage, showAndLogWarningMessage, showAndLogInformationMessage, ProgressCallback, withProgress } from './helpers';
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  showAndLogInformationMessage,
+  getPrimaryLanguage,
+  isLikelyDatabaseRoot,
+  ProgressCallback,
+  withProgress
+} from './helpers';
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
 import { DisposableObject } from './vscode-utils/disposable-object';
 import { Logger, logger } from './logging';
@@ -37,11 +45,13 @@ export interface DatabaseOptions {
   displayName?: string;
   ignoreSourceArchive?: boolean;
   dateAdded?: number | undefined;
+  language?: string;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
   dateAdded: number | undefined;
+  language: string;
 }
 
 interface PersistedDatabaseItem {
@@ -194,6 +204,9 @@ export interface DatabaseItem {
   readonly databaseUri: vscode.Uri;
   /** The name of the database to be displayed in the UI */
   name: string;
+
+  /** The primary language of the database or empty string if unknown */
+  readonly language: string;
   /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
   readonly sourceArchive: vscode.Uri | undefined;
   /**
@@ -433,6 +446,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return dbInfo.datasetFolder;
   }
 
+  public get language() {
+    return this.options.language || '';
+  }
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive.
    */
@@ -502,18 +519,16 @@ export class DatabaseManager extends DisposableObject {
     progress: ProgressCallback,
     token: vscode.CancellationToken,
     uri: vscode.Uri,
-    options?: DatabaseOptions
   ): Promise<DatabaseItem> {
-
     const contents = await resolveDatabaseContents(uri);
-    const realOptions = options || {};
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj';
     const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ?
-        realOptions.ignoreSourceArchive : isQLTestDatabase,
-      displayName: realOptions.displayName,
-      dateAdded: realOptions.dateAdded || Date.now()
+      ignoreSourceArchive: isQLTestDatabase,
+      // displayName is only set if a user explicitly renames a database
+      displayName: undefined,
+      dateAdded: Date.now(),
+      language: await getPrimaryLanguage(uri.fsPath)
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => {
       this._onDidChangeDatabaseItem.fire(event);
@@ -573,6 +588,7 @@ export class DatabaseManager extends DisposableObject {
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
     let dateAdded = undefined;
+    let language = '';
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -583,11 +599,16 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.dateAdded === 'number') {
         dateAdded = state.options.dateAdded;
       }
+      if (state.options.language) {
+        language = state.options.language;
       }
+    }
+
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive,
       displayName,
-      dateAdded
+      dateAdded,
+      language
     };
     const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri, true), undefined, fullOptions,
       (event) => {
@@ -815,23 +836,3 @@ export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] {
   const uniqueParentDirs = new Set(parentDirs);
   return Array.from(uniqueParentDirs).map(filePath => vscode.Uri.file(filePath));
 }
-
-
-// TODO: Get the list of supported languages from a list that will be auto-updated.
-
-export async function isLikelyDatabaseRoot(fsPath: string) {
-  const [a, b, c] = (await Promise.all([
-    // databases can have either .dbinfo or codeql-database.yml.
-    fs.pathExists(path.join(fsPath, '.dbinfo')),
-    fs.pathExists(path.join(fsPath, 'codeql-database.yml')),
-
-    // they *must* have a db-language folder
-    (await fs.readdir(fsPath)).some(isLikelyDbLanguageFolder)
-  ]));
-
-  return (a || b) && c;
-}
-
-export function isLikelyDbLanguageFolder(dbPath: string) {
-  return !!path.basename(dbPath).startsWith('db-');
-}

extensions/ql-vscode/src/helpers.ts

@@ -358,12 +358,6 @@ function createRateLimitedResult(): RateLimitedResult {
   };
 }
 
-
-export type DatasetFolderInfo = {
-  dbscheme: string;
-  qlpack: string;
-}
-
 export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
   const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
   const packs: { packDir: string | undefined; packName: string }[] =
@@ -391,7 +385,7 @@ export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemeP
   throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
 }
 
-export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise<DatasetFolderInfo> {
+export async function getPrimaryDbscheme(datasetFolder: string): Promise<string> {
   const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'));
 
   if (dbschemes.length < 1) {
@@ -400,12 +394,11 @@ export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFo
 
   dbschemes.sort();
   const dbscheme = dbschemes[0];
+
   if (dbschemes.length > 1) {
     Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
   }
-
-  const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
-  return { dbscheme, qlpack };
+  return dbscheme;
 }
 
 /**
@@ -464,3 +457,71 @@ export class CachedOperation<U> {
     }
   }
 }
+
+
+
+/**
+ * The following functions al heuristically determine metadata about databases.
+ */
+
+const dbSchemeToLanguage = {
+  'semmlecode.javascript.dbscheme': 'javascript',
+  'semmlecode.cpp.dbscheme': 'cpp',
+  'semmlecode.dbscheme': 'java',
+  'semmlecode.python.dbscheme': 'python',
+  'semmlecode.csharp.dbscheme': 'csharp',
+  'go.dbscheme': 'go'
+};
+
+/**
+ * Returns the initial contents for an empty query, based on the language of the selected
+ * databse.
+ *
+ * First try to get the contents text based on language. if that fails, try to get based on
+ * dbscheme. Otherwise return no import statement.
+ *
+ * @param language the database language or empty string if unknown
+ * @param dbscheme path to the dbscheme file
+ *
+ * @returns an import and empty select statement appropriate for the selected language
+ */
+export function getInitialQueryContents(language: string, dbscheme: string) {
+  if (!language) {
+    const dbschemeBase = path.basename(dbscheme) as keyof typeof dbSchemeToLanguage;
+    language = dbSchemeToLanguage[dbschemeBase];
+  }
+
+  return language
+    ? `import ${language}\n\nselect ""`
+    : 'select ""';
+}
+
+export async function isLikelyDatabaseRoot(maybeRoot: string) {
+  const [a, b, c] = (await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    fs.pathExists(path.join(maybeRoot, '.dbinfo')),
+    fs.pathExists(path.join(maybeRoot, 'codeql-database.yml')),
+
+    // they *must* have a db-{language} folder
+    glob('db-*/', { cwd: maybeRoot })
+  ]));
+
+  return !!((a || b) && c);
+}
+
+export function isLikelyDbLanguageFolder(dbPath: string) {
+  return !!path.basename(dbPath).startsWith('db-');
+}
+
+export async function getPrimaryLanguage(root: string) {
+  try {
+    const metadataFile = path.join(root, 'codeql-database.yml');
+    if (await fs.pathExists(metadataFile)) {
+      const metadata = yaml.safeLoad(await fs.readFile(metadataFile, 'utf8')) as { primaryLanguage: string | undefined };
+      return metadata.primaryLanguage || '';
+    }
+  } catch (e) {
+    // could not determine language
+  }
+  return '';
+}

extensions/ql-vscode/src/pure/helpers-pure.ts

@@ -2,7 +2,7 @@
  * helpers-pure.ts
  * ------------
  *
- * Helper functions that don't depend on vscode and therefore can be used by the front-end and pure unit tests.
+ * Helper functions that don't depend on vscode or the CLI and therefore can be used by the front-end and pure unit tests.
  */
 
 /**

extensions/ql-vscode/src/quick-query.ts

@@ -5,8 +5,16 @@ import { CancellationToken, ExtensionContext, window as Window, workspace, Uri }
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import { CodeQLCliServer } from './cli';
 import { DatabaseUI } from './databases-ui';
-import * as helpers from './helpers';
 import { logger } from './logging';
+import {
+  getInitialQueryContents,
+  getPrimaryDbscheme,
+  getQlPackForDbscheme,
+  ProgressCallback,
+  showAndLogErrorMessage,
+  showBinaryChoiceDialog,
+  UserCancellationException
+} from './helpers';
 
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
@@ -16,21 +24,6 @@ export function isQuickQueryPath(queryPath: string): boolean {
   return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
 }
 
-/**
- * `getBaseText` heuristically returns an appropriate import statement
- * prelude based on the filename of the dbscheme file given. TODO: add
- * a 'default import' field to the qlpack itself, and use that.
- */
-function getBaseText(dbschemeBase: string) {
-  if (dbschemeBase == 'semmlecode.javascript.dbscheme') return 'import javascript\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.cpp.dbscheme') return 'import cpp\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.dbscheme') return 'import java\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.python.dbscheme') return 'import python\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.csharp.dbscheme') return 'import csharp\n\nselect ""';
-  if (dbschemeBase == 'go.dbscheme') return 'import go\n\nselect ""';
-  return 'select ""';
-}
-
 function getQuickQueriesDir(ctx: ExtensionContext): string {
   const storagePath = ctx.storagePath;
   if (storagePath === undefined) {
@@ -51,7 +44,7 @@ export async function displayQuickQuery(
   ctx: ExtensionContext,
   cliServer: CodeQLCliServer,
   databaseUI: DatabaseUI,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken
 ) {
 
@@ -85,7 +78,7 @@ export async function displayQuickQuery(
     // being undefined) just let the user know that they're in for a
     // restart.
     if (workspace.workspaceFile === undefined) {
-      const makeMultiRoot = await helpers.showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
+      const makeMultiRoot = await showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
       if (makeMultiRoot) {
         updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
       }
@@ -105,7 +98,9 @@ export async function displayQuickQuery(
     }
 
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
-    const { qlpack, dbscheme } = await helpers.resolveDatasetFolder(cliServer, datasetFolder);
+    const dbscheme = await getPrimaryDbscheme(datasetFolder);
+    const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
+
     const quickQueryQlpackYaml: any = {
       name: 'quick-query',
       version: '1.0.0',
@@ -114,21 +109,21 @@ export async function displayQuickQuery(
 
     const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
     const qlPackFile = path.join(queriesDir, 'qlpack.yml');
-    await fs.writeFile(qlFile, getBaseText(path.basename(dbscheme)), 'utf8');
+    await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
     await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8');
     Window.showTextDocument(await workspace.openTextDocument(qlFile));
   }
 
   // TODO: clean up error handling for top-level commands like this
   catch (e) {
-    if (e instanceof helpers.UserCancellationException) {
+    if (e instanceof UserCancellationException) {
       logger.log(e.message);
     }
     else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
       logger.log(e.message);
     }
     else if (e instanceof Error)
-      helpers.showAndLogErrorMessage(e.message);
+      showAndLogErrorMessage(e.message);
     else
       throw e;
   }

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -12,20 +12,20 @@ import {
   DatabaseManager,
   DatabaseItemImpl,
   DatabaseContents,
-  isLikelyDbLanguageFolder,
   FullDatabaseOptions
 } from '../../databases';
 import { Logger } from '../../logging';
 import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
 import { QueryServerClient } from '../../queryserver-client';
-import { ProgressCallback } from '../../helpers';
 import { registerDatabases } from '../../pure/messages';
+import { isLikelyDbLanguageFolder, ProgressCallback } from '../../helpers';
 
 describe('databases', () => {
 
   const MOCK_DB_OPTIONS: FullDatabaseOptions = {
     dateAdded: 123,
-    ignoreSourceArchive: false
+    ignoreSourceArchive: false,
+    language: ''
   };
 
   let databaseManager: DatabaseManager;

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts

@@ -16,7 +16,7 @@ const expect = chai.expect;
 describe('queryResolver', () => {
   let module: Record<string, Function>;
   let writeFileSpy: sinon.SinonSpy;
-  let resolveDatasetFolderSpy: sinon.SinonStub;
+  let getQlPackForDbschemeSpy: sinon.SinonStub;
   let mockCli: Record<string, sinon.SinonStub>;
   beforeEach(() => {
     mockCli = {
@@ -60,7 +60,7 @@ describe('queryResolver', () => {
 
   describe('qlpackOfDatabase', () => {
     it('should get the qlpack of a database', async () => {
-      resolveDatasetFolderSpy.returns({ qlpack: 'my-qlpack' });
+      getQlPackForDbschemeSpy.resolves('my-qlpack');
       const db = {
         contents: {
           datasetUri: {
@@ -70,20 +70,20 @@ describe('queryResolver', () => {
       };
       const result = await module.qlpackOfDatabase(mockCli, db);
       expect(result).to.eq('my-qlpack');
-      expect(resolveDatasetFolderSpy).to.have.been.calledWith(mockCli, '/path/to/database');
+      expect(getQlPackForDbschemeSpy).to.have.been.calledWith(mockCli, '/path/to/database');
     });
   });
 
   function createModule() {
     writeFileSpy = sinon.spy();
-    resolveDatasetFolderSpy = sinon.stub();
+    getQlPackForDbschemeSpy = sinon.stub();
     return proxyquire('../../../contextual/queryResolver', {
       'fs-extra': {
         writeFile: writeFileSpy
       },
 
       '../helpers': {
-        resolveDatasetFolder: resolveDatasetFolderSpy,
+        getQlPackForDbscheme: getQlPackForDbschemeSpy,
         getOnDiskWorkspaceFolders: () => ({}),
         showAndLogErrorMessage: () => ({})
       }

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -1,7 +1,12 @@
 import { expect } from 'chai';
 import 'mocha';
 import { ExtensionContext, Memento } from 'vscode';
-import { InvocationRateLimiter } from '../../helpers';
+import * as yaml from 'js-yaml';
+import * as tmp from 'tmp';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+
+import { getInitialQueryContents, getPrimaryLanguage, InvocationRateLimiter } from '../../helpers';
 
 describe('Invocation rate limiter', () => {
   // 1 January 2020
@@ -86,6 +91,41 @@ describe('Invocation rate limiter', () => {
   });
 });
 
+describe('codeql-database.yml tests', () => {
+  let dir: tmp.DirResult;
+  beforeEach(() => {
+    dir = tmp.dirSync();
+    const contents = yaml.safeDump({
+      primaryLanguage: 'cpp'
+    });
+    fs.writeFileSync(path.join(dir.name, 'codeql-database.yml'), contents, 'utf8');
+  });
+
+  afterEach(() => {
+    dir.removeCallback();
+  });
+
+  it('should get the language of a database', async () => {
+    expect(await getPrimaryLanguage(dir.name)).to.eq('cpp');
+  });
+
+  it('should get the language of a database when langauge is not known', async () => {
+    expect(await getPrimaryLanguage('xxx')).to.eq('');
+  });
+
+  it('should get initial query contents when language is known', () => {
+    expect(getInitialQueryContents('cpp', 'hucairz')).to.eq('import cpp\n\nselect ""');
+  });
+
+  it('should get initial query contents when dbscheme is known', () => {
+    expect(getInitialQueryContents('', 'semmlecode.cpp.dbscheme')).to.eq('import cpp\n\nselect ""');
+  });
+
+  it('should get initial query contents when nothing is known', () => {
+    expect(getInitialQueryContents('', 'hucairz')).to.eq('select ""');
+  });
+});
+
 class MockExtensionContext implements ExtensionContext {
   subscriptions: { dispose(): unknown }[] = [];
   workspaceState: Memento = new MockMemento();