Merge pull request #1430 from github/dbartol/goto-ql

Initial implementation of sourcemap-based jump-to-QL command

extensions/ql-vscode/package.json

@@ -35,6 +35,7 @@
   },
   "activationEvents": [
     "onLanguage:ql",
+    "onLanguage:ql-summary",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
     "onView:codeQLAstViewer",
@@ -111,6 +112,12 @@
         "extensions": [
           ".qhelp"
         ]
+      },
+      {
+        "id": "ql-summary",
+        "filenames": [
+          "evaluator-log.summary"
+        ]
       }
     ],
     "grammars": [
@@ -621,6 +628,11 @@
           "light": "media/light/clear-all.svg",
           "dark": "media/dark/clear-all.svg"
         }
+      },
+      {
+        "command": "codeQL.gotoQL",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
       }
     ],
     "menus": {
@@ -1115,6 +1127,10 @@
         {
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
+        },
+        {
+          "command": "codeQL.gotoQL",
+          "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
     },

extensions/ql-vscode/src/cli.ts

@@ -683,6 +683,7 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       '--format=text',
       `--end-summary=${endSummaryPath}`,
+      '--sourcemap',
       inputPath,
       outputPath
     ];

extensions/ql-vscode/src/extension.ts

@@ -99,6 +99,7 @@ import { HistoryItemLabelProvider } from './history-item-label-provider';
 import { exportRemoteQueryResults } from './remote-queries/export-results';
 import { RemoteQuery } from './remote-queries/remote-query';
 import { EvalLogViewer } from './eval-log-viewer';
+import { SummaryLanguageSupport } from './log-insights/summary-language-support';
 
 /**
  * extension.ts
@@ -1051,6 +1052,8 @@ async function activateWithInstalledDistribution(
     })
   );
 
+  ctx.subscriptions.push(new SummaryLanguageSupport());
+
   void logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 

extensions/ql-vscode/src/log-insights/summary-language-support.ts

@@ -0,0 +1,154 @@
+import * as fs from 'fs-extra';
+import { RawSourceMap, SourceMapConsumer } from 'source-map';
+import { commands, Position, Selection, TextDocument, TextEditor, TextEditorRevealType, TextEditorSelectionChangeEvent, ViewColumn, window, workspace } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+import { commandRunner } from '../commandRunner';
+import { logger } from '../logging';
+import { getErrorMessage } from '../pure/helpers-pure';
+
+/** A `Position` within a specified file on disk. */
+interface PositionInFile {
+  filePath: string;
+  position: Position;
+}
+
+/**
+ * Opens the specified source location in a text editor.
+ * @param position The position (including file path) to show.
+ */
+async function showSourceLocation(position: PositionInFile): Promise<void> {
+  const document = await workspace.openTextDocument(position.filePath);
+  const editor = await window.showTextDocument(document, ViewColumn.Active);
+  editor.selection = new Selection(position.position, position.position);
+  editor.revealRange(editor.selection, TextEditorRevealType.InCenterIfOutsideViewport);
+}
+
+/**
+ * Simple language support for human-readable evaluator log summaries.
+ *
+ * This class implements the `codeQL.gotoQL` command, which jumps from RA code to the corresponding
+ * QL code that generated it. It also tracks the current selection and active editor to enable and
+ * disable that command based on whether there is a QL mapping for the current selection.
+ */
+export class SummaryLanguageSupport extends DisposableObject {
+  /**
+   * The last `TextDocument` (with language `ql-summary`) for which we tried to find a sourcemap, or
+   * `undefined` if we have not seen such a document yet.
+   */
+  private lastDocument : TextDocument | undefined = undefined;
+  /**
+   * The sourcemap for `lastDocument`, or `undefined` if there was no such sourcemap or document.
+   */
+  private sourceMap : SourceMapConsumer | undefined = undefined;
+
+  constructor() {
+    super();
+
+    this.push(window.onDidChangeActiveTextEditor(this.handleDidChangeActiveTextEditor));
+    this.push(window.onDidChangeTextEditorSelection(this.handleDidChangeTextEditorSelection));
+    this.push(workspace.onDidCloseTextDocument(this.handleDidCloseTextDocument));
+
+    this.push(commandRunner('codeQL.gotoQL', this.handleGotoQL));
+  }
+
+  /**
+   * Gets the location of the QL code that generated the RA at the current selection in the active
+   * editor, or `undefined` if there is no mapping.
+   */
+  private async getQLSourceLocation(): Promise<PositionInFile | undefined> {
+    const editor = window.activeTextEditor;
+    if (editor === undefined) {
+      return undefined;
+    }
+
+    const document = editor.document;
+    if (document.languageId !== 'ql-summary') {
+      return undefined;
+    }
+
+    if (document.uri.scheme !== 'file') {
+      return undefined;
+    }
+
+    if (this.lastDocument !== document) {
+      this.clearCache();
+
+      const mapPath = document.uri.fsPath + '.map';
+
+      try {
+        const sourceMapText = await fs.readFile(mapPath, 'utf-8');
+        const rawMap: RawSourceMap = JSON.parse(sourceMapText);
+        this.sourceMap = await new SourceMapConsumer(rawMap);
+      } catch (e: unknown) {
+        // Error reading sourcemap. Pretend there was no sourcemap.
+        void logger.log(`Error reading sourcemap file '${mapPath}': ${getErrorMessage(e)}`);
+        this.sourceMap = undefined;
+      }
+      this.lastDocument = document;
+    }
+
+    if (this.sourceMap === undefined) {
+      return undefined;
+    }
+
+    const qlPosition = this.sourceMap.originalPositionFor({
+      line: editor.selection.start.line + 1,
+      column: editor.selection.start.character,
+      bias: SourceMapConsumer.GREATEST_LOWER_BOUND
+    });
+
+    if ((qlPosition.source === null) || (qlPosition.line === null)) {
+      // No position found.
+      return undefined;
+    }
+    const line = qlPosition.line - 1;  // In `source-map`, lines are 1-based...
+    const column = qlPosition.column ?? 0;  // ...but columns are 0-based :(
+
+    return {
+      filePath: qlPosition.source,
+      position: new Position(line, column)
+    };
+  }
+
+  /**
+   * Clears the cached sourcemap and its corresponding `TextDocument`.
+   */
+  private clearCache(): void {
+    if (this.sourceMap !== undefined) {
+      this.sourceMap.destroy();
+      this.sourceMap = undefined;
+      this.lastDocument = undefined;
+    }
+  }
+
+  /**
+   * Updates the `codeql.hasQLSource` context variable based on the current selection. This variable
+   * controls whether or not the `codeQL.gotoQL` command is enabled.
+   */
+  private async updateContext(): Promise<void> {
+    const position = await this.getQLSourceLocation();
+
+    await commands.executeCommand('setContext', 'codeql.hasQLSource', position !== undefined);
+  }
+
+  handleDidChangeActiveTextEditor = async (_editor: TextEditor | undefined): Promise<void> => {
+    await this.updateContext();
+  }
+
+  handleDidChangeTextEditorSelection = async (_e: TextEditorSelectionChangeEvent): Promise<void> => {
+    await this.updateContext();
+  }
+
+  handleDidCloseTextDocument = (document: TextDocument): void => {
+    if (this.lastDocument === document) {
+      this.clearCache();
+    }
+  }
+
+  handleGotoQL = async (): Promise<void> => {
+    const position = await this.getQLSourceLocation();
+    if (position !== undefined) {
+      await showSourceLocation(position);
+    }
+  };
+}

extensions/ql-vscode/src/pure/log-summary-parser.ts

@@ -1,10 +1,10 @@
 // TODO(angelapwen): Only load in necessary information and
-// location in bytes for this log to save memory. 
+// location in bytes for this log to save memory.
 export interface EvalLogData {
   predicateName: string;
   millis: number;
   resultSize: number;
-  // Key: pipeline identifier; Value: array of pipeline steps 
+  // Key: pipeline identifier; Value: array of pipeline steps
   ra: Record<string, string[]>;
 }
 

extensions/ql-vscode/src/pure/messages.ts

@@ -155,6 +155,10 @@ export interface CompilationOptions {
    * get reported anyway. Useful for universal compilation options.
    */
   computeDefaultStrings: boolean;
+  /**
+   * Emit debug information in compiled query.
+   */
+  emitDebugInfo: boolean;
 }
 
 /**

extensions/ql-vscode/src/run-queries.ts

@@ -241,7 +241,8 @@ export class QueryEvaluationInfo {
           localChecking: false,
           noComputeGetUrl: false,
           noComputeToString: false,
-          computeDefaultStrings: true
+          computeDefaultStrings: true,
+          emitDebugInfo: true
         },
         extraOptions: {
           timeoutSecs: qs.config.timeoutSecs

extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts

@@ -151,7 +151,8 @@ describe('using the query server', function() {
             localChecking: false,
             noComputeGetUrl: false,
             noComputeToString: false,
-            computeDefaultStrings: true
+            computeDefaultStrings: true,
+            emitDebugInfo: true
           },
           queryToCheck: qlProgram,
           resultPath: COMPILED_QUERY_PATH,

extensions/ql-vscode/src/vscode-tests/cli-integration/sourcemap.test.ts

@@ -0,0 +1,66 @@
+import { fail } from 'assert';
+import { commands, Selection, window, workspace } from 'vscode';
+import * as path from 'path';
+import * as assert from 'assert';
+import { expect } from 'chai';
+import { tmpDir } from '../../helpers';
+import * as fs from 'fs-extra';
+
+/**
+ * Integration tests for queries
+ */
+describe('SourceMap', function() {
+  this.timeout(20000);
+
+  it('should jump to QL code', async () => {
+    try {
+      const root = workspace.workspaceFolders![0].uri.fsPath;
+      const srcFiles = {
+        summary: path.join(root, 'log-summary', 'evaluator-log.summary'),
+        summaryMap: path.join(root, 'log-summary', 'evaluator-log.summary.map')
+      };
+      // We need to modify the source map so that its paths point to the actual location of the
+      // workspace root on this machine. We'll copy the summary and its source map to a temp
+      // directory, modify the source map their, and open that summary.
+      const tempFiles = await copyFilesToTempDirectory(srcFiles);
+
+      // The checked-in sourcemap has placeholders of the form `${root}`, which we need to replace
+      // with the actual root directory.
+      const mapText = await fs.readFile(tempFiles.summaryMap, 'utf-8');
+      // Always use forward slashes, since they work everywhere.
+      const slashRoot = root.replaceAll('\\', '/');
+      const newMapText = mapText.replaceAll('${root}', slashRoot);
+      await fs.writeFile(tempFiles.summaryMap, newMapText);
+
+      const summaryDocument = await workspace.openTextDocument(tempFiles.summary);
+      assert(summaryDocument.languageId === 'ql-summary');
+      const summaryEditor = await window.showTextDocument(summaryDocument);
+      summaryEditor.selection = new Selection(356, 10, 356, 10);
+      await commands.executeCommand('codeQL.gotoQL');
+
+      const newEditor = window.activeTextEditor;
+      expect(newEditor).to.be.not.undefined;
+      const newDocument = newEditor!.document;
+      expect(path.basename(newDocument.fileName)).to.equal('Namespace.qll');
+      const newSelection = newEditor!.selection;
+      expect(newSelection.start.line).to.equal(60);
+      expect(newSelection.start.character).to.equal(2);
+    } catch (e) {
+      console.error('Test Failed');
+      fail(e as Error);
+    }
+  });
+
+  async function copyFilesToTempDirectory<T extends Record<string, string>>(files: T): Promise<T> {
+    const tempDir = path.join(tmpDir.name, 'log-summary');
+    await fs.ensureDir(tempDir);
+    const result: Record<string, string> = {};
+    for (const [key, srcPath] of Object.entries(files)) {
+      const destPath = path.join(tempDir, path.basename(srcPath));
+      await fs.copy(srcPath, destPath);
+      result[key] = destPath;
+    }
+
+    return result as T;
+  }
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts

@@ -171,7 +171,8 @@ describe('run-queries', () => {
             localChecking: false,
             noComputeGetUrl: false,
             noComputeToString: false,
-            computeDefaultStrings: true
+            computeDefaultStrings: true,
+            emitDebugInfo: true,
           },
           extraOptions: {
             timeoutSecs: 5

extensions/ql-vscode/test/data/lib/semmle/code/cpp/Namespace.qll

@@ -0,0 +1,237 @@
+/**
+ * This code isn't compiled. It's just the target for the sourcemap tests that jump from RA to QL.
+ */
+
+import semmle.code.cpp.Element
+import semmle.code.cpp.Type
+import semmle.code.cpp.metrics.MetricNamespace
+
+/**
+ * A C++ namespace. For example the (single) namespace `A` in the following
+ * code:
+ * ```
+ * namespace A
+ * {
+ *   // ...
+ * }
+ *
+ * // ...
+ *
+ * namespace A
+ * {
+ *   // ...
+ * }
+ * ```
+ * Note that namespaces are somewhat nebulous entities, as they do not in
+ * general have a single well-defined location in the source code. The
+ * related notion of a `NamespaceDeclarationEntry` is rather more concrete,
+ * and should be used when a location is required. For example, the `std::`
+ * namespace is particularly nebulous, as parts of it are defined across a
+ * wide range of headers. As a more extreme example, the global namespace
+ * is never explicitly declared, but might correspond to a large proportion
+ * of the source code.
+ */
+class Namespace extends NameQualifyingElement, @namespace {
+  /**
+   * Gets the location of the namespace. Most namespaces do not have a
+   * single well-defined source location, so a dummy location is returned,
+   * unless the namespace has exactly one declaration entry.
+   */
+  override Location getLocation() {
+    if strictcount(this.getADeclarationEntry()) = 1
+    then result = this.getADeclarationEntry().getLocation()
+    else result instanceof UnknownDefaultLocation
+  }
+
+  /** Gets the simple name of this namespace. */
+  override string getName() { namespaces(underlyingElement(this), result) }
+
+  /** Holds if this element is named `name`. */
+  predicate hasName(string name) { name = this.getName() }
+
+  /** Holds if this namespace is anonymous. */
+  predicate isAnonymous() { this.hasName("(unnamed namespace)") }
+
+  /** Gets the name of the parent namespace, if it exists. */
+  private string getParentName() {
+    result = this.getParentNamespace().getName() and
+    result != ""
+  }
+
+  /** Gets the qualified name of this namespace. For example: `a::b`. */
+  string getQualifiedName() {
+    if exists(this.getParentName())
+    then result = this.getParentNamespace().getQualifiedName() + "::" + this.getName()
+    else result = this.getName()
+  }
+
+  /** Gets the parent namespace, if any. */
+  Namespace getParentNamespace() {
+    namespacembrs(unresolveElement(result), underlyingElement(this))
+    or
+    not namespacembrs(_, underlyingElement(this)) and result instanceof GlobalNamespace
+  }
+
+  /** Gets a child declaration of this namespace. */
+  Declaration getADeclaration() { namespacembrs(underlyingElement(this), unresolveElement(result)) }
+
+  /** Gets a child namespace of this namespace. */
+  Namespace getAChildNamespace() {
+    namespacembrs(underlyingElement(this), unresolveElement(result))
+  }
+
+  /** Holds if the namespace is inline. */
+  predicate isInline() { namespace_inline(underlyingElement(this)) }
+
+  /** Holds if this namespace may be from source. */
+  override predicate fromSource() { this.getADeclaration().fromSource() }
+
+  /** Gets the metric namespace. */
+  MetricNamespace getMetrics() { result = this }
+
+  /** Gets a version of the `QualifiedName` that is more suitable for display purposes. */
+  string getFriendlyName() { result = this.getQualifiedName() }
+
+  final override string toString() { result = this.getFriendlyName() }
+
+  /** Gets a declaration of (part of) this namespace. */
+  NamespaceDeclarationEntry getADeclarationEntry() { result.getNamespace() = this }
+
+  /** Gets a file which declares (part of) this namespace. */
+  File getAFile() { result = this.getADeclarationEntry().getLocation().getFile() }
+}
+
+/**
+ * A declaration of (part of) a C++ namespace. This corresponds to a single
+ * `namespace N { ... }` occurrence in the source code. For example the two
+ * mentions of `A` in the following code:
+ * ```
+ * namespace A
+ * {
+ *   // ...
+ * }
+ *
+ * // ...
+ *
+ * namespace A
+ * {
+ *   // ...
+ * }
+ * ```
+ */
+class NamespaceDeclarationEntry extends Locatable, @namespace_decl {
+  /**
+   * Get the namespace that this declaration entry corresponds to.  There
+   * is a one-to-many relationship between `Namespace` and
+   * `NamespaceDeclarationEntry`.
+   */
+  Namespace getNamespace() {
+    namespace_decls(underlyingElement(this), unresolveElement(result), _, _)
+  }
+
+  override string toString() { result = this.getNamespace().getFriendlyName() }
+
+  /**
+   * Gets the location of the token preceding the namespace declaration
+   * entry's body.
+   *
+   * For named declarations, such as "namespace MyStuff { ... }", this will
+   * give the "MyStuff" token.
+   *
+   * For anonymous declarations, such as "namespace { ... }", this will
+   * give the "namespace" token.
+   */
+  override Location getLocation() { namespace_decls(underlyingElement(this), _, result, _) }
+
+  /**
+   * Gets the location of the namespace declaration entry's body. For
+   * example: the "{ ... }" in "namespace N { ... }".
+   */
+  Location getBodyLocation() { namespace_decls(underlyingElement(this), _, _, result) }
+
+  override string getAPrimaryQlClass() { result = "NamespaceDeclarationEntry" }
+}
+
+/**
+ * A C++ `using` directive or `using` declaration.
+ */
+class UsingEntry extends Locatable, @using {
+  override Location getLocation() { usings(underlyingElement(this), _, result) }
+}
+
+/**
+ * A C++ `using` declaration. For example:
+ * ```
+ * using std::string;
+ * ```
+ */
+class UsingDeclarationEntry extends UsingEntry {
+  UsingDeclarationEntry() {
+    not exists(Namespace n | usings(underlyingElement(this), unresolveElement(n), _))
+  }
+
+  /**
+   * Gets the declaration that is referenced by this using declaration. For
+   * example, `std::string` in `using std::string`.
+   */
+  Declaration getDeclaration() { usings(underlyingElement(this), unresolveElement(result), _) }
+
+  override string toString() { result = "using " + this.getDeclaration().getDescription() }
+}
+
+/**
+ * A C++ `using` directive. For example:
+ * ```
+ * using namespace std;
+ * ```
+ */
+class UsingDirectiveEntry extends UsingEntry {
+  UsingDirectiveEntry() {
+    exists(Namespace n | usings(underlyingElement(this), unresolveElement(n), _))
+  }
+
+  /**
+   * Gets the namespace that is referenced by this using directive. For
+   * example, `std` in `using namespace std`.
+   */
+  Namespace getNamespace() { usings(underlyingElement(this), unresolveElement(result), _) }
+
+  override string toString() { result = "using namespace " + this.getNamespace().getFriendlyName() }
+}
+
+/**
+ * Holds if `g` is an instance of `GlobalNamespace`. This predicate
+ * is used suppress a warning in `GlobalNamespace.getADeclaration()`
+ * by providing a fake use of `this`.
+ */
+private predicate suppressWarningForUnused(GlobalNamespace g) { any() }
+
+/**
+ * The C/C++ global namespace.
+ */
+class GlobalNamespace extends Namespace {
+  GlobalNamespace() { this.hasName("") }
+
+  override Declaration getADeclaration() {
+    suppressWarningForUnused(this) and
+    result.isTopLevel() and
+    not namespacembrs(_, unresolveElement(result))
+  }
+
+  /** Gets a child namespace of the global namespace. */
+  override Namespace getAChildNamespace() {
+    suppressWarningForUnused(this) and
+    not namespacembrs(unresolveElement(result), _)
+  }
+
+  override Namespace getParentNamespace() { none() }
+
+  override string getFriendlyName() { result = "(global namespace)" }
+}
+
+/**
+ * The C++ `std::` namespace.
+ */
+class StdNamespace extends Namespace {
+  StdNamespace() { this.hasName("std") and this.getParentNamespace() instanceof GlobalNamespace }
+}