Merge pull request #304 from jcreedcmu/jcreed/v1.1.1

Update CHANGELOG for release.

.editorconfig

@@ -4,3 +4,4 @@ indent_size = 2
 charset = utf-8
 end_of_line = lf
 trim_trailing_whitespace = true
+insert_final_newline = true

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,20 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new-extension-release.md

@@ -0,0 +1,18 @@
+---
+name: New extension release
+about: Create an issue with a checklist for the release steps (write access required
+  for the steps)
+title: Release Checklist for version xx.xx.xx
+labels: ''
+assignees: ''
+
+---
+
+- [ ] Update this issue title to refer to the version of the release
+- [ ] Trigger a release build on Actions by adding a new tag on master of the format `vxx.xx.xx`
+- [ ] Monitor the status of the release build in the `Release` workflow in the Actions tab.
+- [ ] Download the VSIX from the draft GitHub release that is created when the release build finishes.
+- [ ] Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
+- [ ] Click the `...` menu in the CodeQL row and click **Update**.
+- [ ] Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
+- [ ] Publish the draft GitHub release and confirm the new release is marked as the latest release at https://github.com/github/vscode-codeql/releases.

.github/pull_request_template.md

@@ -0,0 +1,12 @@
+<!-- Thank you for submitting a pull request. Please read our pull request guidelines before
+  submitting your pull request:
+  https://github.com/github/vscode-codeql/blob/master/CONTRIBUTING.md#submitting-a-pull-request.
+-->
+
+Replace this with a description of the changes your pull request makes.
+
+## Checklist
+
+- [ ] [CHANGELOG.md](../extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
+- [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
+- [ ] `@github/product-docs-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/main.yml

@@ -10,10 +10,14 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.18.1'
+
       - name: Build
         run: |
           cd build
@@ -42,10 +46,14 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.18.1'
+
       # We have to build the dependencies in `lib` before running any tests.
       - name: Build
         run: |
@@ -54,9 +62,25 @@ jobs:
           npm run build-ci
         shell: bash
 
-      - name: Run unit tests
+      - name: Install CodeQL
+        run: |
+          mkdir codeql-home
+          curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
+          unzip -q -o codeql-home/codeql.zip -d codeql-home
+          rm codeql-home/codeql.zip
+        shell: bash
+
+      - name: Run unit tests (Linux)
+        if: matrix.os == 'ubuntu-latest'
         run: |
           cd extensions/ql-vscode
+          CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
+
+      - name: Run unit tests (Windows)
+        if: matrix.os == 'windows-latest'
+        run: |
+          cd extensions/ql-vscode
+          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
@@ -70,4 +94,4 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: |
           cd extensions/ql-vscode
-          npm run integration
+          npm run integration

.github/workflows/release.yml

@@ -27,7 +27,11 @@ jobs:
     # TODO Share steps with the main workflow.
     steps:
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.18.1'
 
       - name: Build
         run: |
@@ -47,8 +51,8 @@ jobs:
           VSIX_PATH="$(ls dist/*.vsix)"
           echo "::set-output name=vsix_path::$VSIX_PATH"
           # Transform the GitHub ref so it can be used in a filename.
-          # This is mainly needed for testing branches that modify this workflow.
+          # The last sed invocation is used for testing branches that modify this workflow.
-          REF_NAME="$(echo ${{ github.ref }} | sed -e 's:/:-:g')"
+          REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
       # Uploading artifacts is not necessary to create a release.
@@ -87,4 +91,32 @@ jobs:
           # Get the `vsix_path` and `ref_name` from the `prepare-artifacts` step above.
           asset_path: ${{ steps.prepare-artifacts.outputs.vsix_path }}
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
-          asset_content_type: application/zip
+          asset_content_type: application/zip
+
+      # The checkout action does not fetch the master branch.
+      # Fetch the master branch so that we can base the version bump PR against master.
+      - name: Fetch master branch
+        run: |
+          git fetch --depth=1 origin master:master
+          git checkout master
+
+      - name: Bump patch version
+        id: bump-patch-version
+        if: success()
+        run: |
+          cd extensions/ql-vscode
+          # Bump to the next patch version. Major or minor version bumps will have to be done manually.
+          # Record the next version number as an output of this step.
+          NEXT_VERSION="$(npm version patch)"
+          echo "::set-output name=next_version::$NEXT_VERSION"
+
+      - name: Create version bump PR
+        uses: peter-evans/create-pull-request@c7b64af0a489eae91f7890f2c1b63d13cc2d8ab7 # v2.4.2
+        if: success()
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Bump version to ${{ steps.bump-patch-version.outputs.next_version }}
+          title: Bump version to ${{ steps.bump-patch-version.outputs.next_version }}
+          body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
+          branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
+          base: master

.gitignore

@@ -7,6 +7,7 @@ out/
 server/
 node_modules/
 gen/
+artifacts/
 
 # Integration test artifacts
 **/.vscode-test/**
@@ -17,3 +18,4 @@ gen/
 # Rush files
 /common/temp/**
 package-deps.json
+**/.rush/temp

.vscode/launch.json

@@ -9,7 +9,7 @@
       "runtimeExecutable": "${execPath}",
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--disable-extensions"
+        "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -24,25 +24,30 @@
     },
     {
       "name": "Launch Unit Tests (vscode-codeql)",
-      "type": "extensionHost",
+      "type": "node",
       "request": "launch",
-      "runtimeExecutable": "${execPath}",
+      "program": "${workspaceFolder}/extensions/ql-vscode/node_modules/mocha/bin/_mocha",
-      "args": [
+      "showAsyncStacks": true,
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/test",
+      "runtimeArgs": [
-        "--disable-extensions"
+        "--inspect=9229"
       ],
+      "args": [
+        "--exit",
+        "-u",
+        "bdd",
+        "--colors",
+        "--diff",
+        "-r",
+        "ts-node/register",
+        "test/pure-tests/**/*.ts"
+      ],
+      "port": 9229,
       "stopOnEntry": false,
       "sourceMaps": true,
-      "outFiles": [
+      "preLaunchTask": "Build",
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
+      "console": "integratedTerminal",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-bqrs/out/**/*.js",
+      "internalConsoleOptions": "neverOpen"
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io-node/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-vscode-utils/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/test/**/*.js"
-      ],
-      "preLaunchTask": "Build"
     },
     {
       "name": "Launch Integration Tests - No Workspace (vscode-codeql)",
@@ -51,8 +56,7 @@
       "runtimeExecutable": "${execPath}",
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index"
-        "--disable-extensions"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -70,7 +74,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
-        "${workspaceRoot}/extensions/ql-vscode/test/data",
+        "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -81,4 +85,4 @@
       "preLaunchTask": "Build"
     }
   ]
-}
+}

.vscode/settings.json

@@ -1,14 +1,36 @@
 // Place your settings in this file to overwrite default and user settings.
 {
   "files.exclude": {
-    "out": false // set this to true to hide the "out" folder with the compiled JS files
+    "**/out": true, // set this to true to hide the "out" folder with the compiled JS files
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
   "files.watcherExclude": {
     "**/.git/**": true,
-    "**/node_modules/*/**": true
+    "**/out": true,
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
   "search.exclude": {
-    "out": true // set this to false to include "out" folder in search results
+    "**/out": true, // set this to false to include "out" folder in search results
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
-  "typescript.tsdk": "./common/temp/node_modules/typescript/lib" // we want to use the TS server from our node_modules folder to control its version
+  "typescript.tsdk": "./common/temp/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
-}
+  "eslint.validate": [
+    "javascript",
+    "javascriptreact",
+    "typescript",
+    "typescriptreact"
+  ],
+  "eslint.options": {
+    // This is necessary so that eslint can properly resolve its plugins
+    "resolvePluginsRelativeTo": "./extensions/ql-vscode"
+  }
+}

.vscode/tasks.json

@@ -28,7 +28,7 @@
             "file": 1,
             "location": 2,
             "message": 3
-          },
+          }
         },
         "$ts-webpack"
       ]
@@ -100,6 +100,15 @@
         "clear": true
       },
       "problemMatcher": []
+    },
+    {
+      "type": "npm",
+      "script": "watch",
+      "path": "extensions/ql-vscode/",
+      "problemMatcher": [
+        "$gulp-tsc"
+      ],
+      "group": "build"
     }
   ]
 }

CHANGELOG.md

@@ -1,14 +0,0 @@
-# CodeQL for Visual Studio Code: Changelog
-
-## 1.0.1 - 21 November 2019
-
-- Change `codeQL.cli.executablePath` to a per-machine setting, so it can no longer be set at the user or workspace level. This helps prevent arbitrary code execution when using a VS Code workspace from an untrusted source.
-- Improve the highlighting of the selected query result within the source code.
-- Improve the performance of switching between result tables in the CodeQL Query Results view.
-- Fix the automatic upgrading of CodeQL databases when using upgrade scripts from the workspace.
-- Allow removal of items from the CodeQL Query History view.
-
-
-## 1.0.0 - 14 November 2019
-
-Initial release of CodeQL for Visual Studio Code.

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-## Contributing
+# Contributing
 
 [fork]: https://github.com/github/vscode-codeql/fork
 [pr]: https://github.com/github/vscode-codeql/compare
@@ -13,19 +13,19 @@ Please note that this project is released with a [Contributor Code of Conduct][c
 
 ## Submitting a pull request
 
-0. [Fork][fork] and clone the repository
+1. [Fork][fork] and clone the repository
-0. Set up a local build
+1. Set up a local build
-0. Create a new branch: `git checkout -b my-branch-name`
+1. Create a new branch: `git checkout -b my-branch-name`
-0. Make your change
+1. Make your change
-0. Push to your fork and [submit a pull request][pr]
+1. Push to your fork and [submit a pull request][pr]
-0. Pat yourself on the back and wait for your pull request to be reviewed and merged.
+1. Pat yourself on the back and wait for your pull request to be reviewed and merged.
 
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 
-- Follow the [style guide][style].
+* Follow the [style guide][style].
-- Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
+* Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
-- Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
+* Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
@@ -42,12 +42,22 @@ If you plan on building from the command line, it's easiest if Rush is installed
 npm install -g @microsoft/rush
 ```
 
+To get started, run:
+
+```shell
+rush update && rush build
+```
+
 Note that when you run the `rush` command from the globally installed version, it will examine the
 `rushVersion` property in the repo's `rush.json`, and if it differs from the globally installed
 version, it will download, cache, and run the version of Rush specified in the `rushVersion`
 property.
 
-If you plan on only building via VS Code tasks, you don't need Rush installed at all, since those
+A few more things to know about using rush:
+
+* Avoid running `npm` for any commands that install/link dependencies
+* Instead use the *rush* equivalent: `rush add <package>`, `rush update`, etc.
+* If you plan on only building via VS Code tasks, you don't need Rush installed at all, since those
 tasks run `common/scripts/install-run-rush.js` to bootstrap a locally installed and cached copy of
 Rush.
 
@@ -66,7 +76,7 @@ a single-project repo. With Rush, you need to do an "update" instead:
 ##### From the command line
 
 ```shell
-$ rush update
+rush update
 ```
 
 #### Building all projects (instead of `gulp`)
@@ -100,6 +110,8 @@ force a full rebuild of all projects:
 rush rebuild --verbose
 ```
 
+Note that `rush rebuild` performs a complete rebuild, whereas `rush build` performs an incremental build and in many cases will not need to do anything at all.
+
 ### Installing
 
 You can install the `.vsix` file from within VS Code itself, from the Extensions container in the sidebar:
@@ -118,18 +130,40 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
+### Running the unit/integration tests
+
+Ensure the `CODEQL_PATH` environment variable is set to point to the `codeql` cli executable.
+
+Outside of vscode, run:
+
+```shell
+npm run test && npm run integration
+```
+
+Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
+
 ## Releasing (write access required)
 
+1. Double-check the `CHANGELOG.md` contains all desired change comments
+   and has the version to be released with date at the top.
+1. Double-check that the extension `package.json` has the version you intend to release.
+   If you are doing a patch release (as opposed to minor or major version) this should already
+   be correct.
 1. Trigger a release build on Actions by adding a new tag on master of the format `vxx.xx.xx`
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
-1. Download the VSIX from the draft GitHub release that is created when the release build finishes.
+1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
+1. Optionally unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
+   or look at the source if there's any doubt the right code is being shipped.
 1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
 1. Click the `...` menu in the CodeQL row and click **Update**.
 1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Publish the GitHub release.
+1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
+1. If documentation changes need to be published, notify documentation team that release has been made.
+1. Review and merge the version bump PR that is automatically created by Actions.
 
 ## Resources
 
-- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+* [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
-- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+* [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
-- [GitHub Help](https://help.github.com)
+* [GitHub Help](https://help.github.com)

README.md

@@ -4,7 +4,10 @@ This project is an extension for Visual Studio Code that adds rich language supp
 
 The extension is released. You can download it from the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql).
 
-![CI status badge](https://github.com/github/vscode-codeql/workflows/Build%20Extension/badge.svg)
+To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
+
+[![CI status badge](https://github.com/github/vscode-codeql/workflows/Build%20Extension/badge.svg)](https://github.com/github/vscode-codeql/actions?query=workflow%3A%22Build+Extension%22+branch%3Amaster)
+[![VS Marketplace badge](https://vsmarketplacebadge.apphb.com/version/github.vscode-codeql.svg)](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql)
 
 ## Features
 

build/package.json

@@ -14,4 +14,4 @@
     "build-release": "rush install && rush build --release"
   },
   "author": "GitHub"
-}
+}

common/scripts/install-run-rush.js

@@ -1,52 +1,67 @@
-"use strict";
+"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
+// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
+Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
+// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
+//
-// This script is intended for usage in an automated build environment where the Rush command may not have
+// This script is intended for usage in an automated build environment where the Rush command may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
+// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
-// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it.
+// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it.
-// An example usage would be:
+// An example usage would be:
-//
+//
-//    node common/scripts/install-run-rush.js install
+//    node common/scripts/install-run-rush.js install
-//
+//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
+// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const path = require("path");
+const path = require("path");
-const fs = require("fs");
+const fs = require("fs");
-const install_run_1 = require("./install-run");
+const install_run_1 = require("./install-run");
-const PACKAGE_NAME = '@microsoft/rush';
+const PACKAGE_NAME = '@microsoft/rush';
-function getRushVersion() {
+const RUSH_PREVIEW_VERSION = 'RUSH_PREVIEW_VERSION';
-    const rushJsonFolder = install_run_1.findRushJsonFolder();
+function _getRushVersion() {
-    const rushJsonPath = path.join(rushJsonFolder, install_run_1.RUSH_JSON_FILENAME);
+    const rushPreviewVersion = process.env[RUSH_PREVIEW_VERSION];
-    try {
+    if (rushPreviewVersion !== undefined) {
-        const rushJsonContents = fs.readFileSync(rushJsonPath, 'utf-8');
+        console.log(`Using Rush version from environment variable ${RUSH_PREVIEW_VERSION}=${rushPreviewVersion}`);
-        // Use a regular expression to parse out the rushVersion value because rush.json supports comments,
+        return rushPreviewVersion;
-        // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script.
+    }
-        const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/);
+    const rushJsonFolder = install_run_1.findRushJsonFolder();
-        return rushJsonMatches[1];
+    const rushJsonPath = path.join(rushJsonFolder, install_run_1.RUSH_JSON_FILENAME);
-    }
+    try {
-    catch (e) {
+        const rushJsonContents = fs.readFileSync(rushJsonPath, 'utf-8');
-        throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` +
+        // Use a regular expression to parse out the rushVersion value because rush.json supports comments,
-            'The \'rushVersion\' field is either not assigned in rush.json or was specified ' +
+        // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script.
-            'using an unexpected syntax.');
+        const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/);
-    }
+        return rushJsonMatches[1];
-}
+    }
-function run() {
+    catch (e) {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ ...packageBinArgs /* [build, --to, myproject] */] = process.argv;
+        throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` +
-    if (!nodePath || !scriptPath) {
+            'The \'rushVersion\' field is either not assigned in rush.json or was specified ' +
-        throw new Error('Unexpected exception: could not detect node path or script path');
+            'using an unexpected syntax.');
-    }
+    }
-    if (process.argv.length < 3) {
+}
-        console.log('Usage: install-run-rush.js <command> [args...]');
+function _run() {
-        console.log('Example: install-run-rush.js build --to myproject');
+    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ ...packageBinArgs /* [build, --to, myproject] */] = process.argv;
-        process.exit(1);
+    // Detect if this script was directly invoked, or if the install-run-rushx script was invokved to select the
-    }
+    // appropriate binary inside the rush package to run
-    install_run_1.runWithErrorAndStatusCode(() => {
+    const scriptName = path.basename(scriptPath);
-        const version = getRushVersion();
+    const bin = scriptName.toLowerCase() === 'install-run-rushx.js' ? 'rushx' : 'rush';
-        console.log(`The rush.json configuration requests Rush version ${version}`);
+    if (!nodePath || !scriptPath) {
-        return install_run_1.installAndRun(PACKAGE_NAME, version, 'rush', packageBinArgs);
+        throw new Error('Unexpected exception: could not detect node path or script path');
-    });
+    }
-}
+    if (process.argv.length < 3) {
-run();
+        console.log(`Usage: ${scriptName} <command> [args...]`);
+        if (scriptName === 'install-run-rush.js') {
+            console.log(`Example: ${scriptName} build --to myproject`);
+        }
+        else {
+            console.log(`Example: ${scriptName} custom-command`);
+        }
+        process.exit(1);
+    }
+    install_run_1.runWithErrorAndStatusCode(() => {
+        const version = _getRushVersion();
+        console.log(`The rush.json configuration requests Rush version ${version}`);
+        return install_run_1.installAndRun(PACKAGE_NAME, version, bin, packageBinArgs);
+    });
+}
+_run();
 //# sourceMappingURL=install-run-rush.js.map

common/scripts/install-run-rushx.js

@@ -0,0 +1,18 @@
+"use strict";
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See the @microsoft/rush package's LICENSE file for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
+//
+// This script is intended for usage in an automated build environment where the Rush command may not have
+// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
+// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the
+// rushx command.
+//
+// An example usage would be:
+//
+//    node common/scripts/install-run-rushx.js custom-command
+//
+// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
+require("./install-run-rush");
+//# sourceMappingURL=install-run-rushx.js.map

common/scripts/install-run.js

@@ -1,399 +1,433 @@
-"use strict";
+"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
+// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
+Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
+// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
+//
-// This script is intended for usage in an automated build environment where a Node tool may not have
+// This script is intended for usage in an automated build environment where a Node tool may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the specified
+// been preinstalled, or may have an unpredictable version.  This script will automatically install the specified
-// version of the specified tool (if not already installed), and then pass a command-line to it.
+// version of the specified tool (if not already installed), and then pass a command-line to it.
-// An example usage would be:
+// An example usage would be:
-//
+//
-//    node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io
+//    node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io
-//
+//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
+// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const childProcess = require("child_process");
+const childProcess = require("child_process");
-const fs = require("fs");
+const fs = require("fs");
-const os = require("os");
+const os = require("os");
-const path = require("path");
+const path = require("path");
-exports.RUSH_JSON_FILENAME = 'rush.json';
+exports.RUSH_JSON_FILENAME = 'rush.json';
-const INSTALLED_FLAG_FILENAME = 'installed.flag';
+const RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME = 'RUSH_TEMP_FOLDER';
-const NODE_MODULES_FOLDER_NAME = 'node_modules';
+const INSTALLED_FLAG_FILENAME = 'installed.flag';
-const PACKAGE_JSON_FILENAME = 'package.json';
+const NODE_MODULES_FOLDER_NAME = 'node_modules';
-/**
+const PACKAGE_JSON_FILENAME = 'package.json';
- * Parse a package specifier (in the form of name\@version) into name and version parts.
+/**
- */
+ * Parse a package specifier (in the form of name\@version) into name and version parts.
-function parsePackageSpecifier(rawPackageSpecifier) {
+ */
-    rawPackageSpecifier = (rawPackageSpecifier || '').trim();
+function _parsePackageSpecifier(rawPackageSpecifier) {
-    const separatorIndex = rawPackageSpecifier.lastIndexOf('@');
+    rawPackageSpecifier = (rawPackageSpecifier || '').trim();
-    let name;
+    const separatorIndex = rawPackageSpecifier.lastIndexOf('@');
-    let version = undefined;
+    let name;
-    if (separatorIndex === 0) {
+    let version = undefined;
-        // The specifier starts with a scope and doesn't have a version specified
+    if (separatorIndex === 0) {
-        name = rawPackageSpecifier;
+        // The specifier starts with a scope and doesn't have a version specified
-    }
+        name = rawPackageSpecifier;
-    else if (separatorIndex === -1) {
+    }
-        // The specifier doesn't have a version
+    else if (separatorIndex === -1) {
-        name = rawPackageSpecifier;
+        // The specifier doesn't have a version
-    }
+        name = rawPackageSpecifier;
-    else {
+    }
-        name = rawPackageSpecifier.substring(0, separatorIndex);
+    else {
-        version = rawPackageSpecifier.substring(separatorIndex + 1);
+        name = rawPackageSpecifier.substring(0, separatorIndex);
-    }
+        version = rawPackageSpecifier.substring(separatorIndex + 1);
-    if (!name) {
+    }
-        throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`);
+    if (!name) {
-    }
+        throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`);
-    return { name, version };
+    }
-}
+    return { name, version };
-/**
+}
- * Resolve a package specifier to a static version
+/**
- */
+ * As a workaround, copyAndTrimNpmrcFile() copies the .npmrc file to the target folder, and also trims
-function resolvePackageVersion(rushCommonFolder, { name, version }) {
+ * unusable lines from the .npmrc file.
-    if (!version) {
+ *
-        version = '*'; // If no version is specified, use the latest version
+ * Why are we trimming the .npmrc lines?  NPM allows environment variables to be specified in
-    }
+ * the .npmrc file to provide different authentication tokens for different registry.
-    if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) {
+ * However, if the environment variable is undefined, it expands to an empty string, which
-        // If the version contains only characters that we recognize to be used in static version specifiers,
+ * produces a valid-looking mapping with an invalid URL that causes an error.  Instead,
-        // pass the version through
+ * we'd prefer to skip that line and continue looking in other places such as the user's
-        return version;
+ * home directory.
-    }
+ *
-    else {
+ * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._copyNpmrcFile()
-        // version resolves to
+ */
-        try {
+function _copyAndTrimNpmrcFile(sourceNpmrcPath, targetNpmrcPath) {
-            const rushTempFolder = ensureAndJoinPath(rushCommonFolder, 'temp');
+    console.log(`Copying ${sourceNpmrcPath} --> ${targetNpmrcPath}`); // Verbose
-            const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
+    let npmrcFileLines = fs.readFileSync(sourceNpmrcPath).toString().split('\n');
-            syncNpmrc(sourceNpmrcFolder, rushTempFolder);
+    npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim());
-            const npmPath = getNpmPath();
+    const resultLines = [];
-            // This returns something that looks like:
+    // Trim out lines that reference environment variables that aren't defined
-            //  @microsoft/rush@3.0.0 '3.0.0'
+    for (const line of npmrcFileLines) {
-            //  @microsoft/rush@3.0.1 '3.0.1'
+        // This finds environment variable tokens that look like "${VAR_NAME}"
-            //  ...
+        const regex = /\$\{([^\}]+)\}/g;
-            //  @microsoft/rush@3.0.20 '3.0.20'
+        const environmentVariables = line.match(regex);
-            //  <blank line>
+        let lineShouldBeTrimmed = false;
-            const npmVersionSpawnResult = childProcess.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], {
+        if (environmentVariables) {
-                cwd: rushTempFolder,
+            for (const token of environmentVariables) {
-                stdio: []
+                // Remove the leading "${" and the trailing "}" from the token
-            });
+                const environmentVariableName = token.substring(2, token.length - 1);
-            if (npmVersionSpawnResult.status !== 0) {
+                if (!process.env[environmentVariableName]) {
-                throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`);
+                    lineShouldBeTrimmed = true;
-            }
+                    break;
-            const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString();
+                }
-            const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line);
+            }
-            const latestVersion = versionLines[versionLines.length - 1];
+        }
-            if (!latestVersion) {
+        if (lineShouldBeTrimmed) {
-                throw new Error('No versions found for the specified version range.');
+            // Example output:
-            }
+            // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}"
-            const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/);
+            resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line);
-            if (!versionMatches) {
+        }
-                throw new Error(`Invalid npm output ${latestVersion}`);
+        else {
-            }
+            resultLines.push(line);
-            return versionMatches[1];
+        }
-        }
+    }
-        catch (e) {
+    fs.writeFileSync(targetNpmrcPath, resultLines.join(os.EOL));
-            throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`);
+}
-        }
+/**
-    }
+ * syncNpmrc() copies the .npmrc file to the target folder, and also trims unusable lines from the .npmrc file.
-}
+ * If the source .npmrc file not exist, then syncNpmrc() will delete an .npmrc that is found in the target folder.
-let _npmPath = undefined;
+ *
-/**
+ * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc()
- * Get the absolute path to the npm executable
+ */
- */
+function _syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder, useNpmrcPublish) {
-function getNpmPath() {
+    const sourceNpmrcPath = path.join(sourceNpmrcFolder, !useNpmrcPublish ? '.npmrc' : '.npmrc-publish');
-    if (!_npmPath) {
+    const targetNpmrcPath = path.join(targetNpmrcFolder, '.npmrc');
-        try {
+    try {
-            if (os.platform() === 'win32') {
+        if (fs.existsSync(sourceNpmrcPath)) {
-                // We're on Windows
+            _copyAndTrimNpmrcFile(sourceNpmrcPath, targetNpmrcPath);
-                const whereOutput = childProcess.execSync('where npm', { stdio: [] }).toString();
+        }
-                const lines = whereOutput.split(os.EOL).filter((line) => !!line);
+        else if (fs.existsSync(targetNpmrcPath)) {
-                // take the last result, we are looking for a .cmd command
+            // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target
-                // see https://github.com/Microsoft/web-build-tools/issues/759
+            console.log(`Deleting ${targetNpmrcPath}`); // Verbose
-                _npmPath = lines[lines.length - 1];
+            fs.unlinkSync(targetNpmrcPath);
-            }
+        }
-            else {
+    }
-                // We aren't on Windows - assume we're on *NIX or Darwin
+    catch (e) {
-                _npmPath = childProcess.execSync('which npm', { stdio: [] }).toString();
+        throw new Error(`Error syncing .npmrc file: ${e}`);
-            }
+    }
-        }
+}
-        catch (e) {
+let _npmPath = undefined;
-            throw new Error(`Unable to determine the path to the NPM tool: ${e}`);
+/**
-        }
+ * Get the absolute path to the npm executable
-        _npmPath = _npmPath.trim();
+ */
-        if (!fs.existsSync(_npmPath)) {
+function getNpmPath() {
-            throw new Error('The NPM executable does not exist');
+    if (!_npmPath) {
-        }
+        try {
-    }
+            if (os.platform() === 'win32') {
-    return _npmPath;
+                // We're on Windows
-}
+                const whereOutput = childProcess.execSync('where npm', { stdio: [] }).toString();
-exports.getNpmPath = getNpmPath;
+                const lines = whereOutput.split(os.EOL).filter((line) => !!line);
-let _rushJsonFolder;
+                // take the last result, we are looking for a .cmd command
-/**
+                // see https://github.com/microsoft/rushstack/issues/759
- * Find the absolute path to the folder containing rush.json
+                _npmPath = lines[lines.length - 1];
- */
+            }
-function findRushJsonFolder() {
+            else {
-    if (!_rushJsonFolder) {
+                // We aren't on Windows - assume we're on *NIX or Darwin
-        let basePath = __dirname;
+                _npmPath = childProcess.execSync('which npm', { stdio: [] }).toString();
-        let tempPath = __dirname;
+            }
-        do {
+        }
-            const testRushJsonPath = path.join(basePath, exports.RUSH_JSON_FILENAME);
+        catch (e) {
-            if (fs.existsSync(testRushJsonPath)) {
+            throw new Error(`Unable to determine the path to the NPM tool: ${e}`);
-                _rushJsonFolder = basePath;
+        }
-                break;
+        _npmPath = _npmPath.trim();
-            }
+        if (!fs.existsSync(_npmPath)) {
-            else {
+            throw new Error('The NPM executable does not exist');
-                basePath = tempPath;
+        }
-            }
+    }
-        } while (basePath !== (tempPath = path.dirname(basePath))); // Exit the loop when we hit the disk root
+    return _npmPath;
-        if (!_rushJsonFolder) {
+}
-            throw new Error('Unable to find rush.json.');
+exports.getNpmPath = getNpmPath;
-        }
+function _ensureFolder(folderPath) {
-    }
+    if (!fs.existsSync(folderPath)) {
-    return _rushJsonFolder;
+        const parentDir = path.dirname(folderPath);
-}
+        _ensureFolder(parentDir);
-exports.findRushJsonFolder = findRushJsonFolder;
+        fs.mkdirSync(folderPath);
-/**
+    }
- * Create missing directories under the specified base directory, and return the resolved directory.
+}
- *
+/**
- * Does not support "." or ".." path segments.
+ * Create missing directories under the specified base directory, and return the resolved directory.
- * Assumes the baseFolder exists.
+ *
- */
+ * Does not support "." or ".." path segments.
-function ensureAndJoinPath(baseFolder, ...pathSegments) {
+ * Assumes the baseFolder exists.
-    let joinedPath = baseFolder;
+ */
-    try {
+function _ensureAndJoinPath(baseFolder, ...pathSegments) {
-        for (let pathSegment of pathSegments) {
+    let joinedPath = baseFolder;
-            pathSegment = pathSegment.replace(/[\\\/]/g, '+');
+    try {
-            joinedPath = path.join(joinedPath, pathSegment);
+        for (let pathSegment of pathSegments) {
-            if (!fs.existsSync(joinedPath)) {
+            pathSegment = pathSegment.replace(/[\\\/]/g, '+');
-                fs.mkdirSync(joinedPath);
+            joinedPath = path.join(joinedPath, pathSegment);
-            }
+            if (!fs.existsSync(joinedPath)) {
-        }
+                fs.mkdirSync(joinedPath);
-    }
+            }
-    catch (e) {
+        }
-        throw new Error(`Error building local installation folder (${path.join(baseFolder, ...pathSegments)}): ${e}`);
+    }
-    }
+    catch (e) {
-    return joinedPath;
+        throw new Error(`Error building local installation folder (${path.join(baseFolder, ...pathSegments)}): ${e}`);
-}
+    }
-/**
+    return joinedPath;
- * As a workaround, _syncNpmrc() copies the .npmrc file to the target folder, and also trims
+}
- * unusable lines from the .npmrc file.  If the source .npmrc file not exist, then _syncNpmrc()
+function _getRushTempFolder(rushCommonFolder) {
- * will delete an .npmrc that is found in the target folder.
+    const rushTempFolder = process.env[RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME];
- *
+    if (rushTempFolder !== undefined) {
- * Why are we trimming the .npmrc lines?  NPM allows environment variables to be specified in
+        _ensureFolder(rushTempFolder);
- * the .npmrc file to provide different authentication tokens for different registry.
+        return rushTempFolder;
- * However, if the environment variable is undefined, it expands to an empty string, which
+    }
- * produces a valid-looking mapping with an invalid URL that causes an error.  Instead,
+    else {
- * we'd prefer to skip that line and continue looking in other places such as the user's
+        return _ensureAndJoinPath(rushCommonFolder, 'temp');
- * home directory.
+    }
- *
+}
- * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc()
+/**
- */
+ * Resolve a package specifier to a static version
-function syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder) {
+ */
-    const sourceNpmrcPath = path.join(sourceNpmrcFolder, '.npmrc');
+function _resolvePackageVersion(rushCommonFolder, { name, version }) {
-    const targetNpmrcPath = path.join(targetNpmrcFolder, '.npmrc');
+    if (!version) {
-    try {
+        version = '*'; // If no version is specified, use the latest version
-        if (fs.existsSync(sourceNpmrcPath)) {
+    }
-            let npmrcFileLines = fs.readFileSync(sourceNpmrcPath).toString().split('\n');
+    if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) {
-            npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim());
+        // If the version contains only characters that we recognize to be used in static version specifiers,
-            const resultLines = [];
+        // pass the version through
-            // Trim out lines that reference environment variables that aren't defined
+        return version;
-            for (const line of npmrcFileLines) {
+    }
-                // This finds environment variable tokens that look like "${VAR_NAME}"
+    else {
-                const regex = /\$\{([^\}]+)\}/g;
+        // version resolves to
-                const environmentVariables = line.match(regex);
+        try {
-                let lineShouldBeTrimmed = false;
+            const rushTempFolder = _getRushTempFolder(rushCommonFolder);
-                if (environmentVariables) {
+            const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-                    for (const token of environmentVariables) {
+            _syncNpmrc(sourceNpmrcFolder, rushTempFolder);
-                        // Remove the leading "${" and the trailing "}" from the token
+            const npmPath = getNpmPath();
-                        const environmentVariableName = token.substring(2, token.length - 1);
+            // This returns something that looks like:
-                        if (!process.env[environmentVariableName]) {
+            //  @microsoft/rush@3.0.0 '3.0.0'
-                            lineShouldBeTrimmed = true;
+            //  @microsoft/rush@3.0.1 '3.0.1'
-                            break;
+            //  ...
-                        }
+            //  @microsoft/rush@3.0.20 '3.0.20'
-                    }
+            //  <blank line>
-                }
+            const npmVersionSpawnResult = childProcess.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], {
-                if (lineShouldBeTrimmed) {
+                cwd: rushTempFolder,
-                    // Example output:
+                stdio: []
-                    // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}"
+            });
-                    resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line);
+            if (npmVersionSpawnResult.status !== 0) {
-                }
+                throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`);
-                else {
+            }
-                    resultLines.push(line);
+            const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString();
-                }
+            const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line);
-            }
+            const latestVersion = versionLines[versionLines.length - 1];
-            fs.writeFileSync(targetNpmrcPath, resultLines.join(os.EOL));
+            if (!latestVersion) {
-        }
+                throw new Error('No versions found for the specified version range.');
-        else if (fs.existsSync(targetNpmrcPath)) {
+            }
-            // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target
+            const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/);
-            fs.unlinkSync(targetNpmrcPath);
+            if (!versionMatches) {
-        }
+                throw new Error(`Invalid npm output ${latestVersion}`);
-    }
+            }
-    catch (e) {
+            return versionMatches[1];
-        throw new Error(`Error syncing .npmrc file: ${e}`);
+        }
-    }
+        catch (e) {
-}
+            throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`);
-/**
+        }
- * Detects if the package in the specified directory is installed
+    }
- */
+}
-function isPackageAlreadyInstalled(packageInstallFolder) {
+let _rushJsonFolder;
-    try {
+/**
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
+ * Find the absolute path to the folder containing rush.json
-        if (!fs.existsSync(flagFilePath)) {
+ */
-            return false;
+function findRushJsonFolder() {
-        }
+    if (!_rushJsonFolder) {
-        const fileContents = fs.readFileSync(flagFilePath).toString();
+        let basePath = __dirname;
-        return fileContents.trim() === process.version;
+        let tempPath = __dirname;
-    }
+        do {
-    catch (e) {
+            const testRushJsonPath = path.join(basePath, exports.RUSH_JSON_FILENAME);
-        return false;
+            if (fs.existsSync(testRushJsonPath)) {
-    }
+                _rushJsonFolder = basePath;
-}
+                break;
-/**
+            }
- * Removes the following files and directories under the specified folder path:
+            else {
- *  - installed.flag
+                basePath = tempPath;
- *  -
+            }
- *  - node_modules
+        } while (basePath !== (tempPath = path.dirname(basePath))); // Exit the loop when we hit the disk root
- */
+        if (!_rushJsonFolder) {
-function cleanInstallFolder(rushCommonFolder, packageInstallFolder) {
+            throw new Error('Unable to find rush.json.');
-    try {
+        }
-        const flagFile = path.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME);
+    }
-        if (fs.existsSync(flagFile)) {
+    return _rushJsonFolder;
-            fs.unlinkSync(flagFile);
+}
-        }
+exports.findRushJsonFolder = findRushJsonFolder;
-        const packageLockFile = path.resolve(packageInstallFolder, 'package-lock.json');
+/**
-        if (fs.existsSync(packageLockFile)) {
+ * Detects if the package in the specified directory is installed
-            fs.unlinkSync(packageLockFile);
+ */
-        }
+function _isPackageAlreadyInstalled(packageInstallFolder) {
-        const nodeModulesFolder = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME);
+    try {
-        if (fs.existsSync(nodeModulesFolder)) {
+        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-            const rushRecyclerFolder = ensureAndJoinPath(rushCommonFolder, 'temp', 'rush-recycler', `install-run-${Date.now().toString()}`);
+        if (!fs.existsSync(flagFilePath)) {
-            fs.renameSync(nodeModulesFolder, rushRecyclerFolder);
+            return false;
-        }
+        }
-    }
+        const fileContents = fs.readFileSync(flagFilePath).toString();
-    catch (e) {
+        return fileContents.trim() === process.version;
-        throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`);
+    }
-    }
+    catch (e) {
-}
+        return false;
-function createPackageJson(packageInstallFolder, name, version) {
+    }
-    try {
+}
-        const packageJsonContents = {
+/**
-            'name': 'ci-rush',
+ * Removes the following files and directories under the specified folder path:
-            'version': '0.0.0',
+ *  - installed.flag
-            'dependencies': {
+ *  -
-                [name]: version
+ *  - node_modules
-            },
+ */
-            'description': 'DON\'T WARN',
+function _cleanInstallFolder(rushTempFolder, packageInstallFolder) {
-            'repository': 'DON\'T WARN',
+    try {
-            'license': 'MIT'
+        const flagFile = path.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        };
+        if (fs.existsSync(flagFile)) {
-        const packageJsonPath = path.join(packageInstallFolder, PACKAGE_JSON_FILENAME);
+            fs.unlinkSync(flagFile);
-        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2));
+        }
-    }
+        const packageLockFile = path.resolve(packageInstallFolder, 'package-lock.json');
-    catch (e) {
+        if (fs.existsSync(packageLockFile)) {
-        throw new Error(`Unable to create package.json: ${e}`);
+            fs.unlinkSync(packageLockFile);
-    }
+        }
-}
+        const nodeModulesFolder = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME);
-/**
+        if (fs.existsSync(nodeModulesFolder)) {
- * Run "npm install" in the package install folder.
+            const rushRecyclerFolder = _ensureAndJoinPath(rushTempFolder, 'rush-recycler', `install-run-${Date.now().toString()}`);
- */
+            fs.renameSync(nodeModulesFolder, rushRecyclerFolder);
-function installPackage(packageInstallFolder, name, version) {
+        }
-    try {
+    }
-        console.log(`Installing ${name}...`);
+    catch (e) {
-        const npmPath = getNpmPath();
+        throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`);
-        const result = childProcess.spawnSync(npmPath, ['install'], {
+    }
-            stdio: 'inherit',
+}
-            cwd: packageInstallFolder,
+function _createPackageJson(packageInstallFolder, name, version) {
-            env: process.env
+    try {
-        });
+        const packageJsonContents = {
-        if (result.status !== 0) {
+            'name': 'ci-rush',
-            throw new Error('"npm install" encountered an error');
+            'version': '0.0.0',
-        }
+            'dependencies': {
-        console.log(`Successfully installed ${name}@${version}`);
+                [name]: version
-    }
+            },
-    catch (e) {
+            'description': 'DON\'T WARN',
-        throw new Error(`Unable to install package: ${e}`);
+            'repository': 'DON\'T WARN',
-    }
+            'license': 'MIT'
-}
+        };
-/**
+        const packageJsonPath = path.join(packageInstallFolder, PACKAGE_JSON_FILENAME);
- * Get the ".bin" path for the package.
+        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2));
- */
+    }
-function getBinPath(packageInstallFolder, binName) {
+    catch (e) {
-    const binFolderPath = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin');
+        throw new Error(`Unable to create package.json: ${e}`);
-    const resolvedBinName = (os.platform() === 'win32') ? `${binName}.cmd` : binName;
+    }
-    return path.resolve(binFolderPath, resolvedBinName);
+}
-}
+/**
-/**
+ * Run "npm install" in the package install folder.
- * Write a flag file to the package's install directory, signifying that the install was successful.
+ */
- */
+function _installPackage(packageInstallFolder, name, version) {
-function writeFlagFile(packageInstallFolder) {
+    try {
-    try {
+        console.log(`Installing ${name}...`);
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
+        const npmPath = getNpmPath();
-        fs.writeFileSync(flagFilePath, process.version);
+        const result = childProcess.spawnSync(npmPath, ['install'], {
-    }
+            stdio: 'inherit',
-    catch (e) {
+            cwd: packageInstallFolder,
-        throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`);
+            env: process.env
-    }
+        });
-}
+        if (result.status !== 0) {
-function installAndRun(packageName, packageVersion, packageBinName, packageBinArgs) {
+            throw new Error('"npm install" encountered an error');
-    const rushJsonFolder = findRushJsonFolder();
+        }
-    const rushCommonFolder = path.join(rushJsonFolder, 'common');
+        console.log(`Successfully installed ${name}@${version}`);
-    const packageInstallFolder = ensureAndJoinPath(rushCommonFolder, 'temp', 'install-run', `${packageName}@${packageVersion}`);
+    }
-    if (!isPackageAlreadyInstalled(packageInstallFolder)) {
+    catch (e) {
-        // The package isn't already installed
+        throw new Error(`Unable to install package: ${e}`);
-        cleanInstallFolder(rushCommonFolder, packageInstallFolder);
+    }
-        const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
+}
-        syncNpmrc(sourceNpmrcFolder, packageInstallFolder);
+/**
-        createPackageJson(packageInstallFolder, packageName, packageVersion);
+ * Get the ".bin" path for the package.
-        installPackage(packageInstallFolder, packageName, packageVersion);
+ */
-        writeFlagFile(packageInstallFolder);
+function _getBinPath(packageInstallFolder, binName) {
-    }
+    const binFolderPath = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin');
-    const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`;
+    const resolvedBinName = (os.platform() === 'win32') ? `${binName}.cmd` : binName;
-    const statusMessageLine = new Array(statusMessage.length + 1).join('-');
+    return path.resolve(binFolderPath, resolvedBinName);
-    console.log(os.EOL + statusMessage + os.EOL + statusMessageLine + os.EOL);
+}
-    const binPath = getBinPath(packageInstallFolder, packageBinName);
+/**
-    const result = childProcess.spawnSync(binPath, packageBinArgs, {
+ * Write a flag file to the package's install directory, signifying that the install was successful.
-        stdio: 'inherit',
+ */
-        cwd: process.cwd(),
+function _writeFlagFile(packageInstallFolder) {
-        env: process.env
+    try {
-    });
+        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-    return result.status;
+        fs.writeFileSync(flagFilePath, process.version);
-}
+    }
-exports.installAndRun = installAndRun;
+    catch (e) {
-function runWithErrorAndStatusCode(fn) {
+        throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`);
-    process.exitCode = 1;
+    }
-    try {
+}
-        const exitCode = fn();
+function installAndRun(packageName, packageVersion, packageBinName, packageBinArgs) {
-        process.exitCode = exitCode;
+    const rushJsonFolder = findRushJsonFolder();
-    }
+    const rushCommonFolder = path.join(rushJsonFolder, 'common');
-    catch (e) {
+    const rushTempFolder = _getRushTempFolder(rushCommonFolder);
-        console.error(os.EOL + os.EOL + e.toString() + os.EOL + os.EOL);
+    const packageInstallFolder = _ensureAndJoinPath(rushTempFolder, 'install-run', `${packageName}@${packageVersion}`);
-    }
+    if (!_isPackageAlreadyInstalled(packageInstallFolder)) {
-}
+        // The package isn't already installed
-exports.runWithErrorAndStatusCode = runWithErrorAndStatusCode;
+        _cleanInstallFolder(rushTempFolder, packageInstallFolder);
-function run() {
+        const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ rawPackageSpecifier, /* qrcode@^1.2.0 */ packageBinName, /* qrcode */ ...packageBinArgs /* [-f, myproject/lib] */] = process.argv;
+        _syncNpmrc(sourceNpmrcFolder, packageInstallFolder);
-    if (!nodePath) {
+        _createPackageJson(packageInstallFolder, packageName, packageVersion);
-        throw new Error('Unexpected exception: could not detect node path');
+        _installPackage(packageInstallFolder, packageName, packageVersion);
-    }
+        _writeFlagFile(packageInstallFolder);
-    if (path.basename(scriptPath).toLowerCase() !== 'install-run.js') {
+    }
-        // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control
+    const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`;
-        // to the script that (presumably) imported this file
+    const statusMessageLine = new Array(statusMessage.length + 1).join('-');
-        return;
+    console.log(os.EOL + statusMessage + os.EOL + statusMessageLine + os.EOL);
-    }
+    const binPath = _getBinPath(packageInstallFolder, packageBinName);
-    if (process.argv.length < 4) {
+    const result = childProcess.spawnSync(binPath, packageBinArgs, {
-        console.log('Usage: install-run.js <package>@<version> <command> [args...]');
+        stdio: 'inherit',
-        console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io');
+        cwd: process.cwd(),
-        process.exit(1);
+        env: process.env
-    }
+    });
-    runWithErrorAndStatusCode(() => {
+    if (result.status !== null) {
-        const rushJsonFolder = findRushJsonFolder();
+        return result.status;
-        const rushCommonFolder = ensureAndJoinPath(rushJsonFolder, 'common');
+    }
-        const packageSpecifier = parsePackageSpecifier(rawPackageSpecifier);
+    else {
-        const name = packageSpecifier.name;
+        throw result.error || new Error('An unknown error occurred.');
-        const version = resolvePackageVersion(rushCommonFolder, packageSpecifier);
+    }
-        if (packageSpecifier.version !== version) {
+}
-            console.log(`Resolved to ${name}@${version}`);
+exports.installAndRun = installAndRun;
-        }
+function runWithErrorAndStatusCode(fn) {
-        return installAndRun(name, version, packageBinName, packageBinArgs);
+    process.exitCode = 1;
-    });
+    try {
-}
+        const exitCode = fn();
-run();
+        process.exitCode = exitCode;
+    }
+    catch (e) {
+        console.error(os.EOL + os.EOL + e.toString() + os.EOL + os.EOL);
+    }
+}
+exports.runWithErrorAndStatusCode = runWithErrorAndStatusCode;
+function _run() {
+    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ rawPackageSpecifier, /* qrcode@^1.2.0 */ packageBinName, /* qrcode */ ...packageBinArgs /* [-f, myproject/lib] */] = process.argv;
+    if (!nodePath) {
+        throw new Error('Unexpected exception: could not detect node path');
+    }
+    if (path.basename(scriptPath).toLowerCase() !== 'install-run.js') {
+        // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control
+        // to the script that (presumably) imported this file
+        return;
+    }
+    if (process.argv.length < 4) {
+        console.log('Usage: install-run.js <package>@<version> <command> [args...]');
+        console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io');
+        process.exit(1);
+    }
+    runWithErrorAndStatusCode(() => {
+        const rushJsonFolder = findRushJsonFolder();
+        const rushCommonFolder = _ensureAndJoinPath(rushJsonFolder, 'common');
+        const packageSpecifier = _parsePackageSpecifier(rawPackageSpecifier);
+        const name = packageSpecifier.name;
+        const version = _resolvePackageVersion(rushCommonFolder, packageSpecifier);
+        if (packageSpecifier.version !== version) {
+            console.log(`Resolved to ${name}@${version}`);
+        }
+        return installAndRun(name, version, packageBinName, packageBinArgs);
+    });
+}
+_run();
 //# sourceMappingURL=install-run.js.map

configs/typescript-config/common.tsconfig.json

@@ -15,7 +15,9 @@
     "preserveWatchOutput": true,
     "newLine": "lf",
     "noImplicitReturns": true,
-    "experimentalDecorators": true
+    "experimentalDecorators": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true
   },
   "include": [
     "../../src/**/*.ts"

extensions/ql-vscode/.eslintrc.js

@@ -0,0 +1,38 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    ecmaVersion: 2018,
+    sourceType: 'module',
+    ecmaFeatures: {
+      modules: true,
+    },
+    project: ['tsconfig.json', './src/**/tsconfig.json'],
+  },
+  plugins: ['@typescript-eslint'],
+  env: {
+    node: true,
+    es6: true
+  },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+  rules: {
+    '@typescript-eslint/no-use-before-define': 0,
+    '@typescript-eslint/no-unused-vars': ["warn", {
+      "vars": "all",
+      "args": "none",
+      "ignoreRestSiblings": false
+    }],
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/no-non-null-assertion": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "prefer-const": ["warn", {"destructuring": "all"}],
+    "indent": "off",
+    "@typescript-eslint/indent": ["error", 2, {
+      "SwitchCase": 1,
+      "FunctionDeclaration": { "body": 1, "parameters": 1 }
+    }],
+    "@typescript-eslint/no-throw-literal": "error"
+  },
+};

extensions/ql-vscode/CHANGELOG.md

@@ -0,0 +1,65 @@
+# CodeQL for Visual Studio Code: Changelog
+
+## 1.1.1 - 23 March 2020
+
+- Fix quick evaluation in `.qll` files.
+- Add new command in query history view to view the log file of a
+  query.
+- Request user acknowledgment before updating the CodeQL binaries.
+- Warn when using the deprecated `codeql.cmd` launcher on Windows.
+
+## 1.1.0 - 17 March 2020
+
+- Add functionality for testing custom CodeQL queries by using the VS
+  Code Test Explorer extension and `codeql test`. See the documentation for
+  more details.
+- Add a "Show log" button to all information, error, and warning
+  popups that will display the CodeQL extension log.
+- Display a message when a query times out.
+- Show canceled queries in query history.
+- Improve error messages when attempting to run non-query files.
+
+## 1.0.6 - 28 February 2020
+
+- Add command to restart query server.
+- Enable support for future minor upgrades to the CodeQL CLI.
+
+## 1.0.5 - 13 February 2020
+
+- Add an icon next to any failed query runs in the query history
+  view.
+- Add the ability to sort alerts by alert message.
+
+## 1.0.4 - 24 January 2020
+
+- Disable word-based autocomplete by default.
+- Add command `CodeQL: Quick Query` for easy query creation without
+  having to choose a place in the filesystem to store the query file.
+
+## 1.0.3 - 13 January 2020
+
+- Reduce the frequency of CodeQL CLI update checks to help avoid hitting GitHub API limits of 60 requests per
+hour for unauthenticated IPs.
+- Fix sorting of result sets with names containing special characters.
+
+## 1.0.2 - 13 December 2019
+
+- Fix rendering of negative numbers in results.
+- Allow customization of query history labels from settings and from
+  query history view context menu.
+- Show number of results in results view.
+- Add commands `CodeQL: Show Next Step on Path` and `CodeQL: Show
+  Previous Step on Path` for navigating the steps on the currently
+  shown path result.
+
+## 1.0.1 - 21 November 2019
+
+- Change `codeQL.cli.executablePath` to a per-machine setting, so it can no longer be set at the user or workspace level. This helps prevent arbitrary code execution when using a VS Code workspace from an untrusted source.
+- Improve the highlighting of the selected query result within the source code.
+- Improve the performance of switching between result tables in the CodeQL Query Results view.
+- Fix the automatic upgrading of CodeQL databases when using upgrade scripts from the workspace.
+- Allow removal of items from the CodeQL Query History view.
+
+## 1.0.0 - 14 November 2019
+
+Initial release of CodeQL for Visual Studio Code.

extensions/ql-vscode/README.md

@@ -7,6 +7,8 @@ This project is an extension for Visual Studio Code that adds rich language supp
 * Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/Semmle/ql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
+To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
+
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.

extensions/ql-vscode/gulpfile.js/webpack.ts

@@ -15,7 +15,8 @@ export function compileView(cb: (err?: Error) => void) {
       hash: false,
       entrypoints: false,
       timings: false,
-      modules: false
+      modules: false,
+      errors: true
     }));
     if (stats.hasErrors()) {
       cb(new Error('Compilation errors detected.'));

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.0.1",
+  "version": "1.1.1",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -18,15 +18,21 @@
   "categories": [
     "Programming Languages"
   ],
+  "extensionDependencies": [
+    "hbenl.vscode-test-explorer"
+  ],
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
+    "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
     "onCommand:codeQL.chooseDatabase",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQLDatabases.chooseDatabase",
     "onCommand:codeQLDatabases.setCurrentDatabase",
+    "onCommand:codeQL.quickQuery",
+    "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
     "onFileSystem:codeql-zip-archive"
   ],
@@ -39,6 +45,14 @@
     "language-configuration.json"
   ],
   "contributes": {
+    "configurationDefaults": {
+      "[ql]": {
+        "editor.wordBasedSuggestions": false
+      },
+      "[dbscheme]": {
+        "editor.wordBasedSuggestions": false
+      }
+    },
     "languages": [
       {
         "id": "ql",
@@ -85,7 +99,7 @@
           "scope": "machine",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.cmd` on Windows. This overrides all other CodeQL CLI settings."
+          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -95,14 +109,20 @@
           "description": "Number of threads for running queries."
         },
         "codeQL.runningQueries.timeout": {
-          "type": ["integer", "null"],
+          "type": [
+            "integer",
+            "null"
+          ],
           "default": null,
           "minimum": 0,
           "maximum": 2147483647,
           "description": "Timeout (in seconds) for running queries. Leave blank or set to zero for no timeout."
         },
         "codeQL.runningQueries.memory": {
-          "type": ["integer", "null"],
+          "type": [
+            "integer",
+            "null"
+          ],
           "default": null,
           "minimum": 1024,
           "description": "Memory (in MB) to use for running queries. Leave blank for CodeQL to choose a suitable value based on your system's available memory."
@@ -111,6 +131,19 @@
           "type": "boolean",
           "default": false,
           "description": "Enable debug logging and tuple counting when running CodeQL queries. This information is useful for debugging query performance."
+        },
+        "codeQL.queryHistory.format": {
+          "type": "string",
+          "default": "[%t] %q on %d - %s",
+          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+        },
+        "codeQL.runningTests.numberOfThreads": {
+          "scope": "window",
+          "type": "integer",
+          "default": 1,
+          "minimum": 1,
+          "maximum": 1024,
+          "description": "Number of threads for running CodeQL tests."
         }
       }
     },
@@ -123,6 +156,10 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.quickQuery",
+        "title": "CodeQL: Quick Query"
+      },
       {
         "command": "codeQL.chooseDatabase",
         "title": "CodeQL: Choose Database",
@@ -170,6 +207,34 @@
       {
         "command": "codeQLQueryHistory.itemClicked",
         "title": "Query History Item"
+      },
+      {
+        "command": "codeQLQueryHistory.showQueryLog",
+        "title": "Show Query Log"
+      },
+      {
+        "command": "codeQLQueryResults.nextPathStep",
+        "title": "CodeQL: Show Next Step on Path"
+      },
+      {
+        "command": "codeQLQueryResults.previousPathStep",
+        "title": "CodeQL: Show Previous Step on Path"
+      },
+      {
+        "command": "codeQLQueryHistory.setLabel",
+        "title": "Set Label"
+      },
+      {
+        "command": "codeQL.restartQueryServer",
+        "title": "CodeQL: Restart Query Server"
+      },
+      {
+        "command": "codeQLTests.showOutputDifferences",
+        "title": "CodeQL: Show Test Output Differences"
+      },
+      {
+        "command": "codeQLTests.acceptOutput",
+        "title": "CodeQL: Accept Test Output"
       }
     ],
     "menus": {
@@ -205,6 +270,26 @@
           "command": "codeQLQueryHistory.removeHistoryItem",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.setLabel",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryLog",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "group": "qltest@1",
+          "when": "view == test-explorer && viewItem == testWithSource"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "group": "qltest@2",
+          "when": "view == test-explorer && viewItem == testWithSource"
         }
       ],
       "explorer/context": [
@@ -251,6 +336,14 @@
         {
           "command": "codeQLQueryHistory.itemClicked",
           "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryLog",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.setLabel",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -295,12 +388,15 @@
     "integration": "node ./out/vscode-tests/run-integration-tests.js",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "postinstall": "node ./node_modules/vscode/bin/install",
-    "format": "tsfmt -r"
+    "format": "tsfmt -r",
+    "lint": "eslint . --ext .ts,.tsx"
   },
   "dependencies": {
+    "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
     "fs-extra": "^8.1.0",
     "glob-promise": "^3.4.0",
+    "js-yaml": "^3.12.0",
     "node-fetch": "~2.6.0",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
@@ -308,17 +404,23 @@
     "semmle-io-node": "^0.0.1",
     "semmle-vscode-utils": "^0.0.1",
     "tmp": "^0.1.0",
+    "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
     "vscode-jsonrpc": "^4.0.0",
-    "vscode-languageclient": "^5.2.1"
+    "vscode-languageclient": "^5.2.1",
+    "vscode-test-adapter-api": "~1.7.0",
+    "vscode-test-adapter-util": "~0.7.0",
+    "minimist": "~1.2.5"
   },
   "devDependencies": {
     "@types/chai": "^4.1.7",
+    "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
     "@types/fs-extra": "^8.0.0",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
+    "@types/js-yaml": "~3.12.1",
     "@types/jszip": "~3.1.6",
     "@types/mocha": "~5.2.7",
     "@types/node": "^12.0.8",
@@ -333,14 +435,15 @@
     "@types/xml2js": "~0.4.4",
     "build-tasks": "^0.0.1",
     "chai": "^4.2.0",
-    "child-process-promise": "^2.2.1",
     "css-loader": "~3.1.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
     "mocha": "~6.2.1",
+    "mocha-sinon": "~2.1.0",
     "npm-run-all": "^4.1.5",
+    "sinon": "~9.0.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
     "ts-loader": "^5.4.5",
@@ -352,6 +455,16 @@
     "vsce": "^1.65.0",
     "vscode-test": "^1.0.0",
     "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2"
+    "webpack-cli": "^3.3.2",
+    "eslint": "~6.8.0",
+    "@typescript-eslint/eslint-plugin": "~2.23.0",
+    "@typescript-eslint/parser": "~2.23.0",
+    "chai-as-promised": "~7.1.1",
+    "@types/chai-as-promised": "~7.1.2",
+    "@types/sinon": "~7.5.2",
+    "sinon-chai": "~3.5.0",
+    "@types/sinon-chai": "~3.2.3",
+    "proxyquire": "~2.1.3",
+    "@types/proxyquire": "~1.3.28"
   }
 }

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -51,7 +51,10 @@ export type Entry = File | Directory;
  */
 export type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
 
-export type ZipFileReference = { sourceArchiveZipPath: string, pathWithinSourceArchive: string };
+export type ZipFileReference = {
+  sourceArchiveZipPath: string;
+  pathWithinSourceArchive: string;
+};
 
 /** Encodes a reference to a source file within a zipped source archive into a single URI. */
 export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
@@ -87,7 +90,7 @@ export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
   });
 }
 
-const sourceArchiveUriAuthorityPattern = /^(\d+)\-(\d+)$/;
+const sourceArchiveUriAuthorityPattern = /^(\d+)-(\d+)$/;
 
 class InvalidSourceArchiveUriError extends Error {
   constructor(uri: vscode.Uri) {
@@ -139,8 +142,8 @@ function ensureDir(map: DirectoryHierarchyMap, dir: string) {
 }
 
 type Archive = {
-  unzipped: unzipper.CentralDirectory,
+  unzipped: unzipper.CentralDirectory;
-  dirMap: DirectoryHierarchyMap,
+  dirMap: DirectoryHierarchyMap;
 };
 
 export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
@@ -163,13 +166,13 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
   // metadata
 
   async stat(uri: vscode.Uri): Promise<vscode.FileStat> {
-    return await this._lookup(uri, false);
+    return await this._lookup(uri);
   }
 
   async readDirectory(uri: vscode.Uri): Promise<[string, vscode.FileType][]> {
     const ref = decodeSourceArchiveUri(uri);
     const archive = await this.getArchive(ref.sourceArchiveZipPath);
-    let contents = archive.dirMap.get(ref.pathWithinSourceArchive);
+    const contents = archive.dirMap.get(ref.pathWithinSourceArchive);
     const result = contents === undefined ? [] : Array.from(contents.entries());
     if (result === undefined) {
       throw vscode.FileSystemError.FileNotFound(uri);
@@ -180,7 +183,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
   // file contents
 
   async readFile(uri: vscode.Uri): Promise<Uint8Array> {
-    const data = (await this._lookupAsFile(uri, false)).data;
+    const data = (await this._lookupAsFile(uri)).data;
     if (data) {
       return data;
     }
@@ -189,25 +192,25 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
 
   // write operations, all disabled
 
-  writeFile(uri: vscode.Uri, content: Uint8Array, options: { create: boolean, overwrite: boolean }): void {
+  writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { create: boolean; overwrite: boolean }): void {
     throw this.readOnlyError;
   }
 
-  rename(oldUri: vscode.Uri, newUri: vscode.Uri, options: { overwrite: boolean }): void {
+  rename(_oldUri: vscode.Uri, _newUri: vscode.Uri, _options: { overwrite: boolean }): void {
     throw this.readOnlyError;
   }
 
-  delete(uri: vscode.Uri): void {
+  delete(_uri: vscode.Uri): void {
     throw this.readOnlyError;
   }
 
-  createDirectory(uri: vscode.Uri): void {
+  createDirectory(_uri: vscode.Uri): void {
     throw this.readOnlyError;
   }
 
   // content lookup
 
-  private async _lookup(uri: vscode.Uri, silent: boolean): Promise<Entry> {
+  private async _lookup(uri: vscode.Uri): Promise<Entry> {
     const ref = decodeSourceArchiveUri(uri);
     const archive = await this.getArchive(ref.sourceArchiveZipPath);
 
@@ -238,8 +241,8 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
     throw vscode.FileSystemError.FileNotFound(uri);
   }
 
-  private async _lookupAsFile(uri: vscode.Uri, silent: boolean): Promise<File> {
+  private async _lookupAsFile(uri: vscode.Uri): Promise<File> {
-    let entry = await this._lookup(uri, silent);
+    const entry = await this._lookup(uri);
     if (entry instanceof File) {
       return entry;
     }
@@ -254,7 +257,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
 
   watch(_resource: vscode.Uri): vscode.Disposable {
     // ignore, fires for all changes...
-    return new vscode.Disposable(() => { });
+    return new vscode.Disposable(() => { /**/ });
   }
 }
 

extensions/ql-vscode/src/bqrs-cli-types.ts

@@ -0,0 +1,79 @@
+
+export const PAGE_SIZE = 1000;
+
+export type ColumnKind = "f" | "i" | "s" | "b" | "d" | "e";
+
+export interface Column {
+  name?: string;
+  kind: ColumnKind;
+}
+
+
+export interface ResultSetSchema {
+  name: string;
+  rows: number;
+  columns: Column[];
+  pagination?: PaginationInfo;
+}
+
+export function getResultSetSchema(resultSetName: string, resultSets: BQRSInfo): ResultSetSchema | undefined {
+  for (const schema of resultSets["result-sets"]) {
+    if (schema.name === resultSetName) {
+      return schema;
+    }
+  }
+  return undefined;
+}
+export interface PaginationInfo {
+  "step-size": number;
+  offsets: number[];
+}
+
+export interface BQRSInfo {
+  "result-sets": ResultSetSchema[];
+}
+
+export interface EntityValue {
+  url?: UrlValue;
+  label?: string;
+}
+
+export interface LineColumnLocation {
+  uri: string;
+  startLine: number;
+  startColumn: number;
+  endLine: number;
+  endColumn: number;
+  charOffset: never;
+  charLength: never;
+}
+
+export interface OffsetLengthLocation {
+  uri: string;
+  startLine: never;
+  startColumn: never;
+  endLine: never;
+  endColumn: never;
+  charOffset: number;
+  charLength: number;
+}
+
+export interface WholeFileLocation {
+  uri: string;
+  startLine: never;
+  startColumn: never;
+  endLine: never;
+  endColumn: never;
+  charOffset: never;
+  charLength: never;
+}
+
+export type UrlValue = LineColumnLocation | OffsetLengthLocation | WholeFileLocation | string;
+
+
+export type ColumnValue = EntityValue | number | string | boolean;
+
+export interface DecodedBqrsChunk {
+  tuples: ColumnValue[][];
+  next?: number;
+}

extensions/ql-vscode/src/cli.ts

@@ -1,13 +1,19 @@
-import * as child_process from "child_process";
+/* eslint-disable @typescript-eslint/camelcase */
+import * as cpp from 'child-process-promise';
+import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
+import { Readable } from 'stream';
+import { StringDecoder } from 'string_decoder';
+import * as tk from 'tree-kill';
 import * as util from 'util';
-import { Logger, ProgressReporter } from "./logging";
+import { CancellationToken, Disposable } from 'vscode';
-import { Disposable } from "vscode";
+import { BQRSInfo, DecodedBqrsChunk } from "./bqrs-cli-types";
-import { DistributionProvider } from "./distribution";
+import { DistributionProvider } from './distribution';
-import { SortDirection } from "./interface-types";
+import { assertNever } from './helpers-pure';
-import { assertNever } from "./helpers-pure";
+import { QueryMetadata, SortDirection } from './interface-types';
+import { Logger, ProgressReporter } from './logging';
 
 /**
  * The version of the SARIF format that we are using.
@@ -23,10 +29,10 @@ const LOGGING_FLAGS = ['-v', '--log-to-stderr'];
  * The expected output of `codeql resolve library-path`.
  */
 export interface QuerySetup {
-  libraryPath: string[],
+  libraryPath: string[];
-  dbscheme: string,
+  dbscheme: string;
-  relativeName?: string,
+  relativeName?: string;
-  compilationCache?: string
+  compilationCache?: string;
 }
 
 /**
@@ -51,14 +57,9 @@ export interface UpgradesInfo {
 }
 
 /**
- * The expected output of `codeql resolve metadata`.
+ * The expected output of `codeql resolve qlpacks`.
  */
-export interface QueryMetadata {
+export type QlpacksInfo = { [name: string]: string[] };
-  name?: string,
-  description?: string,
-  id?: string,
-  kind?: string
-}
 
 // `codeql bqrs interpret` requires both of these to be present or
 // both absent.
@@ -67,6 +68,31 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve tests`.
+ */
+export type ResolvedTests = string[];
+
+/**
+ * Options for `codeql test run`.
+ */
+export interface TestRunOptions {
+  cancellationToken?: CancellationToken;
+  logger?: Logger;
+}
+
+/**
+ * Event fired by `codeql test run`.
+ */
+export interface TestCompleted {
+  test: string;
+  pass: boolean;
+  messages: string[];
+  compilationMs: number;
+  evaluationMs: number;
+  expected: string;
+}
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -96,11 +122,11 @@ export class CodeQLCliServer implements Disposable {
   }
 
 
-  dispose() {
+  dispose(): void {
     this.killProcessIfRunning();
   }
 
-  killProcessIfRunning() {
+  killProcessIfRunning(): void {
     if (this.process) {
       // Tell the Java CLI server process to shut down.
       this.logger.log('Sending shutdown request');
@@ -127,8 +153,8 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Restart the server when the current command terminates
    */
-  private restartCliServer() {
+  private restartCliServer(): void {
-    let callback = () => {
+    const callback = (): void => {
       try {
         this.killProcessIfRunning();
       } finally {
@@ -146,19 +172,27 @@ export class CodeQLCliServer implements Disposable {
 
   }
 
+  /**
+   * Get the path to the CodeQL CLI distribution, or throw an exception if not found.
+   */
+  private async getCodeQlPath(): Promise<string> {
+    const codeqlPath = await this.config.getCodeQlPathWithoutVersionCheck();
+    if (!codeqlPath) {
+      throw new Error('Failed to find CodeQL distribution.');
+    }
+    return codeqlPath;
+  }
+
   /**
    * Launch the cli server
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
-    const config = await this.config.getCodeQlPathWithoutVersionCheck();
+    const config = await this.getCodeQlPath();
-    if (!config) {
+    return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, _data => { /**/ })
-      throw new Error("Failed to find codeql distribution")
-    }
-    return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, data => { })
   }
 
   private async runCodeQlCliInternal(command: string[], commandArgs: string[], description: string): Promise<string> {
-    let stderrBuffers: Buffer[] = [];
+    const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
       throw new Error("runCodeQlCliInternal called while cli was running")
     }
@@ -171,7 +205,7 @@ export class CodeQLCliServer implements Disposable {
       // Grab the process so that typescript know that it is always defined.
       const process = this.process;
       // The array of fragments of stdout
-      let stdoutBuffers: Buffer[] = [];
+      const stdoutBuffers: Buffer[] = [];
 
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
@@ -200,20 +234,21 @@ export class CodeQLCliServer implements Disposable {
           process.stdin.write(this.nullBuffer)
         });
         // Join all the data together
-        let fullBuffer = Buffer.concat(stdoutBuffers);
+        const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
-        let data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
+        const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
         this.logger.log(`CLI command succeeded.`);
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
         // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
-        if (stderrBuffers.length == 0) {
+        const newError =
-          throw new Error(`${description} failed: ${err}`)
+          stderrBuffers.length == 0
-        } else {
+            ? new Error(`${description} failed: ${err}`)
-          throw new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString("utf8")}`);
+            : new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString("utf8")}`);
-        }
+        newError.stack += (err.stack || '');
+        throw newError;
       } finally {
         this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
         // Remove the listeners we set up.
@@ -231,13 +266,94 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Run the next command in the queue
    */
-  private runNext() {
+  private runNext(): void {
     const callback = this.commandQueue.shift();
     if (callback) {
       callback();
     }
   }
 
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param cancellationToken CancellationToken to terminate the test process.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  private async* runAsyncCodeQlCliCommandInternal(
+    command: string[],
+    commandArgs: string[],
+    cancellationToken?: CancellationToken,
+    logger?: Logger
+  ): AsyncGenerator<string, void, unknown> {
+    // Add format argument first, in case commandArgs contains positional parameters.
+    const args = [
+      ...command,
+      '--format', 'jsonz',
+      ...commandArgs
+    ];
+
+    // Spawn the CodeQL process
+    const codeqlPath = await this.getCodeQlPath();
+    const childPromise = cpp.spawn(codeqlPath, args);
+    const child = childPromise.childProcess;
+
+    let cancellationRegistration: Disposable | undefined = undefined;
+    try {
+      if (cancellationToken !== undefined) {
+        cancellationRegistration = cancellationToken.onCancellationRequested(_e => {
+          tk(child.pid);
+        });
+      }
+      if (logger !== undefined) {
+        // The human-readable output goes to stderr.
+        logStream(child.stderr!, logger);
+      }
+
+      for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
+        yield event;
+      }
+
+      await childPromise;
+    }
+    finally {
+      if (cancellationRegistration !== undefined) {
+        cancellationRegistration.dispose();
+      }
+    }
+  }
+
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param description Description of the action being run, to be shown in log and error messages.
+   * @param cancellationToken CancellationToken to terminate the test process.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  public async* runAsyncCodeQlCliCommand<EventType>(
+    command: string[],
+    commandArgs: string[],
+    description: string,
+    cancellationToken?: CancellationToken,
+    logger?: Logger
+  ): AsyncGenerator<EventType, void, unknown> {
+    for await (const event of await this.runAsyncCodeQlCliCommandInternal(command, commandArgs,
+      cancellationToken, logger)) {
+      try {
+        yield JSON.parse(event) as EventType;
+      } catch (err) {
+        throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`)
+      }
+    }
+  }
+
   /**
    * Runs a CodeQL CLI command on the server, returning the output as a string.
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
@@ -253,7 +369,7 @@ export class CodeQLCliServer implements Disposable {
 
     return new Promise((resolve, reject) => {
       // Construct the command that actually does the work
-      const callback = () => {
+      const callback = (): void => {
         try {
           this.runCodeQlCliInternal(command, commandArgs, description).then(resolve, reject);
         } catch (err) {
@@ -303,6 +419,40 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, "Resolving library paths");
   }
 
+  /**
+   * Finds all available QL tests in a given directory.
+   * @param testPath Root of directory tree to search for tests.
+   * @returns The list of tests that were found.
+   */
+  public async resolveTests(testPath: string): Promise<ResolvedTests> {
+    const subcommandArgs = [
+      testPath
+    ];
+    return await this.runJsonCodeQlCliCommand<ResolvedTests>(['resolve', 'tests'], subcommandArgs, 'Resolving tests');
+  }
+
+  /**
+   * Runs QL tests.
+   * @param testPaths Full paths of the tests to run.
+   * @param workspaces Workspace paths to use as search paths for QL packs.
+   * @param options Additional options.
+   */
+  public async* runTests(
+    testPaths: string[], workspaces: string[], options: TestRunOptions
+  ): AsyncGenerator<TestCompleted, void, unknown> {
+
+    const subcommandArgs = [
+      '--additional-packs', workspaces.join(path.delimiter),
+      '--threads', '8',
+      ...testPaths
+    ];
+
+    for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
+      subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
+      yield event;
+    }
+  }
+
   /**
    * Gets the metadata for a query.
    * @param queryPath The path to the query.
@@ -315,6 +465,7 @@ export class CodeQLCliServer implements Disposable {
    * Gets the RAM setting for the query server.
    * @param queryMemoryMb The maximum amount of RAM to use, in MB.
    * Leave `undefined` for CodeQL to choose a limit based on the available system memory.
+   * @param progressReporter The progress reporter to send progress information to.
    * @returns String arguments that can be passed to the CodeQL query server,
    * indicating how to split the given RAM limit between heap and off-heap memory.
    */
@@ -325,9 +476,41 @@ export class CodeQLCliServer implements Disposable {
     }
     return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, "Resolving RAM settings", progressReporter);
   }
+  /**
+   * Gets the headers (and optionally pagination info) of a bqrs.
+   * @param bqrsPath The path to the bqrs.
+   * @param pageSize The page size to precompute offsets into the binary file for.
+   */
+  async bqrsInfo(bqrsPath: string, pageSize?: number): Promise<BQRSInfo> {
+    const subcommandArgs = (
+      pageSize ? ["--paginate-rows", pageSize.toString()] : []
+    ).concat(
+      bqrsPath
+    );
+    return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, "Reading bqrs header");
+  }
+
+  /**
+  * Gets the results from a bqrs.
+  * @param bqrsPath The path to the bqrs.
+  * @param resultSet The result set to get.
+  * @param pageSize How many results to get.
+  * @param offset The 0-based index of the first result to get.
+  */
+  async bqrsDecode(bqrsPath: string, resultSet: string, pageSize?: number, offset?: number): Promise<DecodedBqrsChunk> {
+    const subcommandArgs = [
+      "--entities=url,string",
+      "--result-set", resultSet,
+    ].concat(
+      pageSize ? ["--rows", pageSize.toString()] : []
+    ).concat(
+      offset ? ["--start-at", offset.toString()] : []
+    ).concat([bqrsPath]);
+    return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, "Reading bqrs data");
+  }
 
 
-  async interpretBqrs(metadata: { kind: string, id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     const args = [
       `-t=kind=${metadata.kind}`,
       `-t=id=${metadata.id}`,
@@ -396,7 +579,6 @@ export class CodeQLCliServer implements Disposable {
       "Resolving database");
   }
 
-
   /**
    * Gets information necessary for upgrading a database.
    * @param dbScheme the path to the dbscheme of the database to be upgraded.
@@ -412,6 +594,26 @@ export class CodeQLCliServer implements Disposable {
       "Resolving database upgrade scripts",
     );
   }
+
+  /**
+   * Gets information about available qlpacks
+   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
+   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
+   *   the default CLI search path is used.
+   * @returns A dictionary mapping qlpack name to the directory it comes from
+   */
+  resolveQlpacks(additionalPacks: string[], searchPath?: string[]): Promise<QlpacksInfo> {
+    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    if (searchPath !== undefined) {
+      args.push('--search-path', path.join(...searchPath));
+    }
+
+    return this.runJsonCodeQlCliCommand<QlpacksInfo>(
+      ['resolve', 'qlpacks'],
+      args,
+      "Resolving qlpack information",
+    );
+  }
 }
 
 /**
@@ -494,3 +696,101 @@ export async function runCodeQlCliCommand(codeQlPath: string, command: string[],
     throw new Error(`${description} failed: ${err.stderr || err}`)
   }
 }
+
+/**
+ * Buffer to hold state used when splitting a text stream into lines.
+ */
+class SplitBuffer {
+  private readonly decoder = new StringDecoder('utf8');
+  private readonly maxSeparatorLength: number;
+  private buffer = '';
+  private searchIndex = 0;
+
+  constructor(private readonly separators: readonly string[]) {
+    this.maxSeparatorLength = separators.map(s => s.length).reduce((a, b) => Math.max(a, b), 0);
+  }
+
+  /**
+   * Append new text data to the buffer.
+   * @param chunk The chunk of data to append.
+   */
+  public addChunk(chunk: Buffer): void {
+    this.buffer += this.decoder.write(chunk);
+  }
+
+  /**
+   * Signal that the end of the input stream has been reached.
+   */
+  public end(): void {
+    this.buffer += this.decoder.end();
+    this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
+  }
+
+  /**
+   * Extract the next full line from the buffer, if one is available.
+   * @returns The text of the next available full line (without the separator), or `undefined` if no
+   * line is available.
+   */
+  public getNextLine(): string | undefined {
+    while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
+      for (const separator of this.separators) {
+        if (this.buffer.startsWith(separator, this.searchIndex)) {
+          const line = this.buffer.substr(0, this.searchIndex);
+          this.buffer = this.buffer.substr(this.searchIndex + separator.length);
+          this.searchIndex = 0;
+          return line;
+        }
+      }
+      this.searchIndex++;
+    }
+
+    return undefined;
+  }
+}
+
+/**
+ * Splits a text stream into lines based on a list of valid line separators.
+ * @param stream The text stream to split. This stream will be fully consumed.
+ * @param separators The list of strings that act as line separators.
+ * @returns A sequence of lines (not including separators).
+ */
+async function* splitStreamAtSeparators(
+  stream: Readable, separators: string[]
+): AsyncGenerator<string, void, unknown> {
+
+  const buffer = new SplitBuffer(separators);
+  for await (const chunk of stream) {
+    buffer.addChunk(chunk);
+    let line: string | undefined;
+    do {
+      line = buffer.getNextLine();
+      if (line !== undefined) {
+        yield line;
+      }
+    } while (line !== undefined);
+  }
+  buffer.end();
+  let line: string | undefined;
+  do {
+    line = buffer.getNextLine();
+    if (line !== undefined) {
+      yield line;
+    }
+  } while (line !== undefined);
+}
+
+/**
+ *  Standard line endings for splitting human-readable text.
+ */
+const lineEndings = ['\r\n', '\r', '\n'];
+
+/**
+ * Log a text stream to a `Logger` interface.
+ * @param stream The stream to log.
+ * @param logger The logger that will consume the stream output.
+ */
+async function logStream(stream: Readable, logger: Logger): Promise<void> {
+  for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
+    logger.log(line);
+  }
+}

extensions/ql-vscode/src/config.ts

@@ -37,6 +37,8 @@ const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
 const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
 const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
+const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
+const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
 
 /** When these settings change, the distribution should be updated. */
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
@@ -62,14 +64,22 @@ const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
 const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
 
 export interface QueryServerConfig {
-  codeQlPath: string,
+  codeQlPath: string;
-  debug: boolean,
+  debug: boolean;
-  numThreads: number,
+  numThreads: number;
-  queryMemoryMb?: number,
+  queryMemoryMb?: number;
-  timeoutSecs: number,
+  timeoutSecs: number;
   onDidChangeQueryServerConfiguration?: Event<void>;
 }
 
+/** When these settings change, the query history should be refreshed. */
+const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING];
+
+export interface QueryHistoryConfig {
+  format: string;
+  onDidChangeQueryHistoryConfiguration: Event<void>;
+}
+
 abstract class ConfigListener extends DisposableObject {
   protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());
 
@@ -101,7 +111,7 @@ abstract class ConfigListener extends DisposableObject {
 
 export class DistributionConfigListener extends ConfigListener implements DistributionConfig {
   public get customCodeQlPath(): string | undefined {
-    return CUSTOM_CODEQL_PATH_SETTING.getValue() ? CUSTOM_CODEQL_PATH_SETTING.getValue() : undefined;
+    return CUSTOM_CODEQL_PATH_SETTING.getValue() || undefined;
   }
 
   public get includePrerelease(): boolean {
@@ -109,7 +119,7 @@ export class DistributionConfigListener extends ConfigListener implements Distri
   }
 
   public get personalAccessToken(): string | undefined {
-    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() ? PERSONAL_ACCESS_TOKEN_SETTING.getValue() : undefined;
+    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
   public get onDidChangeDistributionConfiguration(): Event<void> {
@@ -176,3 +186,17 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     this.handleDidChangeConfigurationForRelevantSettings(QUERY_SERVER_RESTARTING_SETTINGS, e);
   }
 }
+
+export class QueryHistoryConfigListener extends ConfigListener implements QueryHistoryConfig {
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings(QUERY_HISTORY_SETTINGS, e);
+  }
+
+  public get onDidChangeQueryHistoryConfiguration(): Event<void> {
+    return this._onDidChangeConfiguration.event;
+  }
+
+  public get format(): string {
+    return QUERY_HISTORY_FORMAT_SETTING.getValue<string>();
+  }
+}

extensions/ql-vscode/src/databases-ui.ts

@@ -1,14 +1,15 @@
 import * as path from 'path';
-import { DisposableObject } from "semmle-vscode-utils";
+import { DisposableObject } from 'semmle-vscode-utils';
-import { commands, Event, EventEmitter, ExtensionContext, ProviderResult, TreeDataProvider, TreeItem, Uri, window } from "vscode";
+import { commands, Event, EventEmitter, ExtensionContext, ProviderResult, TreeDataProvider, TreeItem, Uri, window } from 'vscode';
 import * as cli from './cli';
-import { DatabaseItem, DatabaseManager, getUpgradesDirectories } from "./databases";
+import { DatabaseItem, DatabaseManager, getUpgradesDirectories } from './databases';
-import { logger } from "./logging";
+import { getOnDiskWorkspaceFolders } from './helpers';
-import { clearCacheInDatabase, upgradeDatabase, UserCancellationException } from "./queries";
+import { logger } from './logging';
+import { clearCacheInDatabase, UserCancellationException } from './run-queries';
 import * as qsClient from './queryserver-client';
-import { getOnDiskWorkspaceFolders } from "./helpers";
+import { upgradeDatabase } from './upgrades';
 
-type ThemableIconPath = { light: string, dark: string } | string;
+type ThemableIconPath = { light: string; dark: string } | string;
 
 /**
  * Path to icons to display next to currently selected database.
@@ -90,7 +91,7 @@ class DatabaseTreeDataProvider extends DisposableObject
     }
   }
 
-  public getParent(element: DatabaseItem): ProviderResult<DatabaseItem> {
+  public getParent(_element: DatabaseItem): ProviderResult<DatabaseItem> {
     return null;
   }
 
@@ -128,7 +129,7 @@ async function chooseDatabaseDir(): Promise<Uri | undefined> {
 }
 
 export class DatabaseUI extends DisposableObject {
-  public constructor(private ctx: ExtensionContext, private cliserver: cli.CodeQLCliServer, private databaseManager: DatabaseManager,
+  public constructor(ctx: ExtensionContext, private cliserver: cli.CodeQLCliServer, private databaseManager: DatabaseManager,
     private readonly queryServer: qsClient.QueryServerClient | undefined) {
 
     super();

extensions/ql-vscode/src/databases.ts

@@ -24,13 +24,13 @@ import { Logger, logger } from './logging';
  * The name of the key in the workspaceState dictionary in which we
  * persist the current database across sessions.
  */
-const CURRENT_DB: string = 'currentDatabase';
+const CURRENT_DB = 'currentDatabase';
 
 /**
  * The name of the key in the workspaceState dictionary in which we
  * persist the list of databases across sessions.
  */
-const DB_LIST: string = 'databaseList';
+const DB_LIST = 'databaseList';
 
 export interface DatabaseOptions {
   displayName?: string;
@@ -107,8 +107,8 @@ async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
   return vscode.Uri.file(dbAbsolutePath);
 }
 
-async function findSourceArchive(databasePath: string, silent: boolean = false):
+async function findSourceArchive(databasePath: string, silent = false):
-  Promise<vscode.Uri | undefined> {
+Promise<vscode.Uri | undefined> {
 
   const relativePaths = ['src', 'output/src_archive']
 
@@ -128,8 +128,9 @@ async function findSourceArchive(databasePath: string, silent: boolean = false):
   return undefined;
 }
 
-async function resolveDatabase(databasePath: string):
+async function resolveDatabase(
-  Promise<DatabaseContents | undefined> {
+  databasePath: string
+): Promise<DatabaseContents | undefined> {
 
   const name = path.basename(databasePath);
 
@@ -227,15 +228,20 @@ export interface DatabaseItem {
   resolveSourceFile(file: string | undefined): vscode.Uri;
 
   /**
-   * Holds if the database item has a `.dbinfo` file.
+   * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file.
    */
-  hasDbInfo(): boolean;
+  hasMetadataFile(): Promise<boolean>;
 
   /**
    * Returns `sourceLocationPrefix` of exported database.
    */
   getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
 
+  /**
+   * Returns dataset folder of exported database.
+   */
+  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive,
    * as displayed in the filesystem explorer.
@@ -359,9 +365,11 @@ class DatabaseItemImpl implements DatabaseItem {
   /**
    * Holds if the database item refers to an exported snapshot
    */
-  public hasDbInfo(): boolean {
+  public async hasMetadataFile(): Promise<boolean> {
-    return fs.existsSync(path.join(this.databaseUri.fsPath, '.dbinfo'))
+    return (await Promise.all([
-      || fs.existsSync(path.join(this.databaseUri.fsPath, 'codeql-database.yml'));;
+      fs.pathExists(path.join(this.databaseUri.fsPath, '.dbinfo')),
+      fs.pathExists(path.join(this.databaseUri.fsPath, 'codeql-database.yml'))
+    ])).some(x => x);
   }
 
   /**
@@ -383,6 +391,14 @@ class DatabaseItemImpl implements DatabaseItem {
     return dbInfo.sourceLocationPrefix;
   }
 
+  /**
+   * Returns path to dataset folder of database.
+   */
+  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
+    const dbInfo = await this.getDbInfo(server);
+    return dbInfo.datasetFolder;
+  }
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive.
    */
@@ -412,8 +428,8 @@ class DatabaseItemImpl implements DatabaseItem {
  * `event` fires. If waiting for the event takes too long (by default
  * >1000ms) log a warning, and resolve to undefined.
  */
-function eventFired<T>(event: vscode.Event<T>, timeoutMs: number = 1000): Promise<T | undefined> {
+function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | undefined> {
-  return new Promise((res, rej) => {
+  return new Promise((res, _rej) => {
     let timeout: NodeJS.Timeout | undefined;
     let disposable: vscode.Disposable | undefined;
     function dispose() {
@@ -421,22 +437,24 @@ function eventFired<T>(event: vscode.Event<T>, timeoutMs: number = 1000): Promis
       if (disposable !== undefined) disposable.dispose();
     }
     disposable = event(e => {
-      res(e); dispose();
+      res(e);
+      dispose();
     });
     timeout = setTimeout(() => {
       logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
-      res(undefined); dispose();
+      res(undefined);
+      dispose();
     }, timeoutMs);
   });
 }
 
 export class DatabaseManager extends DisposableObject {
   private readonly _onDidChangeDatabaseItem =
-    this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
   readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
 
   private readonly _onDidChangeCurrentDatabaseItem =
-    this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
   readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event;
 
   private readonly _databaseItems: DatabaseItemImpl[] = [];
@@ -451,7 +469,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   public async openDatabase(uri: vscode.Uri, options?: DatabaseOptions):
-    Promise<DatabaseItem> {
+  Promise<DatabaseItem> {
 
     const contents = await resolveDatabaseContents(uri);
     const realOptions = options || {};
@@ -511,7 +529,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async createDatabaseItemFromPersistedState(state: PersistedDatabaseItem):
-    Promise<DatabaseItem> {
+  Promise<DatabaseItem> {
 
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
@@ -556,7 +574,7 @@ export class DatabaseManager extends DisposableObject {
       }
     } catch (e) {
       // database list had an unexpected type - nothing to be done?
-      showAndLogErrorMessage('Database list loading failed: ${}', e.message);
+      showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
     }
   }
 
@@ -569,7 +587,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   public async setCurrentDatabaseItem(item: DatabaseItem | undefined,
-    skipRefresh: boolean = false): Promise<void> {
+    skipRefresh = false): Promise<void> {
 
     if (!skipRefresh && (item !== undefined)) {
       await item.refresh();  // Will throw on invalid database.

extensions/ql-vscode/src/discovery.ts

@@ -0,0 +1,87 @@
+import { DisposableObject } from 'semmle-vscode-utils';
+
+/**
+ * Base class for "discovery" operations, which scan the file system to find specific kinds of
+ * files. This class automatically prevents more than one discovery operation from running at the
+ * same time.
+ */
+export abstract class Discovery<T> extends DisposableObject {
+  private retry = false;
+  private discoveryInProgress = false;
+
+  constructor() {
+    super();
+  }
+
+  /**
+   * Force the discovery process to run. Normally invoked by the derived class when a relevant file
+   * system change is detected.
+   */
+  public refresh(): void {
+    // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
+    // got a storm of refresh requests due to, say, the copying or deletion of a large directory
+    // tree, we could potentially spawn a separate simultaneous discovery operation for each
+    // individual file change notification.
+    // Our approach is to spawn a discovery operation immediately upon receiving the first refresh
+    // request. If we receive any additional refresh requests before the first one is complete, we
+    // record this fact by setting `this.retry = true`. When the original discovery operation
+    // completes, we discard its results and spawn another one to account for that additional
+    // changes that have happened since.
+    // The means that for the common case of a single file being modified, we'll complete the
+    // discovery and update as soon as possible. If multiple files are being modified, we'll
+    // probably wind up doing discovery at least twice.
+    // We could choose to delay the initial discovery request by a second or two to wait for any
+    // other change notifications that might be coming along. However, this would create more
+    // latency in the common case, in order to save a bit of latency in the uncommon case.
+
+    if (this.discoveryInProgress) {
+      // There's already a discovery operation in progress. Tell it to restart when it's done.
+      this.retry = true;
+    }
+    else {
+      // No discovery in progress, so start one now.
+      this.discoveryInProgress = true;
+      this.launchDiscovery();
+    }
+  }
+
+  /**
+   * Starts the asynchronous discovery operation by invoking the `discover` function. When the
+   * discovery operation completes, the `update` function will be invoked with the results of the
+   * discovery.
+   */
+  private launchDiscovery(): void {
+    const discoveryPromise = this.discover();
+    discoveryPromise.then(results => {
+      if (!this.retry) {
+        // Update any listeners with the results of the discovery.
+        this.discoveryInProgress = false;
+        this.update(results);
+      }
+    });
+    discoveryPromise.finally(() => {
+      if (this.retry) {
+        // Another refresh request came in while we were still running a previous discovery
+        // operation. Since the discovery results we just computed are now stale, we'll launch
+        // another discovery operation instead of updating.
+        // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+        // initial discovery operation failed.
+        this.retry = false;
+        this.launchDiscovery();
+      }
+    });
+  }
+
+  /**
+   * Overridden by the derived class to spawn the actual discovery operation, returning the results.
+   */
+  protected abstract discover(): Promise<T>;
+
+  /**
+   * Overridden by the derived class to atomically update the `Discovery` object with the results of
+   * the discovery operation, and to notify any listeners that the discovery results may have
+   * changed.
+   * @param results The discovery results returned by the `discover` function.
+   */
+  protected abstract update(results: T): void;
+}

extensions/ql-vscode/src/distribution.ts

@@ -6,8 +6,9 @@ import * as unzipper from "unzipper";
 import * as url from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "./config";
-import { ProgressUpdate, showAndLogErrorMessage } from "./helpers";
+import { InvocationRateLimiter, InvocationRateLimiterResultKind, ProgressUpdate, showAndLogErrorMessage } from "./helpers";
 import { logger } from "./logging";
+import * as helpers from "./helpers";
 import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-version";
 
 /**
@@ -19,7 +20,7 @@ import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-versi
 
 /**
  * Default value for the owner name of the extension-managed distribution on GitHub.
- * 
+ *
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
@@ -27,7 +28,7 @@ const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";
 
 /**
  * Default value for the repository name of the extension-managed distribution on GitHub.
- * 
+ *
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
@@ -35,19 +36,19 @@ const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-binaries";
 
 /**
  * Version constraint for the CLI.
- * 
+ *
  * This applies to both extension-managed and CLI distributions.
  */
 export const DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT: VersionConstraint = {
-  description: "2.0.*",
+  description: "2.*.*",
   isVersionCompatible: (v: Version) => {
-    return v.majorVersion === 2 && v.minorVersion === 0
+    return v.majorVersion === 2 && v.minorVersion >= 0
   }
 }
 
 export interface DistributionProvider {
-  getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>,
+  getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
-  onDidChangeDistribution?: Event<void>
+  onDidChangeDistribution?: Event<void>;
 }
 
 export class DistributionManager implements DistributionProvider {
@@ -55,6 +56,11 @@ export class DistributionManager implements DistributionProvider {
     this._config = config;
     this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionConstraint);
     this._onDidChangeDistribution = config.onDidChangeDistributionConfiguration;
+    this._updateCheckRateLimiter = new InvocationRateLimiter(
+      extensionContext,
+      "extensionSpecificDistributionUpdateCheck",
+      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+    );
     this._versionConstraint = versionConstraint;
   }
 
@@ -89,18 +95,26 @@ export class DistributionManager implements DistributionProvider {
     };
   }
 
+  public async hasDistribution(): Promise<boolean> {
+    const result = await this.getDistribution();
+    return result.kind !== FindDistributionResultKind.NoDistribution;
+  }
+
   /**
    * Returns the path to a possibly-compatible CodeQL launcher binary, or undefined if a binary not be found.
    */
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath !== undefined) {
+    if (this._config.customCodeQlPath) {
       if (!await fs.pathExists(this._config.customCodeQlPath)) {
         showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
           "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
           "that a CodeQL executable exists at the specified path or remove the setting.");
         return undefined;
       }
+      if (deprecatedCodeQlLauncherName() && this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!)) {
+        warnDeprecatedLauncher();
+      }
       return this._config.customCodeQlPath;
     }
 
@@ -111,8 +125,8 @@ export class DistributionManager implements DistributionProvider {
 
     if (process.env.PATH) {
       for (const searchDirectory of process.env.PATH.split(path.delimiter)) {
-        const expectedLauncherPath = path.join(searchDirectory, codeQlLauncherName());
+        const expectedLauncherPath = await getExecutableFromDirectory(searchDirectory);
-        if (await fs.pathExists(expectedLauncherPath)) {
+        if (expectedLauncherPath) {
           return expectedLauncherPath;
         }
       }
@@ -125,22 +139,29 @@ export class DistributionManager implements DistributionProvider {
   /**
    * Check for updates to the extension-managed distribution.  If one has not already been installed,
    * this will return an update available result with the latest available release.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
-  public async checkForUpdatesToExtensionManagedDistribution(): Promise<DistributionUpdateCheckResult> {
+  public async checkForUpdatesToExtensionManagedDistribution(
+    minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
     const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
     const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (codeQlPath !== undefined && codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
-      return createInvalidDistributionLocationResult();
+      return createInvalidLocationResult();
+    }
+    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    switch (updateCheckResult.kind) {
+      case InvocationRateLimiterResultKind.Invoked:
+        return updateCheckResult.result;
+      case InvocationRateLimiterResultKind.RateLimited:
+        return createAlreadyCheckedRecentlyResult();
     }
-    return this._extensionSpecificDistributionManager.checkForUpdatesToDistribution();
   }
 
   /**
    * Installs a release of the extension-managed distribution.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public installExtensionManagedDistributionRelease(release: Release,
@@ -154,6 +175,7 @@ export class DistributionManager implements DistributionProvider {
 
   private readonly _config: DistributionConfig;
   private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
   private readonly _versionConstraint: VersionConstraint;
 }
@@ -168,12 +190,11 @@ class ExtensionSpecificDistributionManager {
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
     if (this.getInstalledRelease() !== undefined) {
       // An extension specific distribution has been installed.
-      const expectedLauncherPath = path.join(this.getDistributionRootPath(), codeQlLauncherName());
+      const expectedLauncherPath = await getExecutableFromDirectory(this.getDistributionRootPath(), true);
-      if (await fs.pathExists(expectedLauncherPath)) {
+      if (expectedLauncherPath) {
         return expectedLauncherPath;
       }
-      logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+
-        "Will try PATH.");
       try {
         await this.removeDistribution();
       } catch (e) {
@@ -187,7 +208,7 @@ class ExtensionSpecificDistributionManager {
   /**
    * Check for updates to the extension-managed distribution.  If one has not already been installed,
    * this will return an update available result with the latest available release.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async checkForUpdatesToDistribution(): Promise<DistributionUpdateCheckResult> {
@@ -195,15 +216,19 @@ class ExtensionSpecificDistributionManager {
     const extensionSpecificRelease = this.getInstalledRelease();
     const latestRelease = await this.getLatestRelease();
 
-    if (extensionSpecificRelease !== undefined && codeQlPath !== undefined && latestRelease.id === extensionSpecificRelease.id) {
+    if (
-      return createDistributionAlreadyUpToDateResult();
+      extensionSpecificRelease !== undefined &&
+      codeQlPath !== undefined &&
+      latestRelease.id === extensionSpecificRelease.id
+    ) {
+      return createAlreadyUpToDateResult();
     }
     return createUpdateAvailableResult(latestRelease);
   }
 
   /**
    * Installs a release of the extension-managed distribution.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async installDistributionRelease(release: Release,
@@ -234,8 +259,8 @@ class ExtensionSpecificDistributionManager {
 
       if (progressCallback && contentLength !== null) {
         const totalNumBytes = parseInt(contentLength, 10);
-        const bytesToDisplayMB = (numBytes: number) => `${(numBytes/(1024*1024)).toFixed(1)} MB`;
+        const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-        const updateProgress = () => {
+        const updateProgress = (): void => {
           progressCallback({
             step: numBytesDownloaded,
             maxStep: totalNumBytes,
@@ -258,7 +283,7 @@ class ExtensionSpecificDistributionManager {
           .on("error", reject)
       );
 
-      this.bumpDistributionFolderIndex();
+      await this.bumpDistributionFolderIndex();
 
       logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
       await extractZipArchive(archivePath, this.getDistributionStoragePath());
@@ -269,7 +294,7 @@ class ExtensionSpecificDistributionManager {
 
   /**
    * Remove the extension-managed distribution.
-   * 
+   *
    * This should not be called for a distribution that is currently in use, as remove may fail.
    */
   private async removeDistribution(): Promise<void> {
@@ -293,10 +318,10 @@ class ExtensionSpecificDistributionManager {
     return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
   }
 
-  private bumpDistributionFolderIndex(): void {
+  private async bumpDistributionFolderIndex(): Promise<void> {
     const index = this._extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    this._extensionContext.globalState.update(
+    await this._extensionContext.globalState.update(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
   }
 
@@ -317,8 +342,8 @@ class ExtensionSpecificDistributionManager {
     return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
   }
 
-  private storeInstalledRelease(release: Release | undefined): void {
+  private async storeInstalledRelease(release: Release | undefined): Promise<void> {
-    this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
   }
 
   private readonly _config: DistributionConfig;
@@ -344,7 +369,7 @@ export class ReleasesApiConsumer {
     this._repoName = repoName;
   }
 
-  public async getLatestRelease(versionConstraint: VersionConstraint, includePrerelease: boolean = false): Promise<Release> {
+  public async getLatestRelease(versionConstraint: VersionConstraint, includePrerelease = false): Promise<Release> {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases`;
     const allReleases: GithubRelease[] = await (await this.makeApiCall(apiPath)).json();
     const compatibleReleases = allReleases.filter(release => {
@@ -400,6 +425,13 @@ export class ReleasesApiConsumer {
       Object.assign({}, this._defaultHeaders, additionalHeaders));
 
     if (!response.ok) {
+      // Check for rate limiting
+      const rateLimitResetValue = response.headers.get("X-RateLimit-Reset");
+      if (response.status === 403 && rateLimitResetValue) {
+        const secondsToMillisecondsFactor = 1000;
+        const rateLimitResetDate = new Date(parseInt(rateLimitResetValue, 10) * secondsToMillisecondsFactor);
+        throw new GithubRateLimitedError(response.status, await response.text(), rateLimitResetDate);
+      }
       throw new GithubApiError(response.status, await response.text());
     }
     return response;
@@ -408,7 +440,7 @@ export class ReleasesApiConsumer {
   private async makeRawRequest(
     requestUrl: string,
     headers: { [key: string]: string },
-    redirectCount: number = 0): Promise<fetch.Response> {
+    redirectCount = 0): Promise<fetch.Response> {
     const response = await fetch.default(requestUrl, {
       headers,
       redirect: "manual"
@@ -460,7 +492,7 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
 
 /**
  * Comparison of semantic versions.
- * 
+ *
  * Returns a positive number if a is greater than b.
  * Returns 0 if a equals b.
  * Returns a negative number if a is less than b.
@@ -482,7 +514,11 @@ export function versionCompare(a: Version, b: Version): number {
 }
 
 function codeQlLauncherName(): string {
-  return (os.platform() === "win32") ? "codeql.cmd" : "codeql";
+  return (os.platform() === "win32") ? "codeql.exe" : "codeql";
+}
+
+function deprecatedCodeQlLauncherName(): string | undefined {
+  return (os.platform() === "win32") ? "codeql.cmd" : undefined;
 }
 
 function isRedirectStatusCode(statusCode: number): boolean {
@@ -500,13 +536,16 @@ export enum FindDistributionResultKind {
   NoDistribution
 }
 
-export type FindDistributionResult = CompatibleDistributionResult | UnknownCompatibilityDistributionResult |
+export type FindDistributionResult =
-  IncompatibleDistributionResult | NoDistributionResult;
+  | CompatibleDistributionResult
+  | UnknownCompatibilityDistributionResult
+  | IncompatibleDistributionResult
+  | NoDistributionResult;
 
 interface CompatibleDistributionResult {
   codeQlPath: string;
   kind: FindDistributionResultKind.CompatibleDistribution;
-  version: Version
+  version: Version;
 }
 
 interface UnknownCompatibilityDistributionResult {
@@ -525,23 +564,31 @@ interface NoDistributionResult {
 }
 
 export enum DistributionUpdateCheckResultKind {
+  AlreadyCheckedRecentlyResult,
   AlreadyUpToDate,
-  InvalidDistributionLocation,
+  InvalidLocation,
   UpdateAvailable
 }
 
-type DistributionUpdateCheckResult = DistributionAlreadyUpToDateResult | InvalidDistributionLocationResult |
+type DistributionUpdateCheckResult =
-  UpdateAvailableResult;
+  | AlreadyCheckedRecentlyResult
+  | AlreadyUpToDateResult
+  | InvalidLocationResult
+  | UpdateAvailableResult;
 
-export interface DistributionAlreadyUpToDateResult {
+export interface AlreadyCheckedRecentlyResult {
+  kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
+}
+
+export interface AlreadyUpToDateResult {
   kind: DistributionUpdateCheckResultKind.AlreadyUpToDate;
 }
 
 /**
  * The distribution could not be installed or updated because it is not managed by the extension.
  */
-export interface InvalidDistributionLocationResult {
+export interface InvalidLocationResult {
-  kind: DistributionUpdateCheckResultKind.InvalidDistributionLocation;
+  kind: DistributionUpdateCheckResultKind.InvalidLocation;
 }
 
 export interface UpdateAvailableResult {
@@ -549,15 +596,21 @@ export interface UpdateAvailableResult {
   updatedRelease: Release;
 }
 
-function createDistributionAlreadyUpToDateResult(): DistributionAlreadyUpToDateResult {
+function createAlreadyCheckedRecentlyResult(): AlreadyCheckedRecentlyResult {
+  return {
+    kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult
+  };
+}
+
+function createAlreadyUpToDateResult(): AlreadyUpToDateResult {
   return {
     kind: DistributionUpdateCheckResultKind.AlreadyUpToDate
   };
 }
 
-function createInvalidDistributionLocationResult(): InvalidDistributionLocationResult {
+function createInvalidLocationResult(): InvalidLocationResult {
   return {
-    kind: DistributionUpdateCheckResultKind.InvalidDistributionLocation
+    kind: DistributionUpdateCheckResultKind.InvalidLocation
   };
 }
 
@@ -568,6 +621,31 @@ function createUpdateAvailableResult(updatedRelease: Release): UpdateAvailableRe
   };
 }
 
+// Exported for testing
+export async function getExecutableFromDirectory(directory: string, warnWhenNotFound = false): Promise<string | undefined> {
+  const expectedLauncherPath = path.join(directory, codeQlLauncherName());
+  const deprecatedLauncherName = deprecatedCodeQlLauncherName();
+  const alternateExpectedLauncherPath = deprecatedLauncherName ? path.join(directory, deprecatedLauncherName) : undefined;
+  if (await fs.pathExists(expectedLauncherPath)) {
+    return expectedLauncherPath;
+  } else if (alternateExpectedLauncherPath && (await fs.pathExists(alternateExpectedLauncherPath))) {
+    warnDeprecatedLauncher();
+    return alternateExpectedLauncherPath;
+  }
+  if (warnWhenNotFound) {
+    logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+      "Will try PATH.");
+  }
+  return undefined;
+}
+
+function warnDeprecatedLauncher() {
+  helpers.showAndLogWarningMessage(
+    `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
+    `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
+  );
+}
+
 /**
  * A release on GitHub.
  */
@@ -673,3 +751,9 @@ export class GithubApiError extends Error {
     super(`API call failed with status code ${status}, body: ${body}`);
   }
 }
+
+export class GithubRateLimitedError extends GithubApiError {
+  constructor(public status: number, public body: string, public rateLimitResetDate: Date) {
+    super(status, body);
+  }
+}

extensions/ql-vscode/src/extension.ts

@@ -1,19 +1,27 @@
 import { commands, Disposable, ExtensionContext, extensions, ProgressLocation, ProgressOptions, window as Window, Uri } from 'vscode';
-import { ErrorCodes, LanguageClient, ResponseError } from 'vscode-languageclient';
+import { LanguageClient } from 'vscode-languageclient';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { DistributionConfigListener, QueryServerConfigListener } from './config';
+import { DistributionConfigListener, QueryServerConfigListener, QueryHistoryConfigListener } from './config';
 import { DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
-import { DistributionUpdateCheckResultKind, DistributionManager, FindDistributionResult, FindDistributionResultKind, GithubApiError, DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT } from './distribution';
+import {
+  DistributionUpdateCheckResultKind, DistributionManager, FindDistributionResult, FindDistributionResultKind, GithubApiError,
+  DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT, GithubRateLimitedError
+} from './distribution';
 import * as helpers from './helpers';
 import { spawnIdeServer } from './ide-server';
 import { InterfaceManager, WebviewReveal } from './interface';
 import { ideServerLogger, logger, queryServerLogger } from './logging';
-import { compileAndRunQueryAgainstDatabase, EvaluationInfo, tmpDirDisposal, UserCancellationException } from './queries';
+import { compileAndRunQueryAgainstDatabase, tmpDirDisposal, UserCancellationException } from './run-queries';
-import { QueryHistoryItem, QueryHistoryManager } from './query-history';
+import { CompletedQuery } from './query-results';
+import { QueryHistoryManager } from './query-history';
 import * as qsClient from './queryserver-client';
 import { CodeQLCliServer } from './cli';
 import { assertNever } from './helpers-pure';
+import { displayQuickQuery } from './quick-query';
+import { TestHub, testExplorerExtensionId } from 'vscode-test-adapter-api';
+import { QLTestAdapterFactory } from './test-adapter';
+import { TestUIService } from './test-ui';
 
 /**
  * extension.ts
@@ -48,7 +56,7 @@ let isInstallingOrUpdatingDistribution = false;
  *
  * @param excludedCommands List of commands for which we should not register error stubs.
  */
-function registerErrorStubs(ctx: ExtensionContext, excludedCommands: string[], stubGenerator: (command: string) => () => void) {
+function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => void): void {
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
@@ -68,40 +76,53 @@ function registerErrorStubs(ctx: ExtensionContext, excludedCommands: string[], s
 }
 
 export async function activate(ctx: ExtensionContext): Promise<void> {
-  // Initialise logging, and ensure all loggers are disposed upon exit.
-  ctx.subscriptions.push(logger);
   logger.log('Starting CodeQL extension');
 
+  initializeLogging(ctx);
+
   const distributionConfigListener = new DistributionConfigListener();
   ctx.subscriptions.push(distributionConfigListener);
   const distributionManager = new DistributionManager(ctx, distributionConfigListener, DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT);
 
   const shouldUpdateOnNextActivationKey = "shouldUpdateOnNextActivation";
-  
+
-  registerErrorStubs(ctx, [checkForUpdatesCommand], command => () => {
+  registerErrorStubs([checkForUpdatesCommand], command => () => {
-    Window.showErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
+    helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
   });
 
-  async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, isSilentIfCannotUpdate: boolean): Promise<void> {
+  interface DistributionUpdateConfig {
-    const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution();
+    isUserInitiated: boolean;
+    shouldDisplayMessageWhenNoUpdates: boolean;
+    allowAutoUpdating: boolean;
+  }
+
+  async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
+    const minSecondsSinceLastUpdateCheck = config.isUserInitiated ? 0 : 86400;
+    const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
+      helpers.showAndLogInformationMessage : async (message: string) => logger.log(message);
+    const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
+
+    // We do want to auto update if there is no distribution at all
+    const allowAutoUpdating = config.allowAutoUpdating || !await distributionManager.hasDistribution();
+
     switch (result.kind) {
-      case DistributionUpdateCheckResultKind.AlreadyUpToDate:
+      case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
-        if (!isSilentIfCannotUpdate) {
+        logger.log("Didn't perform CodeQL CLI update check since a check was already performed within the previous " +
-          helpers.showAndLogInformationMessage("CodeQL CLI already up to date.");
+          `${minSecondsSinceLastUpdateCheck} seconds.`);
-        }
         break;
-      case DistributionUpdateCheckResultKind.InvalidDistributionLocation:
+      case DistributionUpdateCheckResultKind.AlreadyUpToDate:
-        if (!isSilentIfCannotUpdate) {
+        await noUpdatesLoggingFunc('CodeQL CLI already up to date.');
-          helpers.showAndLogErrorMessage("CodeQL CLI is installed externally so could not be updated.");
+        break;
-        }
+      case DistributionUpdateCheckResultKind.InvalidLocation:
+        await noUpdatesLoggingFunc('CodeQL CLI is installed externally so could not be updated.');
         break;
       case DistributionUpdateCheckResultKind.UpdateAvailable:
-        if (beganMainExtensionActivation) {
+        if (beganMainExtensionActivation || !allowAutoUpdating) {
           const updateAvailableMessage = `Version "${result.updatedRelease.name}" of the CodeQL CLI is now available. ` +
-            "The update will be installed after Visual Studio Code restarts. Restart now to upgrade?";
+            'Do you wish to upgrade?';
-          ctx.globalState.update(shouldUpdateOnNextActivationKey, true);
+          await ctx.globalState.update(shouldUpdateOnNextActivationKey, true);
-          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, "Restart and Upgrade")) {
+          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, 'Restart and Upgrade')) {
-            await commands.executeCommand("workbench.action.reloadWindow");
+            await commands.executeCommand('workbench.action.reloadWindow');
           }
         } else {
           const progressOptions: ProgressOptions = {
@@ -112,7 +133,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
           await helpers.withProgress(progressOptions, progress =>
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
-          ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
+          await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
           helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
         }
         break;
@@ -121,34 +142,36 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     }
   }
 
-  async function installOrUpdateDistribution(isSilentIfCannotUpdate: boolean): Promise<void> {
+  async function installOrUpdateDistribution(config: DistributionUpdateConfig): Promise<void> {
     if (isInstallingOrUpdatingDistribution) {
       throw new Error("Already installing or updating CodeQL CLI");
     }
     isInstallingOrUpdatingDistribution = true;
+    const codeQlInstalled = await distributionManager.getCodeQlPathWithoutVersionCheck() !== undefined;
+    const willUpdateCodeQl = ctx.globalState.get(shouldUpdateOnNextActivationKey);
+    const messageText = willUpdateCodeQl
+      ? "Updating CodeQL CLI"
+      : codeQlInstalled
+        ? "Checking for updates to CodeQL CLI"
+        : "Installing CodeQL CLI";
+
     try {
-      const codeQlInstalled = await distributionManager.getCodeQlPathWithoutVersionCheck() !== undefined;
+      await installOrUpdateDistributionWithProgressTitle(messageText, config);
-      const messageText = ctx.globalState.get(shouldUpdateOnNextActivationKey) ? "Updating CodeQL CLI" :
-        codeQlInstalled ? "Checking for updates to CodeQL CLI" : "Installing CodeQL CLI";
-      await installOrUpdateDistributionWithProgressTitle(messageText, isSilentIfCannotUpdate);
     } catch (e) {
       // Don't rethrow the exception, because if the config is changed, we want to be able to retry installing
       // or updating the distribution.
-      if (e instanceof GithubApiError && (e.status == 404 || e.status == 403 || e.status === 401)) {
+      const alertFunction = (codeQlInstalled && !config.isUserInitiated) ?
-        const errorMessageResponse = Window.showErrorMessage("Unable to download CodeQL CLI. See " +
+        helpers.showAndLogWarningMessage : helpers.showAndLogErrorMessage;
-          "https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/README.md for more details about how " +
+      const taskDescription = (willUpdateCodeQl ? "update" :
-          "to obtain CodeQL CLI.", "Edit Settings");
+        codeQlInstalled ? "check for updates to" : "install") + " CodeQL CLI";
-        // We're deliberately not `await`ing this promise, just
+
-        // asynchronously letting the user follow the convenience link
+      if (e instanceof GithubRateLimitedError) {
-        // if they want to.
+        alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
-        errorMessageResponse.then(response => {
+          `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString()}.`);
-          if (response !== undefined) {
+      } else if (e instanceof GithubApiError) {
-            commands.executeCommand('workbench.action.openSettingsJson');
+        alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
-          }
-        });
-      } else {
-        helpers.showAndLogErrorMessage("Unable to download CodeQL CLI. " + e);
       }
+      alertFunction(`Unable to ${taskDescription}. ` + e);
     } finally {
       isInstallingOrUpdatingDistribution = false;
     }
@@ -176,10 +199,8 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(isSilentIfCannotUpdate: boolean): Promise<void> {
+  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
-    if (!isInstallingOrUpdatingDistribution) {
+    await installOrUpdateDistribution(config);
-      await installOrUpdateDistribution(isSilentIfCannotUpdate);
-    }
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
@@ -187,23 +208,44 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
       await activateWithInstalledDistribution(ctx, distributionManager);
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
-      registerErrorStubs(ctx, [checkForUpdatesCommand], command => async () => {
+      registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = "Install CodeQL CLI";
-        const chosenAction = await Window.showErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, installActionName);
+        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+          items: [installActionName]
+        });
         if (chosenAction === installActionName) {
-          installOrUpdateThenTryActivate(true);
+          installOrUpdateThenTryActivate({
+            isUserInitiated: true,
+            shouldDisplayMessageWhenNoUpdates: false,
+            allowAutoUpdating: true
+          });
         }
       });
     }
   }
 
-  ctx.subscriptions.push(distributionConfigListener.onDidChangeDistributionConfiguration(() => installOrUpdateThenTryActivate(true)));
+  ctx.subscriptions.push(distributionConfigListener.onDidChangeDistributionConfiguration(() => installOrUpdateThenTryActivate({
-  ctx.subscriptions.push(commands.registerCommand(checkForUpdatesCommand, () => installOrUpdateThenTryActivate(false)));
+    isUserInitiated: true,
+    shouldDisplayMessageWhenNoUpdates: false,
+    allowAutoUpdating: true
+  })));
+  ctx.subscriptions.push(commands.registerCommand(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+    isUserInitiated: true,
+    shouldDisplayMessageWhenNoUpdates: true,
+    allowAutoUpdating: true
+  })));
 
-  await installOrUpdateThenTryActivate(true);
+  await installOrUpdateThenTryActivate({
+    isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
+    shouldDisplayMessageWhenNoUpdates: false,
+
+    // only auto update on startup if the user has previously requested an update
+    // otherwise, ask user to accept the update
+    allowAutoUpdating: !!ctx.globalState.get(shouldUpdateOnNextActivationKey)
+  });
 }
 
-async function activateWithInstalledDistribution(ctx: ExtensionContext, distributionManager: DistributionManager) {
+async function activateWithInstalledDistribution(ctx: ExtensionContext, distributionManager: DistributionManager): Promise<void> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
@@ -212,10 +254,6 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
   const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(distributionManager);
   ctx.subscriptions.push(qlConfigurationListener);
 
-  ctx.subscriptions.push(queryServerLogger);
-  ctx.subscriptions.push(ideServerLogger);
-
-
   const cliServer = new CodeQLCliServer(distributionManager, logger);
   ctx.subscriptions.push(cliServer);
 
@@ -230,16 +268,21 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
   const databaseUI = new DatabaseUI(ctx, cliServer, dbm, qs);
   ctx.subscriptions.push(databaseUI);
 
-  const qhm = new QueryHistoryManager(ctx, async item => showResultsForInfo(item.info, WebviewReveal.Forced));
+  const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  const qhm = new QueryHistoryManager(
+    ctx,
+    queryHistoryConfigurationListener,
+    async item => showResultsForCompletedQuery(item, WebviewReveal.Forced)
+  );
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
   archiveFilesystemProvider.activate(ctx);
 
-  async function showResultsForInfo(info: EvaluationInfo, forceReveal: WebviewReveal): Promise<void> {
+  async function showResultsForCompletedQuery(query: CompletedQuery, forceReveal: WebviewReveal): Promise<void> {
-    await intm.showResults(info, forceReveal, false);
+    await intm.showResults(query, forceReveal, false);
   }
 
-  async function compileAndRunQuery(quickEval: boolean, selectedQuery: Uri | undefined) {
+  async function compileAndRunQuery(quickEval: boolean, selectedQuery: Uri | undefined): Promise<void> {
     if (qs !== undefined) {
       try {
         const dbItem = await databaseUI.getDatabaseItem();
@@ -247,27 +290,23 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
           throw new Error('Can\'t run query without a selected database');
         }
         const info = await compileAndRunQueryAgainstDatabase(cliServer, qs, dbItem, quickEval, selectedQuery);
-        await showResultsForInfo(info, WebviewReveal.NotForced);
+        const item = qhm.addQuery(info);
-        qhm.push(new QueryHistoryItem(info));
+        await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      }
+      } catch (e) {
-      catch (e) {
         if (e instanceof UserCancellationException) {
-          logger.log(e.message);
+          helpers.showAndLogWarningMessage(e.message);
-        }
+        } else if (e instanceof Error) {
-        else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-          logger.log(e.message);
-        }
-        else if (e instanceof Error)
           helpers.showAndLogErrorMessage(e.message);
-        else
+        } else {
           throw e;
+        }
       }
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
-  let client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
+  const client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
     documentSelector: [
       { language: 'ql', scheme: 'file' },
       { language: 'yaml', scheme: 'file', pattern: '**/qlpack.yml' }
@@ -279,10 +318,34 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     outputChannel: ideServerLogger.outputChannel
   }, true);
 
+  const testExplorerExtension = extensions.getExtension<TestHub>(testExplorerExtensionId);
+  if (testExplorerExtension) {
+    const testHub = testExplorerExtension.exports;
+    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
+    ctx.subscriptions.push(testAdapterFactory);
+
+    const testUIService = new TestUIService(testHub);
+    ctx.subscriptions.push(testUIService);
+  }
+
   ctx.subscriptions.push(commands.registerCommand('codeQL.runQuery', async (uri: Uri | undefined) => await compileAndRunQuery(false, uri)));
   ctx.subscriptions.push(commands.registerCommand('codeQL.quickEval', async (uri: Uri | undefined) => await compileAndRunQuery(true, uri)));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.quickQuery', async () => displayQuickQuery(ctx, cliServer, databaseUI)));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.restartQueryServer', async () => {
+    await qs.restartQueryServer();
+    helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', { outputLogger: queryServerLogger });
+  }));
 
   ctx.subscriptions.push(client.start());
 }
 
+function initializeLogging(ctx: ExtensionContext): void {
+  logger.init(ctx);
+  queryServerLogger.init(ctx);
+  ideServerLogger.init(ctx);
+  ctx.subscriptions.push(logger);
+  ctx.subscriptions.push(queryServerLogger);
+  ctx.subscriptions.push(ideServerLogger);
+}
+
 const checkForUpdatesCommand = 'codeQL.checkForUpdatesToCLI';

extensions/ql-vscode/src/helpers.ts

@@ -1,7 +1,7 @@
 import * as path from 'path';
-import { CancellationToken, ProgressOptions, window as Window, workspace } from 'vscode';
+import { CancellationToken, ExtensionContext, ProgressOptions, window as Window, workspace } from 'vscode';
 import { logger } from './logging';
-import { EvaluationInfo } from './queries';
+import { QueryInfo } from './run-queries';
 
 export interface ProgressUpdate {
   /**
@@ -46,45 +46,67 @@ export function withProgress<R>(
 /**
  * Show an error message and log it to the console
  *
- * @param message — The message to show.
+ * @param message The message to show.
- * @param items — A set of items that will be rendered as actions in the message.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return — A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogErrorMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+export async function showAndLogErrorMessage(message: string, {
-  logger.log(message);
+  outputLogger = logger,
-  return Window.showErrorMessage(message, ...items);
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
 }
 /**
  * Show a warning message and log it to the console
  *
- * @param message — The message to show.
+ * @param message The message to show.
- * @param items — A set of items that will be rendered as actions in the message.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return — A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogWarningMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+export async function showAndLogWarningMessage(message: string, {
-  logger.log(message);
+  outputLogger = logger,
-  return Window.showWarningMessage(message, ...items);
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showWarningMessage);
 }
 /**
  * Show an information message and log it to the console
  *
- * @param message — The message to show.
+ * @param message The message to show.
- * @param items — A set of items that will be rendered as actions in the message.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return — A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogInformationMessage(message: string, ...items: string[]): Thenable<string | undefined> {
+export async function showAndLogInformationMessage(message: string, {
-  logger.log(message);
+  outputLogger = logger,
-  return Window.showInformationMessage(message, ...items);
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage);
+}
+
+type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
+
+async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
+  fn: ShowMessageFn): Promise<string | undefined> {
+  const label = 'Show Log';
+  outputLogger.log(message);
+  const result = await fn(message, label, ...items);
+  if (result === label) {
+    outputLogger.show();
+  }
+  return result;
 }
 
 /**
  * Opens a modal dialog for the user to make a yes/no choice.
- * @param message — The message to show.
+ * @param message The message to show.
  *
- * @return — `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
+ * @return `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
  */
 export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
   const yesItem = { title: 'Yes', isCloseAffordance: false };
@@ -95,10 +117,10 @@ export async function showBinaryChoiceDialog(message: string): Promise<boolean>
 
 /**
  * Show an information message with a customisable action.
- * @param message — The message to show.
+ * @param message The message to show.
- * @param actionMessage - The call to action message.
+ * @param actionMessage The call to action message.
  *
- * @return — `true` if the user clicks the action, `false` if the user cancels the dialog.
+ * @return `true` if the user clicks the action, `false` if the user cancels the dialog.
  */
 export async function showInformationMessageWithAction(message: string, actionMessage: string): Promise<boolean> {
   const actionItem = { title: actionMessage, isCloseAffordance: false };
@@ -109,7 +131,7 @@ export async function showInformationMessageWithAction(message: string, actionMe
 /** Gets all active workspace folders that are on the filesystem. */
 export function getOnDiskWorkspaceFolders() {
   const workspaceFolders = workspace.workspaceFolders || [];
-  let diskWorkspaceFolders: string[] = [];
+  const diskWorkspaceFolders: string[] = [];
   for (const workspaceFolder of workspaceFolders) {
     if (workspaceFolder.uri.scheme === "file")
       diskWorkspaceFolders.push(workspaceFolder.uri.fsPath)
@@ -121,16 +143,104 @@ export function getOnDiskWorkspaceFolders() {
  * Gets a human-readable name for an evaluated query.
  * Uses metadata if it exists, and defaults to the query file name.
  */
-export function getQueryName(info: EvaluationInfo) {
+export function getQueryName(query: QueryInfo) {
   // Queries run through quick evaluation are not usually the entire query file.
   // Label them differently and include the line numbers.
-  if (info.query.quickEvalPosition !== undefined) {
+  if (query.quickEvalPosition !== undefined) {
-    const { line, endLine, fileName } = info.query.quickEvalPosition;
+    const { line, endLine, fileName } = query.quickEvalPosition;
     const lineInfo = line === endLine ? `${line}` : `${line}-${endLine}`;
     return `Quick evaluation of ${path.basename(fileName)}:${lineInfo}`;
-  } else if (info.query.metadata && info.query.metadata.name) {
+  } else if (query.metadata && query.metadata.name) {
-    return info.query.metadata.name;
+    return query.metadata.name;
   } else {
-    return path.basename(info.query.program.queryPath);
+    return path.basename(query.program.queryPath);
   }
 }
+
+/**
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
+ */
+export class InvocationRateLimiter<T> {
+  constructor(
+    extensionContext: ExtensionContext,
+    funcIdentifier: string,
+    func: () => Promise<T>,
+    createDate: (dateString?: string) => Date = s => s ? new Date(s) : new Date()) {
+    this._createDate = createDate;
+    this._extensionContext = extensionContext;
+    this._func = func;
+    this._funcIdentifier = funcIdentifier;
+  }
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(minSecondsSinceLastInvocation: number): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this._createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 > updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this._func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined =
+      this._extensionContext.globalState.get(InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier);
+    return maybeDateString ? this._createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this._extensionContext.globalState.update(InvocationRateLimiter._invocationRateLimiterPrefix + this._funcIdentifier, date);
+  }
+
+  private readonly _createDate: (dateString?: string) => Date;
+  private readonly _extensionContext: ExtensionContext;
+  private readonly _func: () => Promise<T>;
+  private readonly _funcIdentifier: string;
+
+  private static readonly _invocationRateLimiterPrefix = "invocationRateLimiter_lastInvocationDate_";
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited
+  };
+}

extensions/ql-vscode/src/ide-server.ts

@@ -17,8 +17,8 @@ export async function spawnIdeServer(config: QueryServerConfig): Promise<StreamI
       ['execute', 'language-server'],
       ['--check-errors', 'ON_CHANGE'],
       ideServerLogger,
-      data => ideServerLogger.logWithoutTrailingNewline(data.toString()),
+      data => ideServerLogger.log(data.toString(), { trailingNewline: false }),
-      data => ideServerLogger.logWithoutTrailingNewline(data.toString()),
+      data => ideServerLogger.log(data.toString(), { trailingNewline: false }),
       progressReporter
     );
     return { writer: child.stdin!, reader: child.stdout! };

extensions/ql-vscode/src/interface-types.ts

@@ -16,6 +16,14 @@ export interface DatabaseInfo {
   databaseUri: string;
 }
 
+/** Arbitrary query metadata */
+export interface QueryMetadata {
+  name?: string;
+  description?: string;
+  id?: string;
+  kind?: string;
+}
+
 export interface PreviousExecution {
   queryName: string;
   time: string;
@@ -26,17 +34,22 @@ export interface PreviousExecution {
 export interface Interpretation {
   sourceLocationPrefix: string;
   numTruncatedResults: number;
+  /**
+   * sortState being undefined means don't sort, just present results in the order
+   * they appear in the sarif file.
+   */
+  sortState?: InterpretedResultsSortState;
   sarif: sarif.Log;
 }
 
-export interface ResultsInfo {
+export interface ResultsPaths {
   resultsPath: string;
   interpretedResultsPath: string;
 }
 
 export interface SortedResultSetInfo {
   resultsPath: string;
-  sortState: SortState;
+  sortState: RawResultsSortState;
 }
 
 export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
@@ -53,51 +66,87 @@ export interface ResultsUpdatingMsg {
 export interface SetStateMsg {
   t: 'setState';
   resultsPath: string;
+  origResultsPaths: ResultsPaths;
   sortedResultsMap: SortedResultsMap;
   interpretation: undefined | Interpretation;
   database: DatabaseInfo;
-  kind?: string;
+  metadata?: QueryMetadata;
   /**
    * Whether to keep displaying the old results while rendering the new results.
    *
    * This is useful to prevent properties like scroll state being lost when rendering the sorted results after sorting a column.
    */
   shouldKeepOldResultsWhileRendering: boolean;
-};
+}
 
-export type IntoResultsViewMsg = ResultsUpdatingMsg | SetStateMsg;
+/** Advance to the next or previous path no in the path viewer */
+export interface NavigatePathMsg {
+  t: 'navigatePath';
 
-export type FromResultsViewMsg = ViewSourceFileMsg | ToggleDiagnostics | ChangeSortMsg | ResultViewLoaded;
+  /** 1 for next, -1 for previous */
+  direction: number;
+}
+
+export type IntoResultsViewMsg = ResultsUpdatingMsg | SetStateMsg | NavigatePathMsg;
+
+export type FromResultsViewMsg =
+  | ViewSourceFileMsg
+  | ToggleDiagnostics
+  | ChangeRawResultsSortMsg
+  | ChangeInterpretedResultsSortMsg
+  | ResultViewLoaded;
 
 interface ViewSourceFileMsg {
   t: 'viewSourceFile';
   loc: ResolvableLocationValue;
   databaseUri: string;
-};
+}
 
 interface ToggleDiagnostics {
   t: 'toggleDiagnostics';
   databaseUri: string;
-  resultsPath: string;
+  metadata?: QueryMetadata;
+  origResultsPaths: ResultsPaths;
   visible: boolean;
   kind?: string;
-};
+}
 
 interface ResultViewLoaded {
   t: 'resultViewLoaded';
-};
+}
 
 export enum SortDirection {
   asc, desc
 }
 
-export interface SortState {
+export interface RawResultsSortState {
   columnIndex: number;
-  direction: SortDirection;
+  sortDirection: SortDirection;
 }
 
-interface ChangeSortMsg {
+export type InterpretedResultsSortColumn =
+  'alert-message';
+
+export interface InterpretedResultsSortState {
+  sortBy: InterpretedResultsSortColumn;
+  sortDirection: SortDirection;
+}
+
+interface ChangeRawResultsSortMsg {
   t: 'changeSort';
   resultSetName: string;
-  sortState?: SortState;
+  /**
+   * sortState being undefined means don't sort, just present results in the order
+   * they appear in the sarif file.
+   */
+  sortState?: RawResultsSortState;
+}
+
+interface ChangeInterpretedResultsSortMsg {
+  t: 'changeInterpretedSort';
+  /**
+   * sortState being undefined means don't sort, just present results in the order
+   * they appear in the sarif file.
+   */
+  sortState?: InterpretedResultsSortState;
 }

extensions/ql-vscode/src/interface.ts

@@ -1,20 +1,21 @@
 import * as crypto from 'crypto';
 import * as path from 'path';
-import * as bqrs from 'semmle-bqrs';
+import * as Sarif from 'sarif';
-import { CustomResultSets, FivePartLocation, LocationStyle, LocationValue, PathProblemQueryResults, ProblemQueryResults, ResolvableLocationValue, tryGetResolvableLocation, WholeFileLocation } from 'semmle-bqrs';
+import { FivePartLocation, LocationStyle, LocationValue, ResolvableLocationValue, tryGetResolvableLocation, WholeFileLocation } from 'semmle-bqrs';
-import { FileReader } from 'semmle-io-node';
 import { DisposableObject } from 'semmle-vscode-utils';
 import * as vscode from 'vscode';
-import { Diagnostic, DiagnosticRelatedInformation, DiagnosticSeverity, languages, Location, Position, Range, Uri, window as Window, workspace } from 'vscode';
+import { Diagnostic, DiagnosticRelatedInformation, DiagnosticSeverity, languages, Location, Range, Uri, window as Window, workspace } from 'vscode';
+import * as cli from './cli';
 import { CodeQLCliServer } from './cli';
 import { DatabaseItem, DatabaseManager } from './databases';
-import * as helpers from './helpers';
 import { showAndLogErrorMessage } from './helpers';
 import { assertNever } from './helpers-pure';
-import { FromResultsViewMsg, Interpretation, IntoResultsViewMsg, ResultsInfo, SortedResultSetInfo, SortedResultsMap, INTERPRETED_RESULTS_PER_RUN_LIMIT } from './interface-types';
+import { FromResultsViewMsg, Interpretation, INTERPRETED_RESULTS_PER_RUN_LIMIT, IntoResultsViewMsg, QueryMetadata, ResultsPaths, SortedResultSetInfo, SortedResultsMap, InterpretedResultsSortState, SortDirection } from './interface-types';
 import { Logger } from './logging';
 import * as messages from './messages';
-import { EvaluationInfo, interpretResults, QueryInfo, tmpDir } from './queries';
+import { CompletedQuery, interpretResults } from './query-results';
+import { QueryInfo, tmpDir } from './run-queries';
+import { parseSarifLocation, parseSarifPlainTextMessage } from './sarif-utils';
 
 /**
  * interface.ts
@@ -41,7 +42,11 @@ export enum WebviewReveal {
  * Returns HTML to populate the given webview.
  * Uses a content security policy that only loads the given script.
  */
-function getHtmlForWebview(webview: vscode.Webview, scriptUriOnDisk: vscode.Uri, stylesheetUriOnDisk: vscode.Uri) {
+function getHtmlForWebview(
+  webview: vscode.Webview,
+  scriptUriOnDisk: vscode.Uri,
+  stylesheetUriOnDisk: vscode.Uri
+): void {
   // Convert the on-disk URIs into webview URIs.
   const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
   const stylesheetWebviewUri = webview.asWebviewUri(stylesheetUriOnDisk);
@@ -74,7 +79,7 @@ function getHtmlForWebview(webview: vscode.Webview, scriptUriOnDisk: vscode.Uri,
 
 /** Converts a filesystem URI into a webview URI string that the given panel can use to read the file. */
 export function fileUriToWebviewUri(panel: vscode.WebviewPanel, fileUriOnDisk: Uri): string {
-  return encodeURI(panel.webview.asWebviewUri(fileUriOnDisk).toString(true));
+  return panel.webview.asWebviewUri(fileUriOnDisk).toString();
 }
 
 /** Converts a URI string received from a webview into a local filesystem URI for the same resource. */
@@ -85,20 +90,68 @@ export function webviewUriToFileUri(webviewUri: string): Uri {
   return Uri.file(path);
 }
 
+function sortMultiplier(sortDirection: SortDirection): number {
+  switch (sortDirection) {
+    case SortDirection.asc: return 1;
+    case SortDirection.desc: return -1;
+  }
+}
+
+function sortInterpretedResults(results: Sarif.Result[], sortState: InterpretedResultsSortState | undefined): void {
+  if (sortState !== undefined) {
+    const multiplier = sortMultiplier(sortState.sortDirection);
+    switch (sortState.sortBy) {
+      case 'alert-message':
+        results.sort((a, b) =>
+          a.message.text === undefined ? 0 :
+            b.message.text === undefined ? 0 :
+              multiplier * (a.message.text?.localeCompare(b.message.text)));
+        break;
+      default:
+        assertNever(sortState.sortBy);
+    }
+  }
+}
+
 export class InterfaceManager extends DisposableObject {
-  private _displayedEvaluationInfo?: EvaluationInfo;
+  private _displayedQuery?: CompletedQuery;
   private _panel: vscode.WebviewPanel | undefined;
   private _panelLoaded = false;
   private _panelLoadedCallBacks: (() => void)[] = [];
 
-  private readonly _diagnosticCollection = languages.createDiagnosticCollection(`codeql-query-results`);
+  private readonly _diagnosticCollection = languages.createDiagnosticCollection(
-
+    `codeql-query-results`
-  constructor(public ctx: vscode.ExtensionContext, private databaseManager: DatabaseManager,
+  );
-    public cliServer: CodeQLCliServer, public logger: Logger) {
 
+  constructor(
+    public ctx: vscode.ExtensionContext,
+    private databaseManager: DatabaseManager,
+    public cliServer: CodeQLCliServer,
+    public logger: Logger
+  ) {
     super();
     this.push(this._diagnosticCollection);
-    this.push(vscode.window.onDidChangeTextEditorSelection(this.handleSelectionChange.bind(this)));
+    this.push(
+      vscode.window.onDidChangeTextEditorSelection(
+        this.handleSelectionChange.bind(this)
+      )
+    );
+    this.push(
+      vscode.commands.registerCommand(
+        "codeQLQueryResults.nextPathStep",
+        this.navigatePathStep.bind(this, 1)
+      )
+    );
+    this.push(
+      vscode.commands.registerCommand(
+        "codeQLQueryResults.previousPathStep",
+        this.navigatePathStep.bind(this, -1)
+      )
+    );
+  }
+
+  navigatePathStep(direction: number): void {
+    this.postMessage({ t: "navigatePath", direction });
   }
 
   // Returns the webview panel, creating it if it doesn't already
@@ -106,9 +159,9 @@ export class InterfaceManager extends DisposableObject {
   getPanel(): vscode.WebviewPanel {
     if (this._panel == undefined) {
       const { ctx } = this;
-      const panel = this._panel = Window.createWebviewPanel(
+      const panel = (this._panel = Window.createWebviewPanel(
-        'resultsView', // internal name
+        "resultsView", // internal name
-        'CodeQL Query Results', // user-visible name
+        "CodeQL Query Results", // user-visible name
         { viewColumn: vscode.ViewColumn.Beside, preserveFocus: true },
         {
           enableScripts: true,
@@ -116,50 +169,96 @@ export class InterfaceManager extends DisposableObject {
           retainContextWhenHidden: true,
           localResourceRoots: [
             vscode.Uri.file(tmpDir.name),
-            vscode.Uri.file(path.join(this.ctx.extensionPath, 'out'))
+            vscode.Uri.file(path.join(this.ctx.extensionPath, "out"))
           ]
         }
+      ));
+      this._panel.onDidDispose(
+        () => {
+          this._panel = undefined;
+        },
+        null,
+        ctx.subscriptions
+      );
+      const scriptPathOnDisk = vscode.Uri.file(
+        ctx.asAbsolutePath("out/resultsView.js")
+      );
+      const stylesheetPathOnDisk = vscode.Uri.file(
+        ctx.asAbsolutePath("out/resultsView.css")
+      );
+      getHtmlForWebview(
+        panel.webview,
+        scriptPathOnDisk,
+        stylesheetPathOnDisk
+      );
+      panel.webview.onDidReceiveMessage(
+        async e => this.handleMsgFromView(e),
+        undefined,
+        ctx.subscriptions
       );
-      this._panel.onDidDispose(() => { this._panel = undefined; }, null, ctx.subscriptions);
-      const scriptPathOnDisk = vscode.Uri
-        .file(ctx.asAbsolutePath('out/resultsView.js'));
-      const stylesheetPathOnDisk = vscode.Uri
-        .file(ctx.asAbsolutePath('out/resultsView.css'));
-      getHtmlForWebview(panel.webview, scriptPathOnDisk, stylesheetPathOnDisk);
-      panel.webview.onDidReceiveMessage(async (e) => this.handleMsgFromView(e), undefined, ctx.subscriptions);
     }
     return this._panel;
   }
 
-  private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
+  private async changeSortState(
+    update: (query: CompletedQuery) => Promise<void>
+  ): Promise<void> {
+    if (this._displayedQuery === undefined) {
+      showAndLogErrorMessage(
+        "Failed to sort results since evaluation info was unknown."
+      );
+      return;
+    }
+    // Notify the webview that it should expect new results.
+    await this.postMessage({ t: "resultsUpdating" });
+    await update(this._displayedQuery);
+    await this.showResults(
+      this._displayedQuery,
+      WebviewReveal.NotForced,
+      true
+    );
+  }
+
+  private async handleMsgFromView(
+    msg: FromResultsViewMsg
+  ): Promise<void> {
     switch (msg.t) {
-      case 'viewSourceFile': {
+      case "viewSourceFile": {
-        const databaseItem = this.databaseManager.findDatabaseItem(Uri.parse(msg.databaseUri));
+        const databaseItem = this.databaseManager.findDatabaseItem(
+          Uri.parse(msg.databaseUri)
+        );
         if (databaseItem !== undefined) {
           try {
             await showLocation(msg.loc, databaseItem);
-          }
+          } catch (e) {
-          catch (e) {
             if (e instanceof Error) {
               if (e.message.match(/File not found/)) {
-                vscode.window.showErrorMessage(`Original file of this result is not in the database's source archive.`);
+                vscode.window.showErrorMessage(
+                  `Original file of this result is not in the database's source archive.`
+                );
+              } else {
+                this.logger.log(
+                  `Unable to handleMsgFromView: ${e.message}`
+                );
               }
-              else {
+            } else {
-                this.logger.log(`Unable to handleMsgFromView: ${e.message}`);
-              }
-            }
-            else {
               this.logger.log(`Unable to handleMsgFromView: ${e}`);
             }
           }
         }
         break;
       }
-      case 'toggleDiagnostics': {
+      case "toggleDiagnostics": {
         if (msg.visible) {
-          const databaseItem = this.databaseManager.findDatabaseItem(Uri.parse(msg.databaseUri));
+          const databaseItem = this.databaseManager.findDatabaseItem(
+            Uri.parse(msg.databaseUri)
+          );
           if (databaseItem !== undefined) {
-            await this.showResultsAsDiagnostics(msg.resultsPath, msg.kind, databaseItem);
+            await this.showResultsAsDiagnostics(
+              msg.origResultsPaths,
+              msg.metadata,
+              databaseItem
+            );
           }
         } else {
           // TODO: Only clear diagnostics on the same database.
@@ -172,17 +271,20 @@ export class InterfaceManager extends DisposableObject {
         this._panelLoadedCallBacks.forEach(cb => cb());
         this._panelLoadedCallBacks = [];
         break;
-      case 'changeSort': {
+      case "changeSort":
-        if (this._displayedEvaluationInfo === undefined) {
+        await this.changeSortState(query =>
-          showAndLogErrorMessage("Failed to sort results since evaluation info was unknown.");
+          query.updateSortState(
-          break;
+            this.cliServer,
-        }
+            msg.resultSetName,
-        // Notify the webview that it should expect new results.
+            msg.sortState
-        await this.postMessage({ t: 'resultsUpdating' });
+          )
-        await this._displayedEvaluationInfo.query.updateSortState(this.cliServer, msg.resultSetName, msg.sortState);
+        );
-        await this.showResults(this._displayedEvaluationInfo, WebviewReveal.NotForced, true);
+        break;
+      case "changeInterpretedSort":
+        await this.changeSortState(query =>
+          query.updateInterpretedSortState(this.cliServer, msg.sortState)
+        );
         break;
-      }
       default:
         assertNever(msg);
     }
@@ -193,58 +295,63 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve, reject) => {
+    return new Promise(resolve => {
       if (this._panelLoaded) {
         resolve();
       } else {
-        this._panelLoadedCallBacks.push(resolve)
+        this._panelLoadedCallBacks.push(resolve);
       }
-    })
+    });
   }
 
   /**
-   * Show query results in webview panel.
+  * Show query results in webview panel.
-   * @param info Evaluation info for the executed query.
+  * @param results Evaluation info for the executed query.
-   * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
+  * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
-   * @param forceReveal Force the webview panel to be visible and
+  * @param forceReveal Force the webview panel to be visible and
-   * Appropriate when the user has just performed an explicit
+  * Appropriate when the user has just performed an explicit
-   * UI interaction requesting results, e.g. clicking on a query
+  * UI interaction requesting results, e.g. clicking on a query
-   * history entry.
+  * history entry.
-   */
+  */
-  public async showResults(info: EvaluationInfo, forceReveal: WebviewReveal, shouldKeepOldResultsWhileRendering: boolean = false): Promise<void> {
+  public async showResults(
-    if (info.result.resultType !== messages.QueryResultType.SUCCESS) {
+    results: CompletedQuery,
+    forceReveal: WebviewReveal,
+    shouldKeepOldResultsWhileRendering = false
+  ): Promise<void> {
+    if (results.result.resultType !== messages.QueryResultType.SUCCESS) {
       return;
     }
 
-    const interpretation = await this.interpretResultsInfo(info.query, info.query.resultsInfo);
+    const interpretation = await this.interpretResultsInfo(
+      results.query,
+      results.interpretedResultsSortState
+    );
 
     const sortedResultsMap: SortedResultsMap = {};
-    info.query.sortedResultsInfo.forEach((v, k) =>
+    results.sortedResultsInfo.forEach(
-      sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v));
+      (v, k) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(
+          v
+        ))
+    );
 
-    this._displayedEvaluationInfo = info;
+    this._displayedQuery = results;
 
     const panel = this.getPanel();
     await this.waitForPanelLoaded();
     if (forceReveal === WebviewReveal.Forced) {
       panel.reveal(undefined, true);
-    }
+    } else if (!panel.visible) {
-    else if (!panel.visible) {
       // The results panel exists, (`.getPanel()` guarantees it) but
       // is not visible; it's in a not-currently-viewed tab. Show a
       // more asynchronous message to not so abruptly interrupt
       // user's workflow by immediately revealing the panel.
-      const showButton = 'View Results';
+      const showButton = "View Results";
-      const queryName = helpers.getQueryName(info);
+      const queryName = results.queryName;
-      let queryNameForMessage: string;
-      if (queryName.length > 0) {
-        // lower case the first character
-        queryNameForMessage = queryName.charAt(0).toLowerCase() + queryName.substring(1);
-      } else {
-        queryNameForMessage = 'query';
-      }
       const resultPromise = vscode.window.showInformationMessage(
-        `Finished running ${queryNameForMessage}.`,
+        `Finished running query ${
+          queryName.length > 0 ? ` “${queryName}”` : ""
+        }.`,
         showButton
       );
       // Address this click asynchronously so we still update the
@@ -257,135 +364,219 @@ export class InterfaceManager extends DisposableObject {
     }
 
     await this.postMessage({
-      t: 'setState',
+      t: "setState",
       interpretation,
-      resultsPath: this.convertPathToWebviewUri(info.query.resultsInfo.resultsPath),
+      origResultsPaths: results.query.resultsPaths,
+      resultsPath: this.convertPathToWebviewUri(
+        results.query.resultsPaths.resultsPath
+      ),
       sortedResultsMap,
-      database: info.database,
+      database: results.database,
       shouldKeepOldResultsWhileRendering,
-      kind: info.query.metadata ? info.query.metadata.kind : undefined
+      metadata: results.query.metadata
     });
   }
 
-  private async interpretResultsInfo(query: QueryInfo, resultsInfo: ResultsInfo): Promise<Interpretation | undefined> {
+  private async getTruncatedResults(
+    metadata: QueryMetadata | undefined,
+    resultsPaths: ResultsPaths,
+    sourceInfo: cli.SourceInfo | undefined,
+    sourceLocationPrefix: string,
+    sortState: InterpretedResultsSortState | undefined
+  ): Promise<Interpretation> {
+    const sarif = await interpretResults(
+      this.cliServer,
+      metadata,
+      resultsPaths.resultsPath,
+      sourceInfo
+    );
+    // For performance reasons, limit the number of results we try
+    // to serialize and send to the webview. TODO: possibly also
+    // limit number of paths per result, number of steps per path,
+    // or throw an error if we are in aggregate trying to send
+    // massively too much data, as it can make the extension
+    // unresponsive.
+
+    let numTruncatedResults = 0;
+    sarif.runs.forEach(run => {
+      if (run.results !== undefined) {
+        sortInterpretedResults(run.results, sortState);
+        if (run.results.length > INTERPRETED_RESULTS_PER_RUN_LIMIT) {
+          numTruncatedResults +=
+            run.results.length - INTERPRETED_RESULTS_PER_RUN_LIMIT;
+          run.results = run.results.slice(
+            0,
+            INTERPRETED_RESULTS_PER_RUN_LIMIT
+          );
+        }
+      }
+    });
+    return {
+      sarif,
+      sourceLocationPrefix,
+      numTruncatedResults,
+      sortState
+    };
+  }
+
+  private async interpretResultsInfo(
+    query: QueryInfo,
+    sortState: InterpretedResultsSortState | undefined
+  ): Promise<Interpretation | undefined> {
     let interpretation: Interpretation | undefined = undefined;
-    if (query.hasInterpretedResults()
+    if (
-      && query.quickEvalPosition === undefined // never do results interpretation if quickEval
+      (await query.hasInterpretedResults()) &&
+      query.quickEvalPosition === undefined // never do results interpretation if quickEval
     ) {
       try {
-        const sourceLocationPrefix = await query.dbItem.getSourceLocationPrefix(this.cliServer);
+        const sourceLocationPrefix = await query.dbItem.getSourceLocationPrefix(
+          this.cliServer
+        );
         const sourceArchiveUri = query.dbItem.sourceArchive;
-        const sourceInfo = sourceArchiveUri === undefined ?
+        const sourceInfo =
-          undefined :
+          sourceArchiveUri === undefined
-          { sourceArchive: sourceArchiveUri.fsPath, sourceLocationPrefix };
+            ? undefined
-        const sarif = await interpretResults(this.cliServer, query, resultsInfo, sourceInfo);
+            : {
-        // For performance reasons, limit the number of results we try
+              sourceArchive: sourceArchiveUri.fsPath,
-        // to serialize and send to the webview. TODO: possibly also
+              sourceLocationPrefix
-        // limit number of paths per result, number of steps per path,
+            };
-        // or throw an error if we are in aggregate trying to send
+        interpretation = await this.getTruncatedResults(
-        // massively too much data, as it can make the extension
+          query.metadata,
-        // unresponsive.
+          query.resultsPaths,
-        let numTruncatedResults = 0;
+          sourceInfo,
-        sarif.runs.forEach(run => {
+          sourceLocationPrefix,
-          if (run.results !== undefined) {
+          sortState
-            if (run.results.length > INTERPRETED_RESULTS_PER_RUN_LIMIT) {
+        );
-              numTruncatedResults += run.results.length - INTERPRETED_RESULTS_PER_RUN_LIMIT;
+      } catch (e) {
-              run.results = run.results.slice(0, INTERPRETED_RESULTS_PER_RUN_LIMIT);
-            }
-          }
-        });
-        interpretation = { sarif, sourceLocationPrefix, numTruncatedResults };
-      }
-      catch (e) {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
-        this.logger.log(`Exception during results interpretation: ${e.message}. Will show raw results instead.`);
+        this.logger.log(
+          `Exception during results interpretation: ${e.message}. Will show raw results instead.`
+        );
       }
     }
-
     return interpretation;
   }
 
-  private async showResultsAsDiagnostics(resultsPath: string, kind: string | undefined,
+  private async showResultsAsDiagnostics(
-    database: DatabaseItem) {
+    resultsInfo: ResultsPaths,
+    metadata: QueryMetadata | undefined,
+    database: DatabaseItem
+  ): Promise<void> {
+    const sourceLocationPrefix = await database.getSourceLocationPrefix(
+      this.cliServer
+    );
+    const sourceArchiveUri = database.sourceArchive;
+    const sourceInfo =
+      sourceArchiveUri === undefined
+        ? undefined
+        : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix
+        };
+    const interpretation = await this.getTruncatedResults(
+      metadata,
+      resultsInfo,
+      sourceInfo,
+      sourceLocationPrefix,
+      undefined
+    );
 
-    // URIs from the webview have the vscode-resource scheme, so convert into a filesystem URI first.
-    const resultsPathOnDisk = webviewUriToFileUri(resultsPath).fsPath;
-    const fileReader = await FileReader.open(resultsPathOnDisk);
     try {
-      const resultSets = await bqrs.open(fileReader);
+      await this.showProblemResultsAsDiagnostics(
-      try {
+        interpretation,
-        switch (kind || 'problem') {
+        database
-          case 'problem': {
+      );
-            const customResults = bqrs.createCustomResultSets<ProblemQueryResults>(resultSets, ProblemQueryResults);
+    } catch (e) {
-            await this.showProblemResultsAsDiagnostics(customResults, database);
+      const msg = e instanceof Error ? e.message : e.toString();
-          }
+      this.logger.log(
-            break;
+        `Exception while computing problem results as diagnostics: ${msg}`
-
+      );
-          case 'path-problem': {
+      this._diagnosticCollection.clear();
-            const customResults = bqrs.createCustomResultSets<PathProblemQueryResults>(resultSets, PathProblemQueryResults);
-            await this.showProblemResultsAsDiagnostics(customResults, database);
-          }
-            break;
-
-          default:
-            throw new Error(`Unrecognized query kind '${kind}'.`);
-        }
-      }
-      catch (e) {
-        const msg = e instanceof Error ? e.message : e.toString();
-        this.logger.log(`Exception while computing problem results as diagnostics: ${msg}`);
-        this._diagnosticCollection.clear();
-      }
-    }
-    finally {
-      fileReader.dispose();
     }
   }
 
-  private async showProblemResultsAsDiagnostics(results: CustomResultSets<ProblemQueryResults>,
+  private async showProblemResultsAsDiagnostics(
-    databaseItem: DatabaseItem): Promise<void> {
+    interpretation: Interpretation,
+    databaseItem: DatabaseItem
+  ): Promise<void> {
+    const { sarif, sourceLocationPrefix } = interpretation;
+
+    if (!sarif.runs || !sarif.runs[0].results) {
+      this.logger.log(
+        "Didn't find a run in the sarif results. Error processing sarif?"
+      );
+      return;
+    }
 
     const diagnostics: [Uri, ReadonlyArray<Diagnostic>][] = [];
-    for await (const problemRow of results.problems.readTuples()) {
+
-      const codeLocation = resolveLocation(problemRow.element.location, databaseItem);
+    for (const result of sarif.runs[0].results) {
-      let message: string;
+      const message = result.message.text;
-      const references = problemRow.references;
+      if (message === undefined) {
-      if (references) {
+        this.logger.log("Sarif had result without plaintext message");
-        let referenceIndex = 0;
+        continue;
-        message = problemRow.message.replace(/\$\@/g, sub => {
-          if (referenceIndex < references.length) {
-            const replacement = references[referenceIndex].text;
-            referenceIndex++;
-            return replacement;
-          }
-          else {
-            return sub;
-          }
-        });
       }
-      else {
+      if (!result.locations) {
-        message = problemRow.message;
+        this.logger.log("Sarif had result without location");
+        continue;
       }
-      const diagnostic = new Diagnostic(codeLocation.range, message, DiagnosticSeverity.Warning);
+
-      if (problemRow.references) {
+      const sarifLoc = parseSarifLocation(
-        const relatedInformation: DiagnosticRelatedInformation[] = [];
+        result.locations[0],
-        for (const reference of problemRow.references) {
+        sourceLocationPrefix
-          const referenceLocation = tryResolveLocation(reference.element.location, databaseItem);
+      );
+      if (sarifLoc.t == "NoLocation") {
+        continue;
+      }
+      const resultLocation = tryResolveLocation(sarifLoc, databaseItem);
+      if (!resultLocation) {
+        this.logger.log("Sarif location was not resolvable " + sarifLoc);
+        continue;
+      }
+      const parsedMessage = parseSarifPlainTextMessage(message);
+      const relatedInformation: DiagnosticRelatedInformation[] = [];
+      const relatedLocationsById: { [k: number]: Sarif.Location } = {};
+
+      for (const loc of result.relatedLocations || []) {
+        relatedLocationsById[loc.id!] = loc;
+      }
+      const resultMessageChunks: string[] = [];
+      for (const section of parsedMessage) {
+        if (typeof section === "string") {
+          resultMessageChunks.push(section);
+        } else {
+          resultMessageChunks.push(section.text);
+          const sarifChunkLoc = parseSarifLocation(
+            relatedLocationsById[section.dest],
+            sourceLocationPrefix
+          );
+          if (sarifChunkLoc.t == "NoLocation") {
+            continue;
+          }
+          const referenceLocation = tryResolveLocation(
+            sarifChunkLoc,
+            databaseItem
+          );
+
           if (referenceLocation) {
-            const related = new DiagnosticRelatedInformation(referenceLocation,
+            const related = new DiagnosticRelatedInformation(
-              reference.text);
+              referenceLocation,
+              section.text
+            );
             relatedInformation.push(related);
           }
         }
-        diagnostic.relatedInformation = relatedInformation;
       }
-      diagnostics.push([
+      const diagnostic = new Diagnostic(
-        codeLocation.uri,
+        resultLocation.range,
-        [diagnostic]
+        resultMessageChunks.join(""),
-      ]);
+        DiagnosticSeverity.Warning
-    }
+      );
+      diagnostic.relatedInformation = relatedInformation;
 
+      diagnostics.push([resultLocation.uri, [diagnostic]]);
+    }
     this._diagnosticCollection.set(diagnostics);
   }
 
@@ -393,18 +584,22 @@ export class InterfaceManager extends DisposableObject {
     return fileUriToWebviewUri(this.getPanel(), Uri.file(path));
   }
 
-  private convertPathPropertiesToWebviewUris(info: SortedResultSetInfo): SortedResultSetInfo {
+  private convertPathPropertiesToWebviewUris(
+    info: SortedResultSetInfo
+  ): SortedResultSetInfo {
     return {
       resultsPath: this.convertPathToWebviewUri(info.resultsPath),
       sortState: info.sortState
     };
   }
 
-  private handleSelectionChange(event: vscode.TextEditorSelectionChangeEvent) {
+  private handleSelectionChange(
+    event: vscode.TextEditorSelectionChangeEvent
+  ): void {
     if (event.kind === vscode.TextEditorSelectionChangeKind.Command) {
       return; // Ignore selection events we caused ourselves.
     }
-    let editor = vscode.window.activeTextEditor;
+    const editor = vscode.window.activeTextEditor;
     if (editor !== undefined) {
       editor.setDecorations(shownLocationDecoration, []);
       editor.setDecorations(shownLocationLineDecoration, []);
@@ -428,8 +623,11 @@ async function showLocation(loc: ResolvableLocationValue, databaseItem: Database
   const resolvedLocation = tryResolveLocation(loc, databaseItem);
   if (resolvedLocation) {
     const doc = await workspace.openTextDocument(resolvedLocation.uri);
-    const editor = await Window.showTextDocument(doc, vscode.ViewColumn.One);
+    const editorsWithDoc = Window.visibleTextEditors.filter(e => e.document === doc);
-    let range = resolvedLocation.range;
+    const editor = editorsWithDoc.length > 0
+      ? editorsWithDoc[0]
+      : await Window.showTextDocument(doc, vscode.ViewColumn.One);
+    const range = resolvedLocation.range;
     // When highlighting the range, vscode's occurrence-match and bracket-match highlighting will
     // trigger based on where we place the cursor/selection, and will compete for the user's attention.
     // For reference:
@@ -440,9 +638,9 @@ async function showLocation(loc: ResolvableLocationValue, databaseItem: Database
     // For single-line ranges, select the whole range, mainly to disable bracket highlighting.
     // For multi-line ranges, place the cursor at the beginning to avoid visual artifacts from selected line-breaks.
     // Multi-line ranges are usually large enough to overshadow the noise from bracket highlighting.
-    let selectionEnd = (range.start.line === range.end.line)
+    const selectionEnd = (range.start.line === range.end.line)
-        ? range.end
+      ? range.end
-        : range.start;
+      : range.start;
     editor.selection = new vscode.Selection(range.start, selectionEnd);
     editor.revealRange(range, vscode.TextEditorRevealType.InCenter);
     editor.setDecorations(shownLocationDecoration, [range]);
@@ -477,22 +675,6 @@ function resolveWholeFileLocation(loc: WholeFileLocation, databaseItem: Database
   return new Location(databaseItem.resolveSourceFile(loc.file), range);
 }
 
-/**
- * Resolve the specified CodeQL location to a URI into the source archive.
- * @param loc CodeQL location to resolve
- * @param databaseItem Database in which to resolve the file location.
- */
-function resolveLocation(loc: LocationValue | undefined, databaseItem: DatabaseItem): Location {
-  const resolvedLocation = tryResolveLocation(loc, databaseItem);
-  if (resolvedLocation) {
-    return resolvedLocation;
-  }
-  else {
-    // Return a fake position in the source archive directory itself.
-    return new Location(databaseItem.resolveSourceFile(undefined), new Position(0, 0));
-  }
-}
-
 /**
  * Try to resolve the specified CodeQL location to a URI into the source archive. If no exact location
  * can be resolved, returns `undefined`.

extensions/ql-vscode/src/logging.ts

@@ -1,33 +1,135 @@
-import { window as Window, OutputChannel, Progress } from 'vscode';
+import { window as Window, OutputChannel, Progress, ExtensionContext, Disposable } from 'vscode';
 import { DisposableObject } from 'semmle-vscode-utils';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+interface LogOptions {
+  /** If false, don't output a trailing newline for the log entry. Default true. */
+  trailingNewline?: boolean;
+
+  /** If specified, add this log entry to the log file at the specified location. */
+  additionalLogLocation?: string;
+}
 
 export interface Logger {
-  /** Writes the given log message, followed by a newline. */
+  /** Writes the given log message, optionally followed by a newline. */
-  log(message: string): void;
+  log(message: string, options?: LogOptions): Promise<void>;
-  /** Writes the given log message, not followed by a newline. */
+  /**
-  logWithoutTrailingNewline(message: string): void;
+   * Reveal this channel in the UI.
+   *
+   * @param preserveFocus When `true` the channel will not take focus.
+   */
+  show(preserveFocus?: boolean): void;
+
+  /**
+   * Remove the log at the specified location
+   * @param location log to remove
+   */
+  removeAdditionalLogLocation(location: string | undefined): void;
+
+  /**
+   * The base location location where all side log files are stored.
+   */
+  getBaseLocation(): string | undefined;
 }
 
 export type ProgressReporter = Progress<{ message: string }>;
 
 /** A logger that writes messages to an output channel in the Output tab. */
 export class OutputChannelLogger extends DisposableObject implements Logger {
-  outputChannel: OutputChannel;
+  public readonly outputChannel: OutputChannel;
+  private readonly additionalLocations = new Map<string, AdditionalLogLocation>();
+  private additionalLogLocationPath: string | undefined;
 
-  constructor(title: string) {
+  constructor(private title: string) {
     super();
     this.outputChannel = Window.createOutputChannel(title);
     this.push(this.outputChannel);
   }
 
-  log(message: string) {
+  init(ctx: ExtensionContext): void {
-    this.outputChannel.appendLine(message);
+    this.additionalLogLocationPath = path.join(ctx.storagePath || ctx.globalStoragePath, this.title);
+
+    // clear out any old state from previous runs
+    fs.remove(this.additionalLogLocationPath);
   }
 
-  logWithoutTrailingNewline(message: string) {
+  /**
-    this.outputChannel.append(message);
+   * This function is asynchronous and will only resolve once the message is written
+   * to the side log (if required). It is not necessary to await the results of this
+   * function if you don't need to guarantee that the log writing is complete before
+   * continuing.
+   */
+  async log(message: string, options = { } as LogOptions): Promise<void> {
+    if (options.trailingNewline === undefined) {
+      options.trailingNewline = true;
+    }
+
+    if (options.trailingNewline) {
+      this.outputChannel.appendLine(message);
+    } else {
+      this.outputChannel.append(message);
+    }
+
+    if (this.additionalLogLocationPath && options.additionalLogLocation) {
+      const logPath = path.join(this.additionalLogLocationPath, options.additionalLogLocation);
+      let additional = this.additionalLocations.get(logPath);
+      if (!additional) {
+        const msg = `| Log being saved to ${logPath} |`;
+        const separator = new Array(msg.length).fill('-').join('');
+        this.outputChannel.appendLine(separator);
+        this.outputChannel.appendLine(msg);
+        this.outputChannel.appendLine(separator);
+        additional = new AdditionalLogLocation(logPath);
+        this.additionalLocations.set(logPath, additional);
+        this.track(additional);
+      }
+
+      await additional.log(message, options);
+    }
   }
 
+  show(preserveFocus?: boolean): void {
+    this.outputChannel.show(preserveFocus);
+  }
+
+  removeAdditionalLogLocation(location: string | undefined): void {
+    if (this.additionalLogLocationPath && location) {
+      const logPath = location.startsWith(this.additionalLogLocationPath)
+        ? location
+        : path.join(this.additionalLogLocationPath, location);
+      const additional = this.additionalLocations.get(logPath);
+      if (additional) {
+        this.disposeAndStopTracking(additional);
+        this.additionalLocations.delete(logPath);
+      }
+    }
+  }
+
+  getBaseLocation() {
+    return this.additionalLogLocationPath;
+  }
+}
+
+class AdditionalLogLocation extends Disposable {
+  constructor(private location: string) {
+    super(() => { /**/ });
+  }
+
+  async log(message: string, options = { } as LogOptions): Promise<void> {
+    if (options.trailingNewline === undefined) {
+      options.trailingNewline = true;
+    }
+    await fs.ensureFile(this.location);
+
+    await fs.appendFile(this.location, message + (options.trailingNewline ? '\n' : ''), {
+      encoding: 'utf8'
+    });
+  }
+
+  async dispose(): Promise<void> {
+    await fs.remove(this.location);
+  }
 }
 
 /** The global logger for the extension. */
@@ -37,4 +139,9 @@ export const logger = new OutputChannelLogger('CodeQL Extension Log');
 export const queryServerLogger = new OutputChannelLogger('CodeQL Query Server');
 
 /** The logger for messages from the language server. */
-export const ideServerLogger = new OutputChannelLogger('CodeQL Language Server');
+export const ideServerLogger = new OutputChannelLogger(
+  'CodeQL Language Server'
+);
+
+/** The logger for messages from tests. */
+export const testLogger = new OutputChannelLogger('CodeQL Tests');

extensions/ql-vscode/src/messages.ts

@@ -1,6 +1,17 @@
 /**
  * Types for messages exchanged during jsonrpc communication with the
  * the CodeQL query server.
+ *
+ * This file exists in the queryserver and in the vscode extension, and
+ * should be kept in sync between them.
+ *
+ * A note about the namespaces below, which look like they are
+ * essentially enums, namely Severity, ResultColumnKind, and
+ * QueryResultType. By design, for the sake of extensibility, clients
+ * receiving messages of this protocol are supposed to accept any
+ * number for any of these types. We commit to the given meaning of
+ * the numbers listed in constants in the namespaces, and we commit to
+ * the fact that any unknown QueryResultType value counts as an error.
  */
 
 import * as rpc from 'vscode-jsonrpc';
@@ -205,19 +216,19 @@ export interface QlFileSet {
   /**
    * The files imported by the given file
    */
-  imports: { [key: string]: string[]; };
+  imports: { [key: string]: string[] };
   /**
    * An id of each file
    */
-  nodeNumbering: { [key: string]: number; };
+  nodeNumbering: { [key: string]: number };
   /**
    * The code for each file
    */
-  qlCode: { [key: string]: string; };
+  qlCode: { [key: string]: string };
   /**
    * The resolution of an import in each directory.
    */
-  resolvedDirImports: { [key: string]: { [key: string]: string; }; };
+  resolvedDirImports: { [key: string]: { [key: string]: string } };
 }
 
 /**
@@ -294,11 +305,15 @@ export interface CompilationMessage {
   /**
    * The severity of the message
    */
-  severity: number;
+  severity: Severity;
 }
+
+export type Severity = number;
 /**
- * Severity of different messages
+ * Severity of different messages. This namespace is intentionally not
+ * an enum, see "for the sake of extensibility" comment above.
  */
+// eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace Severity {
   /**
    * The message is a compilation error.
@@ -333,7 +348,7 @@ export interface ResultColumn {
    * The kind of the column. See `ResultColumnKind`
    * for the current possible meanings
    */
-  kind: number;
+  kind: ResultColumnKind;
   /**
    * The name of the column.
    * This may be compiler generated for complex select expressions.
@@ -341,9 +356,12 @@ export interface ResultColumn {
   name: string;
 }
 
+export type ResultColumnKind = number;
 /**
- * The kind of a result column.
+ * The kind of a result column. This namespace is intentionally not an enum, see "for the sake of
+ * extensibility" comment above.
  */
+// eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace ResultColumnKind {
   /**
    * A column of type `float`
@@ -619,6 +637,8 @@ export interface EvaluateQueriesParams {
   useSequenceHint: boolean;
 }
 
+export type TemplateDefinitions = { [key: string]: TemplateSource }
+
 /**
  * A single query that should be run
  */
@@ -642,7 +662,7 @@ export interface QueryToRun {
   /**
    * Values to set for each template
    */
-  templateValues?: { [key: string]: TemplateSource; };
+  templateValues?: TemplateDefinitions;
   /**
    * Whether templates without values in the templateValues
    * map should be set to the empty set or give an error.
@@ -730,7 +750,7 @@ export interface ResultSet {
 /**
  * The type returned when the evaluation is complete
  */
-export interface EvaluationComplete { }
+export type EvaluationComplete = {};
 
 /**
  * The result of a single query
@@ -748,7 +768,7 @@ export interface EvaluationResult {
    * The type of the result. See QueryResultType for
    * possible meanings. Any other result should be interpreted as an error.
    */
-  resultType: number;
+  resultType: QueryResultType;
   /**
    * The wall clock time it took to evaluate the query.
    * The time is from when we initially tried to evaluate the query
@@ -760,11 +780,19 @@ export interface EvaluationResult {
    * An error message if an error happened
    */
   message?: string;
+
+  /**
+   * Full path to file with all log messages emitted while this query was active, if one exists
+   */
+  logFileLocation?: string;
 }
 
+export type QueryResultType = number;
 /**
- * The result of running a query,
+ * The result of running a query. This namespace is intentionally not
+ * an enum, see "for the sake of extensibility" comment above.
  */
+// eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace QueryResultType {
   /**
    * The query ran successfully
@@ -818,7 +846,7 @@ export interface RunUpgradeResult {
    * The type of the result. See QueryResultType for
    * possible meanings. Any other result should be interpreted as an error.
    */
-  resultType: number;
+  resultType: QueryResultType;
   /**
    * The error message if an error occurred
    */
@@ -837,11 +865,11 @@ export interface WithProgressId<T> {
   /**
    * The main body
    */
-  body: T,
+  body: T;
   /**
    * The id used to report progress updates
    */
-  progressId: number
+  progressId: number;
 }
 
 export interface ProgressMessage {
@@ -910,7 +938,7 @@ export const runUpgrade = new rpc.RequestType<WithProgressId<RunUpgradeParams>,
  * Request returned to the client to notify completion of a query.
  * The full runQueries job is completed when all queries are acknowledged.
  */
-export const completeQuery = new rpc.RequestType<EvaluationResult, Object, void, void>('evaluation/queryCompleted');
+export const completeQuery = new rpc.RequestType<EvaluationResult, Record<string, any>, void, void>('evaluation/queryCompleted');
 
 /**
  * A notification that the progress has been changed.

extensions/ql-vscode/src/qlpack-discovery.ts

@@ -0,0 +1,59 @@
+import { EventEmitter, Event, Uri, WorkspaceFolder, RelativePattern } from 'vscode';
+import { MultiFileSystemWatcher } from 'semmle-vscode-utils';
+import { CodeQLCliServer, QlpacksInfo } from './cli';
+import { Discovery } from './discovery';
+
+export interface QLPack {
+  name: string;
+  uri: Uri;
+}
+
+/**
+ * Service to discover all available QL packs in a workspace folder.
+ */
+export class QLPackDiscovery extends Discovery<QlpacksInfo> {
+  private readonly _onDidChangeQLPacks = this.push(new EventEmitter<void>());
+  private readonly watcher = this.push(new MultiFileSystemWatcher());
+  private _qlPacks: readonly QLPack[] = [];
+
+  constructor(private readonly workspaceFolder: WorkspaceFolder,
+    private readonly cliServer: CodeQLCliServer) {
+
+    super();
+
+    // Watch for any changes to `qlpack.yml` files in this workspace folder.
+    // TODO: The CLI server should tell us what paths to watch for.
+    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/qlpack.yml'));
+    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/.codeqlmanifest.json'));
+    this.push(this.watcher.onDidChange(this.handleQLPackFileChanged, this));
+
+    this.refresh();
+  }
+
+  public get onDidChangeQLPacks(): Event<void> { return this._onDidChangeQLPacks.event; }
+
+  public get qlPacks(): readonly QLPack[] { return this._qlPacks; }
+
+  private handleQLPackFileChanged(_uri: Uri): void {
+    this.refresh();
+  }
+
+  protected discover(): Promise<QlpacksInfo> {
+    // Only look for QL packs in this workspace folder.
+    return this.cliServer.resolveQlpacks([this.workspaceFolder.uri.fsPath], []);
+  }
+
+  protected update(results: QlpacksInfo): void {
+    const qlPacks: QLPack[] = [];
+    for (const id in results) {
+      qlPacks.push(...results[id].map(fsPath => {
+        return {
+          name: id,
+          uri: Uri.file(fsPath)
+        };
+      }));
+    }
+    this._qlPacks = qlPacks;
+    this._onDidChangeQLPacks.fire();
+  }
+}

extensions/ql-vscode/src/qltest-discovery.ts

@@ -0,0 +1,222 @@
+import * as path from 'path';
+import { QLPackDiscovery } from './qlpack-discovery';
+import { Discovery } from './discovery';
+import { EventEmitter, Event, Uri, RelativePattern } from 'vscode';
+import { MultiFileSystemWatcher } from 'semmle-vscode-utils';
+import { CodeQLCliServer } from './cli';
+
+/**
+ * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`.
+ */
+export abstract class QLTestNode {
+  constructor(private _path: string, private _name: string) {
+  }
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public abstract get children(): readonly QLTestNode[];
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more QL tests or other test directories.
+ */
+export class QLTestDirectory extends QLTestNode {
+  private _children: QLTestNode[] = [];
+
+  constructor(_path: string, _name: string) {
+    super(_path, _name);
+  }
+
+  public get children(): readonly QLTestNode[] {
+    return this._children;
+  }
+
+  public addChild(child: QLTestNode): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): QLTestDirectory {
+    const dirName = path.dirname(relativePath);
+    if (dirName === '.') {
+      return this.createChildDirectory(relativePath);
+    }
+    else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(path.basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    this._children.sort((a, b) => a.name.localeCompare(b.name));
+    for (const child of this._children) {
+      child.finish();
+    }
+  }
+
+  private createChildDirectory(name: string): QLTestDirectory {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return <QLTestDirectory>existingChild;
+    }
+    else {
+      const newChild = new QLTestDirectory(path.join(this.path, name), name);
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single QL test. This will be either a `.ql` file or a `.qlref` file.
+ */
+export class QLTestFile extends QLTestNode {
+  constructor(_path: string, _name: string) {
+    super(_path, _name);
+  }
+
+  public get children(): readonly QLTestNode[] {
+    return [];
+  }
+
+  public finish(): void {
+  }
+}
+
+/**
+ * The results of discovering QL tests.
+ */
+interface QLTestDiscoveryResults {
+  /**
+   * The root test directory for each QL pack that contains tests.
+   */
+  testDirectories: QLTestDirectory[];
+  /**
+   * The list of file system paths to watch. If any of these paths changes, the discovery results
+   * may be out of date.
+   */
+  watchPaths: string[];
+}
+
+/**
+ * Discovers all QL tests contained in the QL packs in a given workspace folder.
+ */
+export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
+  private readonly _onDidChangeTests = this.push(new EventEmitter<void>());
+  private readonly watcher: MultiFileSystemWatcher = this.push(new MultiFileSystemWatcher());
+  private _testDirectories: QLTestDirectory[] = [];
+
+  constructor(private readonly qlPackDiscovery: QLPackDiscovery,
+    private readonly cliServer: CodeQLCliServer) {
+
+    super();
+
+    this.push(this.qlPackDiscovery.onDidChangeQLPacks(this.handleDidChangeQLPacks, this));
+    this.push(this.watcher.onDidChange(this.handleDidChange, this));
+
+    this.refresh();
+  }
+
+  /**
+   * Event to be fired when the set of discovered tests may have changed.
+   */
+  public get onDidChangeTests(): Event<void> { return this._onDidChangeTests.event; }
+
+  /**
+   * The root test directory for each QL pack that contains tests.
+   */
+  public get testDirectories(): QLTestDirectory[] { return this._testDirectories; }
+
+  private handleDidChangeQLPacks(): void {
+    this.refresh();
+  }
+
+  private handleDidChange(uri: Uri): void {
+    if (!QLTestDiscovery.ignoreTestPath(uri.fsPath)) {
+      this.refresh();
+    }
+  }
+
+  protected async discover(): Promise<QLTestDiscoveryResults> {
+    const testDirectories: QLTestDirectory[] = [];
+    const watchPaths: string[] = [];
+    const qlPacks = this.qlPackDiscovery.qlPacks;
+    for (const qlPack of qlPacks) {
+      //HACK: Assume that only QL packs whose name ends with '-tests' contain tests.
+      if (qlPack.name.endsWith('-tests')) {
+        watchPaths.push(qlPack.uri.fsPath);
+        const testPackage = await this.discoverTests(qlPack.uri.fsPath, qlPack.name);
+        if (testPackage !== undefined) {
+          testDirectories.push(testPackage);
+        }
+      }
+    }
+
+    return {
+      testDirectories: testDirectories,
+      watchPaths: watchPaths
+    };
+  }
+
+  protected update(results: QLTestDiscoveryResults): void {
+    this._testDirectories = results.testDirectories;
+
+    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
+    this.watcher.clear();
+    results.watchPaths.forEach(watchPath => {
+      this.watcher.addWatch(new RelativePattern(watchPath, '**/*.{ql,qlref}'));
+    });
+    this._onDidChangeTests.fire();
+  }
+
+  /**
+   * Discover all QL tests in the specified directory and its subdirectories.
+   * @param fullPath The full path of the test directory.
+   * @param name The display name to use for the returned `TestDirectory` object.
+   * @returns A `QLTestDirectory` object describing the contents of the directory, or `undefined` if
+   *   no tests were found.
+   */
+  private async discoverTests(fullPath: string, name: string): Promise<QLTestDirectory | undefined> {
+    const resolvedTests = (await this.cliServer.resolveTests(fullPath))
+      .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
+
+    if (resolvedTests.length === 0) {
+      return undefined;
+    }
+    else {
+      const rootDirectory = new QLTestDirectory(fullPath, name);
+      for (const testPath of resolvedTests) {
+        const relativePath = path.normalize(path.relative(fullPath, testPath));
+        const dirName = path.dirname(relativePath);
+        const parentDirectory = rootDirectory.createDirectory(dirName);
+        parentDirectory.addChild(new QLTestFile(testPath, path.basename(testPath)));
+      }
+
+      rootDirectory.finish();
+
+      return rootDirectory;
+    }
+  }
+
+  /**
+   * Determine if the specified QL test should be ignored based on its filename.
+   * @param testPath Path to the test file.
+   */
+  private static ignoreTestPath(testPath: string): boolean {
+    switch (path.extname(testPath).toLowerCase()) {
+      case '.ql':
+      case '.qlref':
+        return path.basename(testPath).startsWith('__');
+
+      default:
+        return false;
+    }
+  }
+}

extensions/ql-vscode/src/query-history.ts

@@ -1,8 +1,12 @@
+import * as path from 'path';
 import * as vscode from 'vscode';
 import { ExtensionContext, window as Window } from 'vscode';
-import { EvaluationInfo } from './queries';
+import { CompletedQuery } from './query-results';
+import { QueryHistoryConfig } from './config';
+import { QueryWithResults } from './run-queries';
 import * as helpers from './helpers';
-import * as messages from './messages';
+import { logger } from './logging';
+
 /**
  * query-history.ts
  * ------------
@@ -12,48 +16,20 @@ import * as messages from './messages';
  * `TreeDataProvider` subclass below.
  */
 
-/**
+export type QueryHistoryItemOptions = {
- * One item in the user-displayed list of queries that have been run.
+  label?: string; // user-settable label
- */
+  queryText?: string; // stored query for quick query
-export class QueryHistoryItem {
-  queryName: string;
-  time: string;
-  databaseName: string;
-  info: EvaluationInfo;
-
-  constructor(info: EvaluationInfo) {
-    this.queryName = helpers.getQueryName(info);
-    this.databaseName = info.database.name;
-    this.info = info;
-    this.time = new Date().toLocaleString();
-  }
-
-  get statusString(): string {
-    switch (this.info.result.resultType) {
-      case messages.QueryResultType.CANCELLATION:
-        return `cancelled after ${this.info.result.evaluationTime / 1000} seconds`;
-      case messages.QueryResultType.OOM:
-        return `out of memory`;
-      case messages.QueryResultType.SUCCESS:
-        return `finished in ${this.info.result.evaluationTime / 1000} seconds`;
-      case messages.QueryResultType.TIMEOUT:
-        return `timed out after ${this.info.result.evaluationTime / 1000} seconds`;
-      case messages.QueryResultType.OTHER_ERROR:
-      default:
-        return `failed`;
-    }
-  }
-
-  toString(): string {
-    const { databaseName, queryName, time } = this;
-    return `[${time}] ${queryName} on ${databaseName} - ${this.statusString}`;
-  }
 }
 
+/**
+ * Path to icon to display next to a failed query history item.
+ */
+const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
+
 /**
  * Tree data provider for the query history view.
  */
-class HistoryTreeDataProvider implements vscode.TreeDataProvider<QueryHistoryItem> {
+class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery> {
 
   /**
    * XXX: This idiom for how to get a `.fire()`-able event emitter was
@@ -61,23 +37,20 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<QueryHistoryIte
    * involved and I hope there's something better that can be done
    * instead.
    */
-  private _onDidChangeTreeData: vscode.EventEmitter<QueryHistoryItem | undefined> = new vscode.EventEmitter<QueryHistoryItem | undefined>();
+  private _onDidChangeTreeData: vscode.EventEmitter<CompletedQuery | undefined> = new vscode.EventEmitter<CompletedQuery | undefined>();
-  readonly onDidChangeTreeData: vscode.Event<QueryHistoryItem | undefined> = this._onDidChangeTreeData.event;
+  readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this._onDidChangeTreeData.event;
 
-  private ctx: ExtensionContext;
+  private history: CompletedQuery[] = [];
-  private history: QueryHistoryItem[] = [];
 
   /**
    * When not undefined, must be reference-equal to an item in `this.databases`.
    */
-  private current: QueryHistoryItem | undefined;
+  private current: CompletedQuery | undefined;
 
-  constructor(ctx: ExtensionContext) {
+  constructor(private ctx: ExtensionContext) {
-    this.ctx = ctx;
-    this.history = [];
   }
 
-  getTreeItem(element: QueryHistoryItem): vscode.TreeItem {
+  getTreeItem(element: CompletedQuery): vscode.TreeItem {
     const it = new vscode.TreeItem(element.toString());
 
     it.command = {
@@ -86,10 +59,14 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<QueryHistoryIte
       arguments: [element],
     };
 
+    if (!element.didRunSuccessfully) {
+      it.iconPath = path.join(this.ctx.extensionPath, FAILED_QUERY_HISTORY_ITEM_ICON);
+    }
+
     return it;
   }
 
-  getChildren(element?: QueryHistoryItem): vscode.ProviderResult<QueryHistoryItem[]> {
+  getChildren(element?: CompletedQuery): vscode.ProviderResult<CompletedQuery[]> {
     if (element == undefined) {
       return this.history;
     }
@@ -98,25 +75,25 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<QueryHistoryIte
     }
   }
 
-  getParent(element: QueryHistoryItem): vscode.ProviderResult<QueryHistoryItem> {
+  getParent(_element: CompletedQuery): vscode.ProviderResult<CompletedQuery> {
     return null;
   }
 
-  getCurrent(): QueryHistoryItem | undefined {
+  getCurrent(): CompletedQuery | undefined {
     return this.current;
   }
 
-  push(item: QueryHistoryItem): void {
+  push(item: CompletedQuery): void {
     this.current = item;
     this.history.push(item);
-    this._onDidChangeTreeData.fire();
+    this.refresh();
   }
 
-  setCurrentItem(item: QueryHistoryItem) {
+  setCurrentItem(item: CompletedQuery) {
     this.current = item;
   }
 
-  remove(item: QueryHistoryItem) {
+  remove(item: CompletedQuery) {
     if (this.current === item)
       this.current = undefined;
     const index = this.history.findIndex(i => i === item);
@@ -127,9 +104,13 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<QueryHistoryIte
         // are any available.
         this.current = this.history[Math.min(index, this.history.length - 1)];
       }
-      this._onDidChangeTreeData.fire();
+      this.refresh();
     }
   }
+
+  refresh() {
+    this._onDidChangeTreeData.fire();
+  }
 }
 
 /**
@@ -141,24 +122,31 @@ const DOUBLE_CLICK_TIME = 500;
 export class QueryHistoryManager {
   treeDataProvider: HistoryTreeDataProvider;
   ctx: ExtensionContext;
-  treeView: vscode.TreeView<QueryHistoryItem>;
+  treeView: vscode.TreeView<CompletedQuery>;
-  selectedCallback: ((item: QueryHistoryItem) => void) | undefined;
+  selectedCallback: ((item: CompletedQuery) => void) | undefined;
-  lastItemClick: { time: Date, item: QueryHistoryItem } | undefined;
+  lastItemClick: { time: Date; item: CompletedQuery } | undefined;
 
-  async invokeCallbackOn(queryHistoryItem: QueryHistoryItem) {
+  async invokeCallbackOn(queryHistoryItem: CompletedQuery) {
     if (this.selectedCallback !== undefined) {
       const sc = this.selectedCallback;
       await sc(queryHistoryItem);
     }
   }
 
-  async handleOpenQuery(queryHistoryItem: QueryHistoryItem) {
+  async handleOpenQuery(queryHistoryItem: CompletedQuery): Promise<void> {
-    const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.info.query.program.queryPath));
+    const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.query.program.queryPath));
-    await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+    const editor = await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+    const queryText = queryHistoryItem.options.queryText;
+    if (queryText !== undefined) {
+      await editor.edit(edit => edit.replace(textDocument.validateRange(
+        new vscode.Range(0, 0, textDocument.lineCount, 0)), queryText)
+      );
+    }
   }
 
-  async handleRemoveHistoryItem(queryHistoryItem: QueryHistoryItem) {
+  async handleRemoveHistoryItem(queryHistoryItem: CompletedQuery) {
     this.treeDataProvider.remove(queryHistoryItem);
+    queryHistoryItem.dispose();
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
       this.treeView.reveal(current);
@@ -166,7 +154,24 @@ export class QueryHistoryManager {
     }
   }
 
-  async handleItemClicked(queryHistoryItem: QueryHistoryItem) {
+  async handleSetLabel(queryHistoryItem: CompletedQuery) {
+    const response = await vscode.window.showInputBox({
+      prompt: 'Label:',
+      placeHolder: '(use default)',
+      value: queryHistoryItem.getLabel(),
+    });
+    // undefined response means the user cancelled the dialog; don't change anything
+    if (response !== undefined) {
+      if (response === '')
+        // Interpret empty string response as "go back to using default"
+        queryHistoryItem.options.label = undefined;
+      else
+        queryHistoryItem.options.label = response;
+      this.treeDataProvider.refresh();
+    }
+  }
+
+  async handleItemClicked(queryHistoryItem: CompletedQuery) {
     this.treeDataProvider.setCurrentItem(queryHistoryItem);
 
     const now = new Date();
@@ -185,27 +190,88 @@ export class QueryHistoryManager {
     }
   }
 
-  constructor(ctx: ExtensionContext, selectedCallback?: (item: QueryHistoryItem) => Promise<void>) {
+  async handleShowQueryLog(queryHistoryItem: CompletedQuery) {
+    if (queryHistoryItem.logFileLocation) {
+      const uri = vscode.Uri.parse(queryHistoryItem.logFileLocation);
+      try {
+        await vscode.window.showTextDocument(uri, {
+        });
+      } catch (e) {
+        if (e.message.includes('Files above 50MB cannot be synchronized with extensions')) {
+          const res = await helpers.showBinaryChoiceDialog('File is too large to open in the editor, do you want to open exterally?');
+          if (res) {
+            try {
+              await vscode.commands.executeCommand('revealFileInOS', uri);
+            } catch (e) {
+              helpers.showAndLogErrorMessage(e.message);
+            }
+          }
+        } else {
+          helpers.showAndLogErrorMessage(`Could not open log file ${queryHistoryItem.logFileLocation}`);
+          logger.log(e.message);
+          logger.log(e.stack);
+        }
+
+      }
+    } else {
+      helpers.showAndLogWarningMessage('No log file available');
+    }
+  }
+
+  constructor(
+    ctx: ExtensionContext,
+    private queryHistoryConfigListener: QueryHistoryConfig,
+    selectedCallback?: (item: CompletedQuery) => Promise<void>
+  ) {
     this.ctx = ctx;
     this.selectedCallback = selectedCallback;
     const treeDataProvider = this.treeDataProvider = new HistoryTreeDataProvider(ctx);
     this.treeView = Window.createTreeView('codeQLQueryHistory', { treeDataProvider });
+    // Lazily update the tree view selection due to limitations of TreeView API (see
+    // `updateTreeViewSelectionIfVisible` doc for details)
+    this.treeView.onDidChangeVisibility(async _ev => this.updateTreeViewSelectionIfVisible());
+    // Don't allow the selection to become empty
     this.treeView.onDidChangeSelection(async ev => {
       if (ev.selection.length == 0) {
-        const current = this.treeDataProvider.getCurrent();
+        this.updateTreeViewSelectionIfVisible();
-        if (current != undefined)
-          this.treeView.reveal(current); // don't allow selection to become empty
       }
     });
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.openQuery', this.handleOpenQuery));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.removeHistoryItem', this.handleRemoveHistoryItem.bind(this)));
+    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.setLabel', this.handleSetLabel.bind(this)));
+    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryLog', this.handleShowQueryLog.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.itemClicked', async (item) => {
       return this.handleItemClicked(item);
     }));
+    queryHistoryConfigListener.onDidChangeQueryHistoryConfiguration(() => {
+      this.treeDataProvider.refresh();
+    });
   }
 
-  push(item: QueryHistoryItem) {
+  addQuery(info: QueryWithResults): CompletedQuery {
+    const item = new CompletedQuery(info, this.queryHistoryConfigListener);
     this.treeDataProvider.push(item);
-    this.treeView.reveal(item, { select: true });
+    this.updateTreeViewSelectionIfVisible();
+    return item;
+  }
+
+  /**
+   * Update the tree view selection if the tree view is visible.
+   *
+   * If the tree view is not visible, we must wait until it becomes visible before updating the
+   * selection. This is because the only mechanism for updating the selection of the tree view
+   * has the side-effect of revealing the tree view. This changes the active sidebar to CodeQL,
+   * interrupting user workflows such as writing a commit message on the source control sidebar.
+   */
+  private updateTreeViewSelectionIfVisible() {
+    if (this.treeView.visible) {
+      const current = this.treeDataProvider.getCurrent();
+      if (current != undefined) {
+        // We must fire the onDidChangeTreeData event to ensure the current element can be selected
+        // using `reveal` if the tree view was not visible when the current element was added.
+        this.treeDataProvider.refresh();
+        this.treeView.reveal(current);
+      }
+    }
   }
 }

extensions/ql-vscode/src/query-results.ts

@@ -0,0 +1,152 @@
+import { QueryWithResults, tmpDir, QueryInfo } from "./run-queries";
+import * as messages from './messages';
+import * as helpers from './helpers';
+import * as cli from './cli';
+import * as sarif from 'sarif';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState } from "./interface-types";
+import { QueryHistoryConfig } from "./config";
+import { QueryHistoryItemOptions } from "./query-history";
+
+export class CompletedQuery implements QueryWithResults {
+  readonly time: string;
+  readonly query: QueryInfo;
+  readonly result: messages.EvaluationResult;
+  readonly database: DatabaseInfo;
+  readonly logFileLocation?: string
+  options: QueryHistoryItemOptions;
+  dispose: () => void;
+
+  /**
+   * Map from result set name to SortedResultSetInfo.
+   */
+  sortedResultsInfo: Map<string, SortedResultSetInfo>;
+
+  /**
+   * How we're currently sorting alerts. This is not mere interface
+   * state due to truncation; on re-sort, we want to read in the file
+   * again, sort it, and only ship off a reasonable number of results
+   * to the webview. Undefined means to use whatever order is in the
+   * sarif file.
+   */
+  interpretedResultsSortState: InterpretedResultsSortState | undefined;
+
+  constructor(
+    evaluation: QueryWithResults,
+    public config: QueryHistoryConfig,
+  ) {
+    this.query = evaluation.query;
+    this.result = evaluation.result;
+    this.database = evaluation.database;
+    this.logFileLocation = evaluation.logFileLocation;
+    this.options = evaluation.options;
+    this.dispose = evaluation.dispose;
+
+    this.time = new Date().toLocaleString();
+    this.sortedResultsInfo = new Map();
+  }
+
+  get databaseName(): string {
+    return this.database.name;
+  }
+  get queryName(): string {
+    return helpers.getQueryName(this.query);
+  }
+
+  /**
+   * Holds if this query should produce interpreted results.
+   */
+  canInterpretedResults(): Promise<boolean> {
+    return this.query.dbItem.hasMetadataFile();
+  }
+
+  get statusString(): string {
+    switch (this.result.resultType) {
+      case messages.QueryResultType.CANCELLATION:
+        return `cancelled after ${this.result.evaluationTime / 1000} seconds`;
+      case messages.QueryResultType.OOM:
+        return `out of memory`;
+      case messages.QueryResultType.SUCCESS:
+        return `finished in ${this.result.evaluationTime / 1000} seconds`;
+      case messages.QueryResultType.TIMEOUT:
+        return `timed out after ${this.result.evaluationTime / 1000} seconds`;
+      case messages.QueryResultType.OTHER_ERROR:
+      default:
+        return this.result.message ? `failed: ${this.result.message}` : 'failed';
+    }
+  }
+
+
+  interpolate(template: string): string {
+    const { databaseName, queryName, time, statusString } = this;
+    const replacements: { [k: string]: string } = {
+      t: time,
+      q: queryName,
+      d: databaseName,
+      s: statusString,
+      '%': '%',
+    };
+    return template.replace(/%(.)/g, (match, key) => {
+      const replacement = replacements[key];
+      return replacement !== undefined ? replacement : match;
+    });
+  }
+
+  getLabel(): string {
+    if (this.options.label !== undefined)
+      return this.options.label;
+    return this.config.format;
+  }
+
+  get didRunSuccessfully(): boolean {
+    return this.result.resultType === messages.QueryResultType.SUCCESS;
+  }
+
+  toString(): string {
+    return this.interpolate(this.getLabel());
+  }
+
+  async updateSortState(server: cli.CodeQLCliServer, resultSetName: string, sortState: RawResultsSortState | undefined): Promise<void> {
+    if (sortState === undefined) {
+      this.sortedResultsInfo.delete(resultSetName);
+      return;
+    }
+
+    const sortedResultSetInfo: SortedResultSetInfo = {
+      resultsPath: path.join(tmpDir.name, `sortedResults${this.query.queryID}-${resultSetName}.bqrs`),
+      sortState
+    };
+
+    await server.sortBqrs(this.query.resultsPaths.resultsPath, sortedResultSetInfo.resultsPath, resultSetName, [sortState.columnIndex], [sortState.sortDirection]);
+    this.sortedResultsInfo.set(resultSetName, sortedResultSetInfo);
+  }
+
+  async updateInterpretedSortState(_server: cli.CodeQLCliServer, sortState: InterpretedResultsSortState | undefined): Promise<void> {
+    this.interpretedResultsSortState = sortState;
+  }
+}
+
+/**
+ * Call cli command to interpret results.
+ */
+export async function interpretResults(server: cli.CodeQLCliServer, metadata: QueryMetadata | undefined, resultsPath: string, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
+  const interpretedResultsPath = resultsPath + ".interpreted.sarif"
+
+  if (await fs.pathExists(interpretedResultsPath)) {
+    return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
+  }
+  if (metadata === undefined) {
+    throw new Error('Can\'t interpret results without query metadata');
+  }
+  let { kind, id } = metadata;
+  if (kind === undefined) {
+    throw new Error('Can\'t interpret results without query metadata including kind');
+  }
+  if (id === undefined) {
+    // Interpretation per se doesn't really require an id, but the
+    // SARIF format does, so in the absence of one, we use a dummy id.
+    id = "dummy-id";
+  }
+  return await server.interpretBqrs({ kind, id }, resultsPath, interpretedResultsPath, sourceInfo);
+}

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,14 +1,18 @@
 import * as cp from 'child_process';
-import { DisposableObject } from 'semmle-vscode-utils';
+import * as path from 'path';
+// Import from the specific module within `semmle-vscode-utils`, rather than via `index.ts`, because
+// we avoid taking an accidental runtime dependency on `vscode` this way.
+import { DisposableObject } from 'semmle-vscode-utils/out/disposable-object';
 import { Disposable } from 'vscode';
 import { CancellationToken, createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
 import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
 import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './messages';
+import * as messages from './messages';
 
 type ServerOpts = {
-  logger: Logger
+  logger: Logger;
 }
 
 /** A running query server process and its associated message connection. */
@@ -23,7 +27,7 @@ class ServerProcess implements Disposable {
     this.logger = logger;
   }
 
-  dispose() {
+  dispose(): void {
     this.logger.log('Stopping query server...');
     this.connection.dispose();
     this.child.stdin!.end();
@@ -51,12 +55,16 @@ export class QueryServerClient extends DisposableObject {
   nextCallback: number;
   nextProgress: number;
   withProgressReporting: WithProgressReporting;
+  public activeQueryName: string | undefined;
 
   constructor(readonly config: QueryServerConfig, readonly cliServer: cli.CodeQLCliServer, readonly opts: ServerOpts, withProgressReporting: WithProgressReporting) {
     super();
     // When the query server configuration changes, restart the query server.
     if (config.onDidChangeQueryServerConfiguration !== undefined) {
-      this.push(config.onDidChangeQueryServerConfiguration(async () => await this.restartQueryServer(), this));
+      this.push(config.onDidChangeQueryServerConfiguration(async () => {
+        this.logger.log('Restarting query server due to configuration changes...');
+        await this.restartQueryServer();
+      }, this));
     }
     this.withProgressReporting = withProgressReporting;
     this.nextCallback = 0;
@@ -65,10 +73,12 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
-  get logger() { return this.opts.logger; }
+  get logger(): Logger {
+    return this.opts.logger;
+  }
 
   /** Stops the query server by disposing of the current server process. */
-  private stopQueryServer() {
+  private stopQueryServer(): void {
     if (this.serverProcess !== undefined) {
       this.disposeAndStopTracking(this.serverProcess);
     } else {
@@ -77,20 +87,23 @@ export class QueryServerClient extends DisposableObject {
   }
 
   /** Restarts the query server by disposing of the current server process and then starting a new one. */
-  private async restartQueryServer() {
+  async restartQueryServer(): Promise<void> {
-    this.logger.log('Restarting query server due to configuration changes...');
     this.stopQueryServer();
     await this.startQueryServer();
   }
 
+  showLog(): void {
+    this.logger.show();
+  }
+
   /** Starts a new query server process, sending progress messages to the status bar. */
-  async startQueryServer() {
+  async startQueryServer(): Promise<void> {
     // Use an arrow function to preserve the value of `this`.
     return this.withProgressReporting((progress, _) => this.startQueryServerImpl(progress));
   }
 
   /** Starts a new query server process, sending progress messages to the given reporter. */
-  private async startQueryServerImpl(progressReporter: ProgressReporter) {
+  private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
     if (this.config.debug) {
@@ -102,7 +115,10 @@ export class QueryServerClient extends DisposableObject {
       ['execute', 'query-server'],
       args,
       this.logger,
-      data => this.logger.logWithoutTrailingNewline(data.toString()),
+      data => this.logger.log(data.toString(), {
+        trailingNewline: false,
+        additionalLogLocation: this.activeQueryName
+      }),
       undefined, // no listener for stdout
       progressReporter
     );
@@ -113,12 +129,16 @@ export class QueryServerClient extends DisposableObject {
         this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
       }
       else {
+        const baseLocation = this.logger.getBaseLocation();
+        if (baseLocation && this.activeQueryName) {
+          res.logFileLocation = path.join(baseLocation, this.activeQueryName);
+        }
         this.evaluationResultCallbacks[res.runId](res);
       }
       return {};
     })
     connection.onNotification(progress, res => {
-      let callback = this.progressCallbacks[res.id];
+      const callback = this.progressCallbacks[res.id];
       if (callback) {
         callback(res);
       }
@@ -140,7 +160,7 @@ export class QueryServerClient extends DisposableObject {
     return id;
   }
 
-  unRegisterCallback(id: number) {
+  unRegisterCallback(id: number): void {
     delete this.evaluationResultCallbacks[id];
   }
 
@@ -149,8 +169,10 @@ export class QueryServerClient extends DisposableObject {
   }
 
   async sendRequest<P, R, E, RO>(type: RequestType<WithProgressId<P>, R, E, RO>, parameter: P, token?: CancellationToken, progress?: (res: ProgressMessage) => void): Promise<R> {
-    let id = this.nextProgress++;
+    const id = this.nextProgress++;
     this.progressCallbacks[id] = progress;
+
+    this.updateActiveQuery(type.method, parameter);
     try {
       if (this.serverProcess === undefined) {
         throw new Error('No query server process found.');
@@ -160,4 +182,19 @@ export class QueryServerClient extends DisposableObject {
       delete this.progressCallbacks[id];
     }
   }
+
+  /**
+   * Updates the active query every time there is a new request to compile.
+   * The active query is used to specify the side log.
+   *
+   * This isn't ideal because in situations where there are queries running
+   * in parallel, each query's log messages are interleaved. Fixing this
+   * properly will require a change in the query server.
+   */
+  private updateActiveQuery(method: string, parameter: any): void {
+    if (method === messages.compileQuery.method) {
+      const queryPath = parameter?.queryToCheck?.queryPath || 'unknown';
+      this.activeQueryName = `query-${path.basename(queryPath)}-${this.nextProgress}.log`;
+    }
+  }
 }

extensions/ql-vscode/src/quick-query.ts

@@ -0,0 +1,167 @@
+import * as fs from 'fs-extra';
+import * as glob from 'glob-promise';
+import * as yaml from 'js-yaml';
+import * as path from 'path';
+import { ExtensionContext, window as Window, workspace, Uri } from 'vscode';
+import { ErrorCodes, ResponseError } from 'vscode-languageclient';
+import { CodeQLCliServer } from './cli';
+import { DatabaseUI } from './databases-ui';
+import * as helpers from './helpers';
+import { logger } from './logging';
+import { UserCancellationException } from './run-queries';
+
+const QUICK_QUERIES_DIR_NAME = 'quick-queries';
+const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
+const QUICK_QUERY_WORKSPACE_FOLDER_NAME = 'Quick Queries';
+
+export function isQuickQueryPath(queryPath: string): boolean {
+  return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
+}
+
+async function getQlPackFor(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
+  const qlpacks = await cliServer.resolveQlpacks(helpers.getOnDiskWorkspaceFolders());
+  const packs: { packDir: string | undefined; packName: string }[] =
+    Object.entries(qlpacks).map(([packName, dirs]) => {
+      if (dirs.length < 1) {
+        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
+        return { packName, packDir: undefined };
+      }
+      if (dirs.length > 1) {
+        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
+      }
+      return {
+        packName,
+        packDir: dirs[0]
+      }
+    });
+  for (const { packDir, packName } of packs) {
+    if (packDir !== undefined) {
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8'));
+      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
+        return packName;
+      }
+    }
+  }
+  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+}
+
+/**
+ * `getBaseText` heuristically returns an appropriate import statement
+ * prelude based on the filename of the dbscheme file given. TODO: add
+ * a 'default import' field to the qlpack itself, and use that.
+ */
+function getBaseText(dbschemeBase: string) {
+  if (dbschemeBase == 'semmlecode.javascript.dbscheme') return 'import javascript\n\nselect ""';
+  if (dbschemeBase == 'semmlecode.cpp.dbscheme') return 'import cpp\n\nselect ""';
+  if (dbschemeBase == 'semmlecode.dbscheme') return 'import java\n\nselect ""';
+  if (dbschemeBase == 'semmlecode.python.dbscheme') return 'import python\n\nselect ""';
+  if (dbschemeBase == 'semmlecode.csharp.dbscheme') return 'import csharp\n\nselect ""';
+  if (dbschemeBase == 'go.dbscheme') return 'import go\n\nselect ""';
+  return 'select ""';
+}
+
+function getQuickQueriesDir(ctx: ExtensionContext): string {
+  const storagePath = ctx.storagePath;
+  if (storagePath === undefined) {
+    throw new Error('Workspace storage path is undefined');
+  }
+  const queriesPath = path.join(storagePath, QUICK_QUERIES_DIR_NAME);
+  fs.ensureDir(queriesPath, { mode: 0o700 });
+  return queriesPath;
+}
+
+/**
+ * Show a buffer the user can enter a simple query into.
+ */
+export async function displayQuickQuery(ctx: ExtensionContext, cliServer: CodeQLCliServer, databaseUI: DatabaseUI) {
+  try {
+
+    const workspaceFolders = workspace.workspaceFolders || [];
+    const queriesDir = await getQuickQueriesDir(ctx);
+
+    function updateQuickQueryDir(index: number, len: number) {
+      workspace.updateWorkspaceFolders(
+        index,
+        len,
+        { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
+      );
+    }
+
+    // If there is already a quick query open, don't clobber it, just
+    // show it.
+    const existing = workspace.textDocuments.find(doc => path.basename(doc.uri.fsPath) === QUICK_QUERY_QUERY_NAME);
+    if (existing !== undefined) {
+      Window.showTextDocument(existing);
+      return;
+    }
+
+    // We need to have a multi-root workspace to make quick query work
+    // at all. Changing the workspace from single-root to multi-root
+    // causes a restart of the whole extension host environment, so we
+    // basically can't do anything that survives that restart.
+    //
+    // So if we are currently in a single-root workspace (of which the
+    // only reliable signal appears to be `workspace.workspaceFile`
+    // being undefined) just let the user know that they're in for a
+    // restart.
+    if (workspace.workspaceFile === undefined) {
+      const makeMultiRoot = await helpers.showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
+      if (makeMultiRoot) {
+        updateQuickQueryDir(workspaceFolders.length, 0);
+      }
+      return;
+    }
+
+    const index = workspaceFolders.findIndex(folder => folder.name === QUICK_QUERY_WORKSPACE_FOLDER_NAME)
+    if (index === -1)
+      updateQuickQueryDir(workspaceFolders.length, 0);
+    else
+      updateQuickQueryDir(index, 1);
+
+    // We're going to infer which qlpack to use from the current database
+    const dbItem = await databaseUI.getDatabaseItem();
+    if (dbItem === undefined) {
+      throw new Error('Can\'t start quick query without a selected database');
+    }
+
+    const datasetFolder = await dbItem.getDatasetFolder(cliServer);
+    const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'))
+
+    if (dbschemes.length < 1) {
+      throw new Error(`Can't find dbscheme for current database in ${datasetFolder}`);
+    }
+
+    dbschemes.sort();
+    const dbscheme = dbschemes[0];
+    if (dbschemes.length > 1) {
+      Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
+    }
+
+    const qlpack = await getQlPackFor(cliServer, dbscheme);
+    const quickQueryQlpackYaml: any = {
+      name: "quick-query",
+      version: "1.0.0",
+      libraryPathDependencies: [qlpack]
+    };
+
+    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const qlPackFile = path.join(queriesDir, 'qlpack.yml');
+    await fs.writeFile(qlFile, getBaseText(path.basename(dbscheme)), 'utf8');
+    await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8');
+    Window.showTextDocument(await workspace.openTextDocument(qlFile));
+  }
+
+  // TODO: clean up error handling for top-level commands like this
+  catch (e) {
+    if (e instanceof UserCancellationException) {
+      logger.log(e.message);
+    }
+    else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      logger.log(e.message);
+    }
+    else if (e instanceof Error)
+      helpers.showAndLogErrorMessage(e.message);
+    else
+      throw e;
+  }
+}

extensions/ql-vscode/src/result-keys.ts

@@ -0,0 +1,95 @@
+import * as sarif from 'sarif';
+
+/**
+ * Identifies one of the results in a result set by its index in the result list.
+ */
+export interface Result {
+  resultIndex: number;
+}
+
+/**
+ * Identifies one of the paths associated with a result.
+ */
+export interface Path extends Result {
+  pathIndex: number;
+}
+
+/**
+ * Identifies one of the nodes in a path.
+ */
+export interface PathNode extends Path {
+  pathNodeIndex: number;
+}
+
+/** Alias for `undefined` but more readable in some cases */
+export const none: PathNode | undefined = undefined;
+
+/**
+ * Looks up a specific result in a result set.
+ */
+export function getResult(sarif: sarif.Log, key: Result): sarif.Result | undefined {
+  if (sarif.runs.length === 0) return undefined;
+  if (sarif.runs[0].results === undefined) return undefined;
+  const results = sarif.runs[0].results;
+  return results[key.resultIndex];
+}
+
+/**
+ * Looks up a specific path in a result set.
+ */
+export function getPath(sarif: sarif.Log, key: Path): sarif.ThreadFlow | undefined {
+  const result = getResult(sarif, key);
+  if (result === undefined) return undefined;
+  let index = -1;
+  if (result.codeFlows === undefined) return undefined;
+  for (const codeFlows of result.codeFlows) {
+    for (const threadFlow of codeFlows.threadFlows) {
+      ++index;
+      if (index == key.pathIndex)
+        return threadFlow;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Looks up a specific path node in a result set.
+ */
+export function getPathNode(sarif: sarif.Log, key: PathNode): sarif.Location | undefined {
+  const path = getPath(sarif, key);
+  if (path === undefined) return undefined;
+  return path.locations[key.pathNodeIndex];
+}
+
+/**
+ * Returns true if the two keys are both `undefined` or contain the same set of indices.
+ */
+export function equals(key1: PathNode | undefined, key2: PathNode | undefined): boolean {
+  if (key1 === key2) return true;
+  if (key1 === undefined || key2 === undefined) return false;
+  return key1.resultIndex === key2.resultIndex && key1.pathIndex === key2.pathIndex && key1.pathNodeIndex === key2.pathNodeIndex;
+}
+
+/**
+ * Returns true if the two keys contain the same set of indices and neither are `undefined`.
+ */
+export function equalsNotUndefined(key1: PathNode | undefined, key2: PathNode | undefined): boolean {
+  if (key1 === undefined || key2 === undefined) return false;
+  return key1.resultIndex === key2.resultIndex && key1.pathIndex === key2.pathIndex && key1.pathNodeIndex === key2.pathNodeIndex;
+}
+
+/**
+ * Returns the list of paths in the given SARIF result.
+ *
+ * Path nodes indices are relative to this flattened list.
+ */
+export function getAllPaths(result: sarif.Result): sarif.ThreadFlow[] {
+  if (result.codeFlows === undefined) return [];
+  const paths = [];
+  for (const codeFlow of result.codeFlows) {
+    for (const threadFlow of codeFlow.threadFlows) {
+      paths.push(threadFlow);
+    }
+  }
+  return paths;
+}

extensions/ql-vscode/src/run-queries.ts

@@ -1,20 +1,24 @@
 import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import * as sarif from 'sarif';
 import * as tmp from 'tmp';
+import { promisify } from 'util';
 import * as vscode from 'vscode';
+import { ErrorCodes, ResponseError } from 'vscode-languageclient';
+
 import * as cli from './cli';
 import { DatabaseItem, getUpgradesDirectories } from './databases';
 import * as helpers from './helpers';
-import { DatabaseInfo, SortState, ResultsInfo, SortedResultSetInfo } from './interface-types';
+import { DatabaseInfo, QueryMetadata, ResultsPaths } from './interface-types';
 import { logger } from './logging';
 import * as messages from './messages';
+import { QueryHistoryItemOptions } from './query-history';
 import * as qsClient from './queryserver-client';
-import { promisify } from 'util';
+import { isQuickQueryPath } from './quick-query';
+import { upgradeDatabase } from './upgrades';
 
 /**
- * queries.ts
+ * run-queries.ts
  * -------------
  *
  * Compiling and running QL queries.
@@ -22,7 +26,7 @@ import { promisify } from 'util';
 
 // XXX: Tmp directory should be configuarble.
 export const tmpDir = tmp.dirSync({ prefix: 'queries_', keep: false, unsafeCleanup: true });
-const upgradesTmpDir = tmp.dirSync({ dir: tmpDir.name, prefix: 'upgrades_', keep: false, unsafeCleanup: true });
+export const upgradesTmpDir = tmp.dirSync({ dir: tmpDir.name, prefix: 'upgrades_', keep: false, unsafeCleanup: true });
 export const tmpDirDisposal = {
   dispose: () => {
     upgradesTmpDir.removeCallback();
@@ -30,8 +34,6 @@ export const tmpDirDisposal = {
   }
 };
 
-let queryCounter = 0;
-
 export class UserCancellationException extends Error { }
 
 /**
@@ -41,32 +43,31 @@ export class UserCancellationException extends Error { }
  * output and results.
  */
 export class QueryInfo {
-  compiledQueryPath: string;
+  private static nextQueryId = 0;
-  resultsInfo: ResultsInfo;
+
-  /**
+  readonly compiledQueryPath: string;
-   * Map from result set name to SortedResultSetInfo.
+  readonly resultsPaths: ResultsPaths;
-   */
+  readonly dataset: vscode.Uri; // guarantee the existence of a well-defined dataset dir at this point
-  sortedResultsInfo: Map<string, SortedResultSetInfo>;
+  readonly queryID: number;
-  dataset: vscode.Uri; // guarantee the existence of a well-defined dataset dir at this point
 
   constructor(
-    public program: messages.QlProgram,
+    public readonly program: messages.QlProgram,
-    public dbItem: DatabaseItem,
+    public readonly dbItem: DatabaseItem,
-    public queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
+    public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
-    public quickEvalPosition?: messages.Position,
+    public readonly quickEvalPosition?: messages.Position,
-    public metadata?: cli.QueryMetadata,
+    public readonly metadata?: QueryMetadata,
+    public readonly templates?: messages.TemplateDefinitions,
   ) {
-    this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${queryCounter}.qlo`);
+    this.queryID = QueryInfo.nextQueryId++;
-    this.resultsInfo = {
+    this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
-      resultsPath: path.join(tmpDir.name, `results${queryCounter}.bqrs`),
+    this.resultsPaths = {
-      interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${queryCounter}.sarif`)
+      resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`),
+      interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`),
     };
-    this.sortedResultsInfo = new Map();
     if (dbItem.contents === undefined) {
       throw new Error('Can\'t run query on invalid database.');
     }
     this.dataset = dbItem.contents.datasetUri;
-    queryCounter++;
   }
 
   async run(
@@ -77,9 +78,10 @@ export class QueryInfo {
     const callbackId = qs.registerCallback(res => { result = res });
 
     const queryToRun: messages.QueryToRun = {
-      resultsPath: this.resultsInfo.resultsPath,
+      resultsPath: this.resultsPaths.resultsPath,
       qlo: vscode.Uri.file(this.compiledQueryPath).toString(),
       allowUnknownTemplates: true,
+      templateValues: this.templates,
       id: callbackId,
       timeoutSecs: qs.config.timeoutSecs,
     }
@@ -105,13 +107,19 @@ export class QueryInfo {
     } finally {
       qs.unRegisterCallback(callbackId);
     }
-    return result || { evaluationTime: 0, message: "No result from server", queryId: -1, runId: callbackId, resultType: messages.QueryResultType.OTHER_ERROR };
+    return result || {
+      evaluationTime: 0,
+      message: "No result from server",
+      queryId: -1,
+      runId: callbackId,
+      resultType: messages.QueryResultType.OTHER_ERROR
+    };
   }
 
   async compile(
     qs: qsClient.QueryServerClient,
   ): Promise<messages.CompilationMessage[]> {
-    let compiled: messages.CheckQueryResult;
+    let compiled: messages.CheckQueryResult | undefined;
     try {
       const params: messages.CompileQueryParams = {
         compilationOptions: {
@@ -128,10 +136,13 @@ export class QueryInfo {
         },
         queryToCheck: this.program,
         resultPath: this.compiledQueryPath,
-        target: !!this.quickEvalPosition ? { quickEval: { quickEvalPos: this.quickEvalPosition } } : { query: {} }
+        target: this.quickEvalPosition ? {
+          quickEval: { quickEvalPos: this.quickEvalPosition }
+        } : {
+          query: {}
+        }
       };
 
-
       compiled = await helpers.withProgress({
         location: vscode.ProgressLocation.Notification,
         title: "Compiling Query",
@@ -142,233 +153,33 @@ export class QueryInfo {
     } finally {
       qs.logger.log(" - - - COMPILATION DONE - - - ");
     }
-
+    return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
-    return (compiled.messages || []).filter(msg => msg.severity == 0);
   }
 
   /**
    * Holds if this query should produce interpreted results.
    */
-  hasInterpretedResults(): boolean {
+  async hasInterpretedResults(): Promise<boolean> {
-    return this.dbItem.hasDbInfo();
+    const hasMetadataFile = await this.dbItem.hasMetadataFile();
-  }
+    if (!hasMetadataFile) {
-
+      logger.log("Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.");
-  async updateSortState(server: cli.CodeQLCliServer, resultSetName: string, sortState: SortState | undefined): Promise<void> {
-    if (sortState === undefined) {
-      this.sortedResultsInfo.delete(resultSetName);
-      return;
     }
-
+    return hasMetadataFile;
-    const sortedResultSetInfo: SortedResultSetInfo = {
-      resultsPath: path.join(tmpDir.name, `sortedResults${queryCounter}-${resultSetName}.bqrs`),
-      sortState
-    };
-
-    await server.sortBqrs(this.resultsInfo.resultsPath, sortedResultSetInfo.resultsPath, resultSetName, [sortState.columnIndex], [sortState.direction]);
-    this.sortedResultsInfo.set(resultSetName, sortedResultSetInfo);
   }
 }
 
-/**
+export interface QueryWithResults {
- * Call cli command to interpret results.
+  readonly query: QueryInfo;
- */
+  readonly result: messages.EvaluationResult;
-export async function interpretResults(server: cli.CodeQLCliServer, queryInfo: QueryInfo, resultsInfo: ResultsInfo, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
+  readonly database: DatabaseInfo;
-  if (await fs.pathExists(resultsInfo.interpretedResultsPath)) {
+  readonly options: QueryHistoryItemOptions;
-    return JSON.parse(await fs.readFile(resultsInfo.interpretedResultsPath, 'utf8'));
+  readonly logFileLocation?: string;
-  }
+  readonly dispose: () => void;
-  const { metadata } = queryInfo;
-  if (metadata == undefined) {
-    throw new Error('Can\'t interpret results without query metadata');
-  }
-  let { kind, id } = metadata;
-  if (kind == undefined) {
-    throw new Error('Can\'t interpret results without query metadata including kind');
-  }
-  if (id == undefined) {
-    // Interpretation per se doesn't really require an id, but the
-    // SARIF format does, so in the absence of one, we invent one
-    // based on the query path.
-    //
-    // Just to be careful, sanitize to remove '/' since SARIF (section
-    // 3.27.5 "ruleId property") says that it has special meaning.
-    id = queryInfo.program.queryPath.replace(/\//g, '-');
-  }
-  return await server.interpretBqrs({ kind, id }, resultsInfo.resultsPath, resultsInfo.interpretedResultsPath, sourceInfo);
 }
 
-export interface EvaluationInfo {
+export async function clearCacheInDatabase(
-  query: QueryInfo;
+  qs: qsClient.QueryServerClient, dbItem: DatabaseItem
-  result: messages.EvaluationResult;
+): Promise<messages.ClearCacheResult> {
-  database: DatabaseInfo;
-}
-
-/**
- * Checks whether the given database can be upgraded to the given target DB scheme,
- * and whether the user wants to proceed with the upgrade.
- * Reports errors to both the user and the console.
- * @returns the `UpgradeParams` needed to start the upgrade, if the upgrade is possible and was confirmed by the user, or `undefined` otherwise.
- */
-async function checkAndConfirmDatabaseUpgrade(qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]):
-  Promise<messages.UpgradeParams | undefined> {
-  if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
-    helpers.showAndLogErrorMessage("Database is invalid, and cannot be upgraded.");
-    return;
-  }
-  const params: messages.UpgradeParams = {
-    fromDbscheme: db.contents.dbSchemeUri.fsPath,
-    toDbscheme: targetDbScheme.fsPath,
-    additionalUpgrades: upgradesDirectories.map(uri => uri.fsPath)
-  };
-
-  let checkUpgradeResult: messages.CheckUpgradeResult;
-  try {
-    qs.logger.log('Checking database upgrade...');
-    checkUpgradeResult = await checkDatabaseUpgrade(qs, params);
-  }
-  catch (e) {
-    helpers.showAndLogErrorMessage(`Database cannot be upgraded: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done checking database upgrade.');
-  }
-
-  const checkedUpgrades = checkUpgradeResult.checkedUpgrades;
-  if (checkedUpgrades === undefined) {
-    const error = checkUpgradeResult.upgradeError || '[no error message available]';
-    await helpers.showAndLogErrorMessage(`Database cannot be upgraded: ${error}`);
-    return;
-  }
-
-  if (checkedUpgrades.scripts.length === 0) {
-    await helpers.showAndLogInformationMessage('Database is already up to date; nothing to do.');
-    return;
-  }
-
-  let curSha = checkedUpgrades.initialSha;
-  let descriptionMessage = '';
-  for (const script of checkedUpgrades.scripts) {
-    descriptionMessage += `Would perform upgrade: ${script.description}\n`;
-    descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
-    curSha = script.newSha;
-  }
-
-  const targetSha = checkedUpgrades.targetSha;
-  if (curSha != targetSha) {
-    // Newlines aren't rendered in notifications: https://github.com/microsoft/vscode/issues/48900
-    // A modal dialog would be rendered better, but is more intrusive.
-    await helpers.showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
-    Can upgrade from ${checkedUpgrades.initialSha} (current) to ${curSha}, but cannot reach ${targetSha} (target).`);
-    // TODO: give a more informative message if we think the DB is ahead of the target DB scheme
-    return;
-  }
-
-  logger.log(descriptionMessage);
-  // Ask the user to confirm the upgrade.
-  const shouldUpgrade = await helpers.showBinaryChoiceDialog(`Should the database ${db.databaseUri.fsPath} be upgraded?\n\n${descriptionMessage}`);
-  if (shouldUpgrade) {
-    return params;
-  }
-  else {
-    throw new UserCancellationException('User cancelled the database upgrade.');
-  }
-}
-
-/**
- * Command handler for 'Upgrade Database'.
- * Attempts to upgrade the given database to the given target DB scheme, using the given directory of upgrades.
- * First performs a dry-run and prompts the user to confirm the upgrade.
- * Reports errors during compilation and evaluation of upgrades to the user.
- */
-export async function upgradeDatabase(qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]):
-  Promise<messages.RunUpgradeResult | undefined> {
-  const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories);
-
-  if (upgradeParams === undefined) {
-    return;
-  }
-
-  let compileUpgradeResult: messages.CompileUpgradeResult;
-  try {
-    compileUpgradeResult = await compileDatabaseUpgrade(qs, upgradeParams);
-  }
-  catch (e) {
-    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done compiling database upgrade.')
-  }
-
-  if (compileUpgradeResult.compiledUpgrades === undefined) {
-    const error = compileUpgradeResult.error || '[no error message available]';
-    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
-    return;
-  }
-
-  try {
-    qs.logger.log('Running the following database upgrade:');
-    qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
-    return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades);
-  }
-  catch (e) {
-    helpers.showAndLogErrorMessage(`Database upgrade failed: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done running database upgrade.')
-  }
-}
-
-async function checkDatabaseUpgrade(qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams):
-  Promise<messages.CheckUpgradeResult> {
-  return helpers.withProgress({
-    location: vscode.ProgressLocation.Notification,
-    title: "Checking for database upgrades",
-    cancellable: true,
-  }, (progress, token) => qs.sendRequest(messages.checkUpgrade, upgradeParams, token, progress));
-}
-
-async function compileDatabaseUpgrade(qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams):
-  Promise<messages.CompileUpgradeResult> {
-  const params: messages.CompileUpgradeParams = {
-    upgrade: upgradeParams,
-    upgradeTempDir: upgradesTmpDir.name
-  }
-
-  return helpers.withProgress({
-    location: vscode.ProgressLocation.Notification,
-    title: "Compiling database upgrades",
-    cancellable: true,
-  }, (progress, token) => qs.sendRequest(messages.compileUpgrade, params, token, progress));
-}
-
-async function runDatabaseUpgrade(qs: qsClient.QueryServerClient, db: DatabaseItem, upgrades: messages.CompiledUpgrades):
-  Promise<messages.RunUpgradeResult> {
-
-  if (db.contents === undefined || db.contents.datasetUri === undefined) {
-    throw new Error('Can\'t upgrade an invalid database.');
-  }
-  const database: messages.Dataset = {
-    dbDir: db.contents.datasetUri.fsPath,
-    workingSet: 'default'
-  };
-
-  const params: messages.RunUpgradeParams = {
-    db: database,
-    timeoutSecs: qs.config.timeoutSecs,
-    toRun: upgrades
-  };
-
-  return helpers.withProgress({
-    location: vscode.ProgressLocation.Notification,
-    title: "Running database upgrades",
-    cancellable: true,
-  }, (progress, token) => qs.sendRequest(messages.runUpgrade, params, token, progress));
-}
-
-export async function clearCacheInDatabase(qs: qsClient.QueryServerClient, dbItem: DatabaseItem):
-  Promise<messages.ClearCacheResult> {
   if (dbItem.contents === undefined) {
     throw new Error('Can\'t clear the cache in an invalid database.');
   }
@@ -388,7 +199,7 @@ export async function clearCacheInDatabase(qs: qsClient.QueryServerClient, dbIte
     title: "Clearing Cache",
     cancellable: false,
   }, (progress, token) =>
-      qs.sendRequest(messages.clearCache, params, token, progress)
+    qs.sendRequest(messages.clearCache, params, token, progress)
   );
 }
 
@@ -448,7 +259,7 @@ async function checkDbschemeCompatibility(
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
     const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
-    async function hash(filename: string): Promise<string> {
+    const hash = async function (filename: string): Promise<string> {
       return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
     }
 
@@ -484,7 +295,7 @@ async function checkDbschemeCompatibility(
 }
 
 /** Prompts the user to save `document` if it has unsaved changes. */
-async function promptUserToSaveChanges(document: vscode.TextDocument) {
+async function promptUserToSaveChanges(document: vscode.TextDocument): Promise<void> {
   if (document.isDirty) {
     // TODO: add 'always save' button which records preference in configuration
     if (await helpers.showBinaryChoiceDialog('Query file has unsaved changes. Save now?')) {
@@ -494,8 +305,8 @@ async function promptUserToSaveChanges(document: vscode.TextDocument) {
 }
 
 type SelectedQuery = {
-  queryPath: string,
+  queryPath: string;
-  quickEvalPosition?: messages.Position
+  quickEvalPosition?: messages.Position;
 };
 
 /**
@@ -508,7 +319,7 @@ type SelectedQuery = {
  * @param selectedResourceUri The selected resource when the command was run.
  * @param quickEval Whether the command being run is `Quick Evaluation`.
 */
-async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefined, quickEval: boolean): Promise<SelectedQuery> {
+export async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefined, quickEval: boolean): Promise<SelectedQuery> {
   const editor = vscode.window.activeTextEditor;
 
   // Choose which QL file to use.
@@ -529,7 +340,18 @@ async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefine
   if (queryUri.scheme !== 'file') {
     throw new Error('Can only run queries that are on disk.');
   }
-  const queryPath = queryUri.fsPath;
+  const queryPath = queryUri.fsPath || '';
+
+  if (quickEval) {
+    if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) {
+      throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".');
+    }
+  }
+  else {
+    if (!(queryPath.endsWith('.ql'))) {
+      throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".');
+    }
+  }
 
   // Whether we chose the file from the active editor or from a context menu,
   // if the same file is open with unsaved changes in the active editor,
@@ -559,8 +381,9 @@ export async function compileAndRunQueryAgainstDatabase(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
   quickEval: boolean,
-  selectedQueryUri: vscode.Uri | undefined
+  selectedQueryUri: vscode.Uri | undefined,
-): Promise<EvaluationInfo> {
+  templates?: messages.TemplateDefinitions,
+): Promise<QueryWithResults> {
 
   if (!db.contents || !db.contents.dbSchemeUri) {
     throw new Error(`Database ${db.databaseUri} does not have a CodeQL database scheme.`);
@@ -569,6 +392,12 @@ export async function compileAndRunQueryAgainstDatabase(
   // Determine which query to run, based on the selection and the active editor.
   const { queryPath, quickEvalPosition } = await determineSelectedQuery(selectedQueryUri, quickEval);
 
+  // If this is quick query, store the query text
+  const historyItemOptions: QueryHistoryItemOptions = {};
+  if (isQuickQueryPath(queryPath)) {
+    historyItemOptions.queryText = await fs.readFile(queryPath, 'utf8');
+  }
+
   // Get the workspace folder paths.
   const diskWorkspaceFolders = helpers.getOnDiskWorkspaceFolders();
   // Figure out the library path for the query.
@@ -598,7 +427,7 @@ export async function compileAndRunQueryAgainstDatabase(
   };
 
   // Read the query metadata if possible, to use in the UI.
-  let metadata: cli.QueryMetadata | undefined;
+  let metadata: QueryMetadata | undefined;
   try {
     metadata = await cliServer.resolveMetadata(qlProgram.queryPath);
   } catch (e) {
@@ -606,20 +435,38 @@ export async function compileAndRunQueryAgainstDatabase(
     logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
   }
 
-  const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata);
+  const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
   await checkDbschemeCompatibility(cliServer, qs, query);
 
-  const errors = await query.compile(qs);
+  let errors;
-
+  try {
+    errors = await query.compile(qs);
+  } catch (e) {
+    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+    } else {
+      throw e;
+    }
+  }
 
   if (errors.length == 0) {
     const result = await query.run(qs);
+    if (result.resultType !== messages.QueryResultType.SUCCESS) {
+      const message = result.message || 'Failed to run query';
+      logger.log(message);
+      helpers.showAndLogErrorMessage(message);
+    }
     return {
       query,
       result,
       database: {
         name: db.name,
         databaseUri: db.databaseUri.toString(true)
+      },
+      options: historyItemOptions,
+      logFileLocation: result.logFileLocation,
+      dispose: () => {
+        qs.logger.removeAdditionalLogLocation(result.logFileLocation);
       }
     };
   } else {
@@ -629,7 +476,7 @@ export async function compileAndRunQueryAgainstDatabase(
     // However we don't show quick eval errors there so we need to display them anyway.
     qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
 
-    let formattedMessages: string[] = [];
+    const formattedMessages: string[] = [];
 
     for (const error of errors) {
       const message = error.message || "[no error message available]";
@@ -645,19 +492,33 @@ export async function compileAndRunQueryAgainstDatabase(
         " and the query and database use the same target language. For more details on the error, go to View > Output," +
         " and choose CodeQL Query Server from the dropdown.");
     }
-    return {
+
-      query,
+    return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
-      result: {
-        evaluationTime: 0,
-        resultType: messages.QueryResultType.OTHER_ERROR,
-        queryId: -1,
-        runId: -1,
-        message: "Query had compilation errors"
-      },
-      database: {
-        name: db.name,
-        databaseUri: db.databaseUri.toString(true)
-      }
-    };
   }
 }
+
+function createSyntheticResult(
+  query: QueryInfo,
+  db: DatabaseItem,
+  historyItemOptions: QueryHistoryItemOptions,
+  message: string,
+  resultType: number
+): QueryWithResults {
+
+  return {
+    query,
+    result: {
+      evaluationTime: 0,
+      resultType: resultType,
+      queryId: -1,
+      runId: -1,
+      message
+    },
+    database: {
+      name: db.name,
+      databaseUri: db.databaseUri.toString(true)
+    },
+    options: historyItemOptions,
+    dispose: () => { /**/ },
+  };
+}

extensions/ql-vscode/src/sarif-utils.ts

@@ -0,0 +1,124 @@
+import * as Sarif from "sarif"
+import * as path from "path"
+import { LocationStyle, ResolvableLocationValue } from "semmle-bqrs";
+
+export interface SarifLink {
+  dest: number;
+  text: string;
+}
+
+
+type ParsedSarifLocation =
+  | ResolvableLocationValue
+  // Resolvable locations have a `file` field, but it will sometimes include
+  // a source location prefix, which contains build-specific information the user
+  // doesn't really need to see. We ensure that `userVisibleFile` will not contain
+  // that, and is appropriate for display in the UI.
+  & { userVisibleFile: string }
+  | { t: 'NoLocation'; hint: string };
+
+export type SarifMessageComponent = string | SarifLink
+
+/**
+ * Unescape "[", "]" and "\\" like in sarif plain text messages
+ */
+export function unescapeSarifText(message: string): string {
+  return message.replace(/\\\[/g, "[").replace(/\\\]/g, "]").replace(/\\\\/, "\\");
+}
+
+export function parseSarifPlainTextMessage(message: string): SarifMessageComponent[] {
+  const results: SarifMessageComponent[] = [];
+
+  // We want something like "[linkText](4)", except that "[" and "]" may be escaped. The lookbehind asserts
+  // that the initial [ is not escaped. Then we parse a link text with "[" and "]" escaped. Then we parse the numerical target.
+  // Technically we could have any uri in the target but we don't output that yet.
+  // The possibility of escaping outside the link is not mentioned in the sarif spec but we always output sartif this way.
+  const linkRegex = /(?<=(?<!\\)(\\\\)*)\[(?<linkText>([^\\\]\[]|\\\\|\\\]|\\\[)*)\]\((?<linkTarget>[0-9]+)\)/g;
+  let result: RegExpExecArray | null;
+  let curIndex = 0;
+  while ((result = linkRegex.exec(message)) !== null) {
+    results.push(unescapeSarifText(message.substring(curIndex, result.index)));
+    const linkText = result.groups!["linkText"];
+    const linkTarget = +result.groups!["linkTarget"];
+    results.push({ dest: linkTarget, text: unescapeSarifText(linkText) });
+    curIndex = result.index + result[0].length;
+  }
+  results.push(unescapeSarifText(message.substring(curIndex, message.length)));
+  return results;
+}
+
+
+/**
+ * Computes a path normalized to reflect conventional normalization
+ * of windows paths into zip archive paths.
+ * @param sourceLocationPrefix The source location prefix of a database. May be
+ * unix style `/foo/bar/baz` or windows-style `C:\foo\bar\baz`.
+ * @param sarifRelativeUri A uri relative to sourceLocationPrefix.
+ * @returns A string that is valid for the `.file` field of a `FivePartLocation`:
+ * directory separators are normalized, but drive letters `C:` may appear.
+ */
+export function getPathRelativeToSourceLocationPrefix(sourceLocationPrefix: string, sarifRelativeUui: string) {
+  const normalizedSourceLocationPrefix = sourceLocationPrefix.replace(/\\/g, '/');
+  return path.join(normalizedSourceLocationPrefix, decodeURIComponent(sarifRelativeUui));
+}
+
+export function parseSarifLocation(loc: Sarif.Location, sourceLocationPrefix: string): ParsedSarifLocation {
+  const physicalLocation = loc.physicalLocation;
+  if (physicalLocation === undefined)
+    return { t: 'NoLocation', hint: 'no physical location' };
+  if (physicalLocation.artifactLocation === undefined)
+    return { t: 'NoLocation', hint: 'no artifact location' };
+  if (physicalLocation.artifactLocation.uri === undefined)
+    return { t: 'NoLocation', hint: 'artifact location has no uri' };
+
+  // This is not necessarily really an absolute uri; it could either be a
+  // file uri or a relative uri.
+  const uri = physicalLocation.artifactLocation.uri;
+
+  const fileUriRegex = /^file:/;
+  const effectiveLocation = uri.match(fileUriRegex) ?
+    decodeURIComponent(uri.replace(fileUriRegex, '')) :
+    getPathRelativeToSourceLocationPrefix(sourceLocationPrefix, uri);
+  const userVisibleFile = uri.match(fileUriRegex) ?
+    decodeURIComponent(uri.replace(fileUriRegex, '')) :
+    uri;
+
+  if (physicalLocation.region === undefined) {
+    // If the region property is absent, the physicalLocation object refers to the entire file.
+    // Source: https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Toc16012638.
+    // TODO: Do we get here if we provide a non-filesystem URL?
+    return {
+      t: LocationStyle.WholeFile,
+      file: effectiveLocation,
+      userVisibleFile,
+    };
+  } else {
+    const region = physicalLocation.region;
+    // We assume that the SARIF we're given always has startLine
+    // This is not mandated by the SARIF spec, but should be true of
+    // SARIF output by our own tools.
+    const lineStart = region.startLine!;
+
+    // These defaults are from SARIF 2.1.0 spec, section 3.30.2, "Text Regions"
+    // https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Ref493492556
+    const lineEnd = region.endLine === undefined ? lineStart : region.endLine;
+    const colStart = region.startColumn === undefined ? 1 : region.startColumn;
+
+    // We also assume that our tools will always supply `endColumn` field, which is
+    // fortunate, since the SARIF spec says that it defaults to the end of the line, whose
+    // length we don't know at this point in the code.
+    //
+    // It is off by one with respect to the way vscode counts columns in selections.
+    const colEnd = region.endColumn! - 1;
+
+    return {
+      t: LocationStyle.FivePart,
+      file: effectiveLocation,
+      userVisibleFile,
+      lineStart,
+      colStart,
+      lineEnd,
+      colEnd,
+    };
+  }
+}

extensions/ql-vscode/src/test-adapter.ts

@@ -0,0 +1,239 @@
+import * as path from 'path';
+import * as vscode from 'vscode';
+import {
+  TestAdapter,
+  TestLoadStartedEvent,
+  TestLoadFinishedEvent,
+  TestRunStartedEvent,
+  TestRunFinishedEvent,
+  TestSuiteEvent,
+  TestEvent,
+  TestSuiteInfo,
+  TestInfo,
+  TestHub
+} from 'vscode-test-adapter-api';
+import { TestAdapterRegistrar } from 'vscode-test-adapter-util';
+import { QLTestFile, QLTestNode, QLTestDirectory, QLTestDiscovery } from './qltest-discovery';
+import { Event, EventEmitter, CancellationTokenSource, CancellationToken } from 'vscode';
+import { DisposableObject } from 'semmle-vscode-utils';
+import { QLPackDiscovery } from './qlpack-discovery';
+import { CodeQLCliServer } from './cli';
+import { getOnDiskWorkspaceFolders } from './helpers';
+import { testLogger } from './logging';
+
+/**
+ * Get the full path of the `.expected` file for the specified QL test.
+ * @param testPath The full path to the test file.
+ */
+export function getExpectedFile(testPath: string): string {
+  return getTestOutputFile(testPath, '.expected');
+}
+
+/**
+ * Get the full path of the `.actual` file for the specified QL test.
+ * @param testPath The full path to the test file.
+ */
+export function getActualFile(testPath: string): string {
+  return getTestOutputFile(testPath, '.actual');
+}
+
+/**
+ * Get the directory containing the specified QL test.
+ * @param testPath The full path to the test file.
+ */
+export function getTestDirectory(testPath: string): string {
+  return path.dirname(testPath);
+}
+
+/**
+ * Gets the the full path to a particular output file of the specified QL test.
+ * @param testPath The full path to the QL test.
+ * @param extension The file extension of the output file.
+ */
+function getTestOutputFile(testPath: string, extension: string): string {
+  return changeExtension(testPath, extension);
+}
+
+/**
+ * A factory service that creates `QLTestAdapter` objects for workspace folders on demand.
+ */
+export class QLTestAdapterFactory extends DisposableObject {
+  constructor(testHub: TestHub, cliServer: CodeQLCliServer) {
+    super();
+
+    // this will register a QLTestAdapter for each WorkspaceFolder
+    this.push(new TestAdapterRegistrar(
+      testHub,
+      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer)
+    ));
+  }
+}
+
+/**
+ * Change the file extension of the specified path.
+ * @param p The original file path.
+ * @param ext The new extension, including the `.`.
+ */
+function changeExtension(p: string, ext: string): string {
+  return p.substr(0, p.length - path.extname(p).length) + ext;
+}
+
+/**
+ * Test adapter for QL tests.
+ */
+export class QLTestAdapter extends DisposableObject implements TestAdapter {
+  private readonly qlPackDiscovery: QLPackDiscovery;
+  private readonly qlTestDiscovery: QLTestDiscovery;
+  private readonly _tests = this.push(
+    new EventEmitter<TestLoadStartedEvent | TestLoadFinishedEvent>());
+  private readonly _testStates = this.push(
+    new EventEmitter<TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent |
+    TestEvent>());
+  private readonly _autorun = this.push(new EventEmitter<void>());
+  private runningTask?: vscode.CancellationTokenSource = undefined;
+
+  constructor(
+    public readonly workspaceFolder: vscode.WorkspaceFolder,
+    private readonly cliServer: CodeQLCliServer
+  ) {
+    super();
+
+    this.qlPackDiscovery = this.push(new QLPackDiscovery(workspaceFolder, cliServer));
+    this.qlTestDiscovery = this.push(new QLTestDiscovery(this.qlPackDiscovery, cliServer));
+
+    this.push(this.qlTestDiscovery.onDidChangeTests(this.discoverTests, this));
+  }
+
+  public get tests(): Event<TestLoadStartedEvent | TestLoadFinishedEvent> {
+    return this._tests.event;
+  }
+
+  public get testStates(): Event<TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent |
+  TestEvent> {
+
+    return this._testStates.event;
+  }
+
+  public get autorun(): Event<void> | undefined {
+    return this._autorun.event;
+  }
+
+  private static createTestOrSuiteInfos(testNodes: readonly QLTestNode[]):
+  (TestSuiteInfo | TestInfo)[] {
+
+    return testNodes.map((childNode) => {
+      return QLTestAdapter.createTestOrSuiteInfo(childNode);
+    });
+  }
+
+  private static createTestOrSuiteInfo(testNode: QLTestNode): TestSuiteInfo | TestInfo {
+    if (testNode instanceof QLTestFile) {
+      return QLTestAdapter.createTestInfo(testNode);
+    }
+    else if (testNode instanceof QLTestDirectory) {
+      return QLTestAdapter.createTestSuiteInfo(testNode, testNode.name);
+    }
+    else {
+      throw new Error('Unexpected test type.');
+    }
+  }
+
+  private static createTestInfo(testFile: QLTestFile): TestInfo {
+    return {
+      type: 'test',
+      id: testFile.path,
+      label: testFile.name,
+      tooltip: testFile.path,
+      file: testFile.path
+    };
+  }
+
+  private static createTestSuiteInfo(testDirectory: QLTestDirectory, label: string):
+  TestSuiteInfo {
+
+    return {
+      type: 'suite',
+      id: testDirectory.path,
+      label: label,
+      children: QLTestAdapter.createTestOrSuiteInfos(testDirectory.children),
+      tooltip: testDirectory.path
+    };
+  }
+
+  public async load(): Promise<void> {
+    this.discoverTests();
+  }
+
+  private discoverTests(): void {
+    this._tests.fire(<TestLoadStartedEvent>{ type: 'started' });
+
+    const testDirectories = this.qlTestDiscovery.testDirectories;
+    const children = testDirectories.map(
+      testDirectory => QLTestAdapter.createTestSuiteInfo(testDirectory, testDirectory.name)
+    );
+    const testSuite: TestSuiteInfo = {
+      type: 'suite',
+      label: 'CodeQL',
+      id: '.',
+      children
+    };
+
+    this._tests.fire(<TestLoadFinishedEvent>{
+      type: 'finished',
+      suite: children.length > 0 ? testSuite : undefined
+    });
+  }
+
+  public async run(tests: string[]): Promise<void> {
+    if (this.runningTask !== undefined) {
+      throw new Error('Tests already running.');
+    }
+
+    testLogger.outputChannel.clear();
+    testLogger.outputChannel.show(true);
+
+    this.runningTask = this.track(new CancellationTokenSource());
+
+    this._testStates.fire(<TestRunStartedEvent>{ type: 'started', tests: tests });
+
+    const testAdapter = this;
+
+    try {
+      await this.runTests(tests, this.runningTask.token);
+    }
+    catch (e) {
+    }
+    testAdapter._testStates.fire(<TestRunFinishedEvent>{ type: 'finished' });
+    testAdapter.clearTask();
+  }
+
+  private clearTask(): void {
+    if (this.runningTask !== undefined) {
+      const runningTask = this.runningTask;
+      this.runningTask = undefined;
+      this.disposeAndStopTracking(runningTask);
+    }
+  }
+
+  public cancel(): void {
+    if (this.runningTask !== undefined) {
+      testLogger.log('Cancelling test run...');
+      this.runningTask.cancel();
+      this.clearTask();
+    }
+  }
+
+  private async runTests(tests: string[], cancellationToken: CancellationToken): Promise<void> {
+    const workspacePaths = await getOnDiskWorkspaceFolders();
+    for await (const event of await this.cliServer.runTests(tests, workspacePaths, {
+      cancellationToken: cancellationToken,
+      logger: testLogger
+    })) {
+      this._testStates.fire({
+        type: 'test',
+        state: event.pass ? 'passed' : 'failed',
+        test: event.test
+      });
+    }
+  }
+}

extensions/ql-vscode/src/test-tree-node.ts

@@ -0,0 +1,9 @@
+import { TestSuiteInfo, TestInfo } from 'vscode-test-adapter-api';
+
+/**
+ * Tree view node for a test, suite, or collection. This object is passed as the argument to the
+ * command handler of a context menu item for a tree view item. 
+ */
+export interface TestTreeNode {
+  readonly info: TestSuiteInfo | TestInfo;
+}

extensions/ql-vscode/src/test-ui.ts

@@ -0,0 +1,85 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { Uri, TextDocumentShowOptions, commands, window } from 'vscode';
+import { TestTreeNode } from './test-tree-node';
+import { DisposableObject, UIService } from 'semmle-vscode-utils';
+import { TestHub, TestController, TestAdapter, TestRunStartedEvent, TestRunFinishedEvent, TestEvent, TestSuiteEvent } from 'vscode-test-adapter-api';
+import { QLTestAdapter, getExpectedFile, getActualFile } from './test-adapter';
+
+/**
+ * Test event listener. Currently unused, but left in to keep the plumbing hooked up for future use.
+ */
+class QLTestListener extends DisposableObject {
+  constructor(adapter: TestAdapter) {
+    super();
+
+    this.push(adapter.testStates(this.onTestStatesEvent, this));
+  }
+
+  private onTestStatesEvent(_e: TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent | TestEvent): void {
+  }
+}
+
+/**
+ * Service that implements all UI and commands for QL tests.
+ */
+export class TestUIService extends UIService implements TestController {
+  private readonly listeners: Map<TestAdapter, QLTestListener> = new Map();
+
+  constructor(private readonly testHub: TestHub) {
+    super();
+
+    this.registerCommand('codeQLTests.showOutputDifferences', this.showOutputDifferences);
+    this.registerCommand('codeQLTests.acceptOutput', this.acceptOutput);
+
+    testHub.registerTestController(this);
+  }
+
+  public dispose(): void {
+    this.testHub.unregisterTestController(this);
+
+    super.dispose();
+  }
+
+  public registerTestAdapter(adapter: TestAdapter): void {
+    this.listeners.set(adapter, new QLTestListener(adapter));
+  }
+
+  public unregisterTestAdapter(adapter: TestAdapter): void {
+    if (adapter instanceof QLTestAdapter) {
+      this.listeners.delete(adapter);
+    }
+  }
+
+  private async acceptOutput(node: TestTreeNode): Promise<void> {
+    const testId = node.info.id;
+    const stat = await fs.lstat(testId);
+    if (stat.isFile()) {
+      const expectedPath = getExpectedFile(testId);
+      const actualPath = getActualFile(testId);
+      await fs.copy(actualPath, expectedPath, { overwrite: true });
+    }
+  }
+
+  private async showOutputDifferences(node: TestTreeNode): Promise<void> {
+    const testId = node.info.id;
+    const stat = await fs.lstat(testId);
+    if (stat.isFile()) {
+      const expectedPath = getExpectedFile(testId);
+      const expectedUri = Uri.file(expectedPath);
+      const actualPath = getActualFile(testId);
+      const options: TextDocumentShowOptions = {
+        preserveFocus: true,
+        preview: true
+      };
+      if (await fs.pathExists(actualPath)) {
+        const actualUri = Uri.file(actualPath);
+        await commands.executeCommand<void>('vscode.diff', expectedUri, actualUri,
+          `Expected vs. Actual for ${path.basename(testId)}`, options);
+      }
+      else {
+        await window.showTextDocument(expectedUri, options);
+      }
+    }
+  }
+}

extensions/ql-vscode/src/upgrades.ts

@@ -0,0 +1,203 @@
+import * as vscode from 'vscode';
+import { DatabaseItem } from './databases';
+import * as helpers from './helpers';
+import { logger } from './logging';
+import * as messages from './messages';
+import * as qsClient from './queryserver-client';
+import { upgradesTmpDir, UserCancellationException } from './run-queries';
+
+/**
+ * Maximum number of lines to include from database upgrade message,
+ * to work around the fact that we can't guarantee a scrollable text
+ * box for it when displaying in dialog boxes.
+ */
+const MAX_UPGRADE_MESSAGE_LINES = 10;
+
+/**
+ * Checks whether the given database can be upgraded to the given target DB scheme,
+ * and whether the user wants to proceed with the upgrade.
+ * Reports errors to both the user and the console.
+ * @returns the `UpgradeParams` needed to start the upgrade, if the upgrade is possible and was confirmed by the user, or `undefined` otherwise.
+ */
+async function checkAndConfirmDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]
+): Promise<messages.UpgradeParams | undefined> {
+  if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
+    helpers.showAndLogErrorMessage("Database is invalid, and cannot be upgraded.");
+    return;
+  }
+  const params: messages.UpgradeParams = {
+    fromDbscheme: db.contents.dbSchemeUri.fsPath,
+    toDbscheme: targetDbScheme.fsPath,
+    additionalUpgrades: upgradesDirectories.map(uri => uri.fsPath)
+  };
+
+  let checkUpgradeResult: messages.CheckUpgradeResult;
+  try {
+    qs.logger.log('Checking database upgrade...');
+    checkUpgradeResult = await checkDatabaseUpgrade(qs, params);
+  }
+  catch (e) {
+    helpers.showAndLogErrorMessage(`Database cannot be upgraded: ${e}`);
+    return;
+  }
+  finally {
+    qs.logger.log('Done checking database upgrade.');
+  }
+
+  const checkedUpgrades = checkUpgradeResult.checkedUpgrades;
+  if (checkedUpgrades === undefined) {
+    const error = checkUpgradeResult.upgradeError || '[no error message available]';
+    await helpers.showAndLogErrorMessage(`Database cannot be upgraded: ${error}`);
+    return;
+  }
+
+  if (checkedUpgrades.scripts.length === 0) {
+    await helpers.showAndLogInformationMessage('Database is already up to date; nothing to do.');
+    return;
+  }
+
+  let curSha = checkedUpgrades.initialSha;
+  let descriptionMessage = '';
+  for (const script of checkedUpgrades.scripts) {
+    descriptionMessage += `Would perform upgrade: ${script.description}\n`;
+    descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
+    curSha = script.newSha;
+  }
+
+  const targetSha = checkedUpgrades.targetSha;
+  if (curSha != targetSha) {
+    // Newlines aren't rendered in notifications: https://github.com/microsoft/vscode/issues/48900
+    // A modal dialog would be rendered better, but is more intrusive.
+    await helpers.showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
+    Can upgrade from ${checkedUpgrades.initialSha} (current) to ${curSha}, but cannot reach ${targetSha} (target).`);
+    // TODO: give a more informative message if we think the DB is ahead of the target DB scheme
+    return;
+  }
+
+  logger.log(descriptionMessage);
+  // Ask the user to confirm the upgrade.
+
+  const showLogItem: vscode.MessageItem = { title: 'No, Show Changes', isCloseAffordance: true };
+  const yesItem = { title: 'Yes', isCloseAffordance: false };
+  const noItem = { title: 'No', isCloseAffordance: true }
+  const dialogOptions: vscode.MessageItem[] = [yesItem, noItem];
+
+  let messageLines = descriptionMessage.split('\n');
+  if (messageLines.length > MAX_UPGRADE_MESSAGE_LINES) {
+    messageLines = messageLines.slice(0, MAX_UPGRADE_MESSAGE_LINES);
+    messageLines.push(`The list of upgrades was truncated, click "No, Show Changes" to see the full list.`);
+    dialogOptions.push(showLogItem);
+  }
+
+  const message = `Should the database ${db.databaseUri.fsPath} be upgraded?\n\n${messageLines.join("\n")}`;
+  const chosenItem = await vscode.window.showInformationMessage(message, { modal: true }, ...dialogOptions);
+
+  if (chosenItem === showLogItem) {
+    logger.outputChannel.show();
+  }
+
+  if (chosenItem === yesItem) {
+    return params;
+  }
+  else {
+    throw new UserCancellationException('User cancelled the database upgrade.');
+  }
+}
+
+/**
+ * Command handler for 'Upgrade Database'.
+ * Attempts to upgrade the given database to the given target DB scheme, using the given directory of upgrades.
+ * First performs a dry-run and prompts the user to confirm the upgrade.
+ * Reports errors during compilation and evaluation of upgrades to the user.
+ */
+export async function upgradeDatabase(
+  qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]
+): Promise<messages.RunUpgradeResult | undefined> {
+  const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories);
+
+  if (upgradeParams === undefined) {
+    return;
+  }
+
+  let compileUpgradeResult: messages.CompileUpgradeResult;
+  try {
+    compileUpgradeResult = await compileDatabaseUpgrade(qs, upgradeParams);
+  }
+  catch (e) {
+    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
+    return;
+  }
+  finally {
+    qs.logger.log('Done compiling database upgrade.')
+  }
+
+  if (compileUpgradeResult.compiledUpgrades === undefined) {
+    const error = compileUpgradeResult.error || '[no error message available]';
+    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
+    return;
+  }
+
+  try {
+    qs.logger.log('Running the following database upgrade:');
+    qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
+    return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades);
+  }
+  catch (e) {
+    helpers.showAndLogErrorMessage(`Database upgrade failed: ${e}`);
+    return;
+  }
+  finally {
+    qs.logger.log('Done running database upgrade.')
+  }
+}
+
+async function checkDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams
+): Promise<messages.CheckUpgradeResult> {
+  return helpers.withProgress({
+    location: vscode.ProgressLocation.Notification,
+    title: "Checking for database upgrades",
+    cancellable: true,
+  }, (progress, token) => qs.sendRequest(messages.checkUpgrade, upgradeParams, token, progress));
+}
+
+async function compileDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams
+): Promise<messages.CompileUpgradeResult> {
+  const params: messages.CompileUpgradeParams = {
+    upgrade: upgradeParams,
+    upgradeTempDir: upgradesTmpDir.name
+  }
+
+  return helpers.withProgress({
+    location: vscode.ProgressLocation.Notification,
+    title: "Compiling database upgrades",
+    cancellable: true,
+  }, (progress, token) => qs.sendRequest(messages.compileUpgrade, params, token, progress));
+}
+
+async function runDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, db: DatabaseItem, upgrades: messages.CompiledUpgrades
+): Promise<messages.RunUpgradeResult> {
+
+  if (db.contents === undefined || db.contents.datasetUri === undefined) {
+    throw new Error('Can\'t upgrade an invalid database.');
+  }
+  const database: messages.Dataset = {
+    dbDir: db.contents.datasetUri.fsPath,
+    workingSet: 'default'
+  };
+
+  const params: messages.RunUpgradeParams = {
+    db: database,
+    timeoutSecs: qs.config.timeoutSecs,
+    toRun: upgrades
+  };
+
+  return helpers.withProgress({
+    location: vscode.ProgressLocation.Notification,
+    title: "Running database upgrades",
+    cancellable: true,
+  }, (progress, token) => qs.sendRequest(messages.runUpgrade, params, token, progress));
+}

extensions/ql-vscode/src/view/.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+  ecmaFeatures: {
+    jsx: true,
+  },
+  env: {
+    browser: true
+  }
+}

extensions/ql-vscode/src/view/alert-table.tsx

@@ -1,78 +1,25 @@
 import * as path from 'path';
 import * as React from 'react';
 import * as Sarif from 'sarif';
-import { LocationStyle, ResolvableLocationValue } from 'semmle-bqrs';
+import * as Keys from '../result-keys';
+import { LocationStyle } from 'semmle-bqrs';
 import * as octicons from './octicons';
-import { className, renderLocation, ResultTableProps, zebraStripe } from './result-table-utils';
+import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection } from './result-table-utils';
-import { PathTableResultSet } from './results';
+import { PathTableResultSet, onNavigation, NavigationEvent, vscode } from './results';
+import { parseSarifPlainTextMessage, parseSarifLocation } from '../sarif-utils';
+import { InterpretedResultsSortColumn, SortDirection, InterpretedResultsSortState } from '../interface-types';
 
 export type PathTableProps = ResultTableProps & { resultSet: PathTableResultSet };
 export interface PathTableState {
   expanded: { [k: string]: boolean };
-}
+  selectedPathNode: undefined | Keys.PathNode;
-
-interface SarifLink {
-  dest: number
-  text: string
-}
-
-type ParsedSarifLocation =
-  | ResolvableLocationValue
-  // Resolvable locations have a `file` field, but it will sometimes include
-  // a source location prefix, which contains build-specific information the user
-  // doesn't really need to see. We ensure that `userVisibleFile` will not contain
-  // that, and is appropriate for display in the UI.
-  & { userVisibleFile: string }
-  | { t: 'NoLocation', hint: string };
-
-type SarifMessageComponent = string | SarifLink
-
-/**
- * Unescape "[", "]" and "\\" like in sarif plain text messages
- */
-function unescapeSarifText(message: string): string {
-  return message.replace(/\\\[/g, "[").replace(/\\\]/g, "]").replace(/\\\\/, "\\");
-}
-
-function parseSarifPlainTextMessage(message: string): SarifMessageComponent[] {
-  let results: SarifMessageComponent[] = [];
-
-  // We want something like "[linkText](4)", except that "[" and "]" may be escaped. The lookbehind asserts
-  // that the initial [ is not escaped. Then we parse a link text with "[" and "]" escaped. Then we parse the numerical target.
-  // Technically we could have any uri in the target but we don't output that yet.
-  // The possibility of escaping outside the link is not mentioned in the sarif spec but we always output sartif this way.
-  const linkRegex = /(?<=(?<!\\)(\\\\)*)\[(?<linkText>([^\\\]\[]|\\\\|\\\]|\\\[)*)\]\((?<linkTarget>[0-9]+)\)/g;
-  let result: RegExpExecArray | null;
-  let curIndex = 0;
-  while ((result = linkRegex.exec(message)) !== null) {
-    results.push(unescapeSarifText(message.substring(curIndex, result.index)));
-    const linkText = result.groups!["linkText"];
-    const linkTarget = +result.groups!["linkTarget"];
-    results.push({ dest: linkTarget, text: unescapeSarifText(linkText) });
-    curIndex = result.index + result[0].length;
-  }
-  results.push(unescapeSarifText(message.substring(curIndex, message.length)));
-  return results;
-}
-
-/**
- * Computes a path normalized to reflect conventional normalization
- * of windows paths into zip archive paths.
- * @param sourceLocationPrefix The source location prefix of a database. May be
- * unix style `/foo/bar/baz` or windows-style `C:\foo\bar\baz`.
- * @param sarifRelativeUri A uri relative to sourceLocationPrefix.
- * @returns A string that is valid for the `.file` field of a `FivePartLocation`:
- * directory separators are normalized, but drive letters `C:` may appear.
- */
-export function getPathRelativeToSourceLocationPrefix(sourceLocationPrefix: string, sarifRelativeUui: string) {
-  const normalizedSourceLocationPrefix = sourceLocationPrefix.replace(/\\/g, '/');
-  return path.join(normalizedSourceLocationPrefix, decodeURIComponent(sarifRelativeUui));
 }
 
 export class PathTable extends React.Component<PathTableProps, PathTableState> {
   constructor(props: PathTableProps) {
     super(props);
-    this.state = { expanded: {} };
+    this.state = { expanded: {}, selectedPathNode: undefined };
+    this.handleNavigationEvent = this.handleNavigationEvent.bind(this);
   }
 
   /**
@@ -97,15 +44,47 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
     e.preventDefault();
   }
 
+  sortClass(column: InterpretedResultsSortColumn): string {
+    const sortState = this.props.resultSet.sortState;
+    if (sortState !== undefined && sortState.sortBy === column) {
+      return sortState.sortDirection === SortDirection.asc ? 'sort-asc' : 'sort-desc';
+    }
+    else {
+      return 'sort-none';
+    }
+  }
+
+  getNextSortState(column: InterpretedResultsSortColumn): InterpretedResultsSortState | undefined {
+    const oldSortState = this.props.resultSet.sortState;
+    const prevDirection = oldSortState && oldSortState.sortBy === column ? oldSortState.sortDirection : undefined;
+    const nextDirection = nextSortDirection(prevDirection, true);
+    return nextDirection === undefined ? undefined :
+      { sortBy: column, sortDirection: nextDirection };
+  }
+
+  toggleSortStateForColumn(column: InterpretedResultsSortColumn): void {
+    vscode.postMessage({
+      t: 'changeInterpretedSort',
+      sortState: this.getNextSortState(column),
+    });
+  }
+
   render(): JSX.Element {
     const { databaseUri, resultSet } = this.props;
 
+    const header = <thead>
+      <tr>
+        <th colSpan={2}></th>
+        <th className={this.sortClass('alert-message') + ' vscode-codeql__alert-message-cell'} colSpan={3} onClick={() => this.toggleSortStateForColumn('alert-message')}>Message</th>
+      </tr>
+    </thead>;
+
     const rows: JSX.Element[] = [];
     const { numTruncatedResults, sourceLocationPrefix } = resultSet;
 
     function renderRelatedLocations(msg: string, relatedLocations: Sarif.Location[]): JSX.Element[] {
       const relatedLocationsById: { [k: string]: Sarif.Location } = {};
-      for (let loc of relatedLocations) {
+      for (const loc of relatedLocations) {
         relatedLocationsById[loc.id!] = loc;
       }
 
@@ -118,7 +97,8 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
         if (typeof part === "string") {
           result.push(<span>{part} </span>);
         } else {
-          const renderedLocation = renderSarifLocationWithText(part.text, relatedLocationsById[part.dest]);
+          const renderedLocation = renderSarifLocationWithText(part.text, relatedLocationsById[part.dest],
+            undefined);
           result.push(<span>{renderedLocation} </span>);
         }
       } return result;
@@ -130,75 +110,23 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
       return <span title={locationHint}>{msg}</span>;
     }
 
-    function parseSarifLocation(loc: Sarif.Location): ParsedSarifLocation {
+    const updateSelectionCallback = (pathNodeKey: Keys.PathNode | undefined) => {
-      const physicalLocation = loc.physicalLocation;
+      return () => {
-      if (physicalLocation === undefined)
+        this.setState(previousState => ({
-        return { t: 'NoLocation', hint: 'no physical location' };
+          ...previousState,
-      if (physicalLocation.artifactLocation === undefined)
+          selectedPathNode: pathNodeKey
-        return { t: 'NoLocation', hint: 'no artifact location' };
+        }));
-      if (physicalLocation.artifactLocation.uri === undefined)
-        return { t: 'NoLocation', hint: 'artifact location has no uri' };
-
-      // This is not necessarily really an absolute uri; it could either be a
-      // file uri or a relative uri.
-      const uri = physicalLocation.artifactLocation.uri;
-
-      const fileUriRegex = /^file:/;
-      const effectiveLocation = uri.match(fileUriRegex) ?
-        decodeURIComponent(uri.replace(fileUriRegex, '')) :
-        getPathRelativeToSourceLocationPrefix(sourceLocationPrefix, uri);
-      const userVisibleFile = uri.match(fileUriRegex) ?
-        decodeURIComponent(uri.replace(fileUriRegex, '')) :
-        uri;
-
-      if (physicalLocation.region === undefined) {
-        // If the region property is absent, the physicalLocation object refers to the entire file.
-        // Source: https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Toc16012638.
-        // TODO: Do we get here if we provide a non-filesystem URL?
-        return {
-          t: LocationStyle.WholeFile,
-          file: effectiveLocation,
-          userVisibleFile,
-        };
-      } else {
-        const region = physicalLocation.region;
-        // We assume that the SARIF we're given always has startLine
-        // This is not mandated by the SARIF spec, but should be true of
-        // SARIF output by our own tools.
-        const lineStart = region.startLine!;
-
-        // These defaults are from SARIF 2.1.0 spec, section 3.30.2, "Text Regions"
-        // https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Ref493492556
-        const lineEnd = region.endLine === undefined ? lineStart : region.endLine;
-        const colStart = region.startColumn === undefined ? 1 : region.startColumn;
-
-        // We also assume that our tools will always supply `endColumn` field, which is
-        // fortunate, since the SARIF spec says that it defaults to the end of the line, whose
-        // length we don't know at this point in the code.
-        //
-        // It is off by one with respect to the way vscode counts columns in selections.
-        const colEnd = region.endColumn! - 1;
-
-        return {
-          t: LocationStyle.FivePart,
-          file: effectiveLocation,
-          userVisibleFile,
-          lineStart,
-          colStart,
-          lineEnd,
-          colEnd,
-        };
       }
-    }
+    };
 
-    function renderSarifLocationWithText(text: string | undefined, loc: Sarif.Location): JSX.Element | undefined {
+    function renderSarifLocationWithText(text: string | undefined, loc: Sarif.Location, pathNodeKey: Keys.PathNode | undefined): JSX.Element | undefined {
-      const parsedLoc = parseSarifLocation(loc);
+      const parsedLoc = parseSarifLocation(loc, sourceLocationPrefix);
       switch (parsedLoc.t) {
         case 'NoLocation':
           return renderNonLocation(text, parsedLoc.hint);
         case LocationStyle.FivePart:
         case LocationStyle.WholeFile:
-          return renderLocation(parsedLoc, text, databaseUri);
+          return renderLocation(parsedLoc, text, databaseUri, undefined, updateSelectionCallback(pathNodeKey));
       }
       return undefined;
     }
@@ -207,8 +135,8 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
      * Render sarif location as a link with the text being simply a
      * human-readable form of the location itself.
      */
-    function renderSarifLocation(loc: Sarif.Location): JSX.Element | undefined {
+    function renderSarifLocation(loc: Sarif.Location, pathNodeKey: Keys.PathNode | undefined): JSX.Element | undefined {
-      const parsedLoc = parseSarifLocation(loc);
+      const parsedLoc = parseSarifLocation(loc, sourceLocationPrefix);
       let shortLocation, longLocation: string;
       switch (parsedLoc.t) {
         case 'NoLocation':
@@ -216,11 +144,11 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
         case LocationStyle.WholeFile:
           shortLocation = `${path.basename(parsedLoc.userVisibleFile)}`;
           longLocation = `${parsedLoc.userVisibleFile}`;
-          return renderLocation(parsedLoc, shortLocation, databaseUri, longLocation);
+          return renderLocation(parsedLoc, shortLocation, databaseUri, longLocation, updateSelectionCallback(pathNodeKey));
         case LocationStyle.FivePart:
           shortLocation = `${path.basename(parsedLoc.userVisibleFile)}:${parsedLoc.lineStart}:${parsedLoc.colStart}`;
           longLocation = `${parsedLoc.userVisibleFile}`;
-          return renderLocation(parsedLoc, shortLocation, databaseUri, longLocation);
+          return renderLocation(parsedLoc, shortLocation, databaseUri, longLocation, updateSelectionCallback(pathNodeKey));
       }
     }
 
@@ -245,7 +173,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
       const currentResultExpanded = this.state.expanded[expansionIndex];
       const indicator = currentResultExpanded ? octicons.chevronDown : octicons.chevronRight;
       const location = result.locations !== undefined && result.locations.length > 0 &&
-        renderSarifLocation(result.locations[0]);
+        renderSarifLocation(result.locations[0], Keys.none);
       const locationCells = <td className="vscode-codeql__location-cell">{location}</td>;
 
       if (result.codeFlows === undefined) {
@@ -260,12 +188,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
         );
       }
       else {
-        const paths: Sarif.ThreadFlow[] = [];
+        const paths: Sarif.ThreadFlow[] = Keys.getAllPaths(result);
-        for (const codeFlow of result.codeFlows) {
-          for (const threadFlow of codeFlow.threadFlows) {
-            paths.push(threadFlow);
-          }
-        }
 
         const indices = paths.length == 1 ?
           [expansionIndex, expansionIndex + 1] : /* if there's exactly one path, auto-expand
@@ -288,7 +211,8 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
         );
         expansionIndex++;
 
-        paths.forEach(path => {
+        paths.forEach((path, pathIndex) => {
+          const pathKey = { resultIndex, pathIndex };
           const currentPathExpanded = this.state.expanded[expansionIndex];
           if (currentResultExpanded) {
             const indicator = currentPathExpanded ? octicons.chevronDown : octicons.chevronRight;
@@ -305,25 +229,27 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
           expansionIndex++;
 
           if (currentResultExpanded && currentPathExpanded) {
-            let pathIndex = 1;
+            const pathNodes = path.locations;
-            for (const step of path.locations) {
+            for (let pathNodeIndex = 0; pathNodeIndex < pathNodes.length; ++pathNodeIndex) {
+              const pathNodeKey: Keys.PathNode = { ...pathKey, pathNodeIndex };
+              const step = pathNodes[pathNodeIndex];
               const msg = step.location !== undefined && step.location.message !== undefined ?
-                renderSarifLocationWithText(step.location.message.text, step.location) :
+                renderSarifLocationWithText(step.location.message.text, step.location, pathNodeKey) :
                 '[no location]';
               const additionalMsg = step.location !== undefined ?
-                renderSarifLocation(step.location) :
+                renderSarifLocation(step.location, pathNodeKey) :
                 '';
-
+              const isSelected = Keys.equalsNotUndefined(this.state.selectedPathNode, pathNodeKey);
-              const stepIndex = resultIndex + pathIndex;
+              const stepIndex = pathNodeIndex + 1; // Convert to 1-based
+              const zebraIndex = resultIndex + stepIndex;
               rows.push(
-                <tr>
+                <tr className={isSelected ? 'vscode-codeql__selected-path-node' : undefined}>
                   <td className="vscode-codeql__icon-cell"><span className="vscode-codeql__vertical-rule"></span></td>
                   <td className="vscode-codeql__icon-cell"><span className="vscode-codeql__vertical-rule"></span></td>
-                  <td {...zebraStripe(stepIndex, 'vscode-codeql__path-index-cell')}>{pathIndex}</td>
+                  <td {...selectableZebraStripe(isSelected, zebraIndex, 'vscode-codeql__path-index-cell')}>{stepIndex}</td>
-                  <td {...zebraStripe(stepIndex)}>{msg} </td>
+                  <td {...selectableZebraStripe(isSelected, zebraIndex)}>{msg} </td>
-                  <td {...zebraStripe(stepIndex, 'vscode-codeql__location-cell')}>{additionalMsg}</td>
+                  <td {...selectableZebraStripe(isSelected, zebraIndex, 'vscode-codeql__location-cell')}>{additionalMsg}</td>
                 </tr>);
-              pathIndex++;
             }
           }
         });
@@ -338,7 +264,39 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
     }
 
     return <table className={className}>
+      {header}
       <tbody>{rows}</tbody>
     </table>;
   }
+
+  private handleNavigationEvent(event: NavigationEvent) {
+    this.setState(prevState => {
+      const { selectedPathNode } = prevState;
+      if (selectedPathNode === undefined) return prevState;
+
+      const path = Keys.getPath(this.props.resultSet.sarif, selectedPathNode);
+      if (path === undefined) return prevState;
+
+      const nextIndex = selectedPathNode.pathNodeIndex + event.direction;
+      if (nextIndex < 0 || nextIndex >= path.locations.length) return prevState;
+
+      const sarifLoc = path.locations[nextIndex].location;
+      if (sarifLoc === undefined) return prevState;
+
+      const loc = parseSarifLocation(sarifLoc, this.props.resultSet.sourceLocationPrefix);
+      if (loc.t === 'NoLocation') return prevState;
+
+      jumpToLocation(loc, this.props.databaseUri);
+      const newSelection = { ...selectedPathNode, pathNodeIndex: nextIndex };
+      return { ...prevState, selectedPathNode: newSelection };
+    });
+  }
+
+  componentDidMount() {
+    onNavigation.addListener(this.handleNavigationEvent);
+  }
+
+  componentWillUnmount() {
+    onNavigation.removeListener(this.handleNavigationEvent);
+  }
 }

extensions/ql-vscode/src/view/event-handler-list.ts

@@ -0,0 +1,25 @@
+export type EventHandler<T> = (event: T) => void;
+
+/**
+ * A set of listeners for events of type `T`.
+ */
+export class EventHandlers<T> {
+  private handlers: EventHandler<T>[] = [];
+
+  public addListener(handler: EventHandler<T>) {
+    this.handlers.push(handler);
+  }
+
+  public removeListener(handler: EventHandler<T>) {
+    const index = this.handlers.indexOf(handler);
+    if (index !== -1) {
+      this.handlers.splice(index, 1);
+    }
+  }
+
+  public fire(event: T) {
+    for (const handler of this.handlers) {
+      handler(event);
+    }
+  }
+}

extensions/ql-vscode/src/view/raw-results-table.tsx

@@ -1,12 +1,11 @@
 import * as React from "react";
-import { renderLocation, ResultTableProps, zebraStripe, className } from "./result-table-utils";
+import { renderLocation, ResultTableProps, zebraStripe, className, nextSortDirection } from "./result-table-utils";
 import { RawTableResultSet, ResultValue, vscode } from "./results";
-import { assertNever } from "../helpers-pure";
+import { SortDirection, RAW_RESULTS_LIMIT, RawResultsSortState } from "../interface-types";
-import { SortDirection, SortState, RAW_RESULTS_LIMIT } from "../interface-types";
 
 export type RawTableProps = ResultTableProps & {
-  resultSet: RawTableResultSet,
+  resultSet: RawTableResultSet;
-  sortState?: SortState;
+  sortState?: RawResultsSortState;
 };
 
 export class RawTable extends React.Component<RawTableProps, {}> {
@@ -55,7 +54,7 @@ export class RawTable extends React.Component<RawTableProps, {}> {
               <th key={-1}><b>#</b></th>,
               ...resultSet.schema.columns.map((col, index) => {
                 const displayName = col.name || `[${index}]`;
-                const sortDirection = this.props.sortState && index === this.props.sortState.columnIndex ? this.props.sortState.direction : undefined;
+                const sortDirection = this.props.sortState && index === this.props.sortState.columnIndex ? this.props.sortState.sortDirection : undefined;
                 return <th className={"sort-" + (sortDirection !== undefined ? SortDirection[sortDirection] : "none")} key={index} onClick={() => this.toggleSortStateForColumn(index)}><b>{displayName}</b></th>;
               })
             ]
@@ -68,13 +67,13 @@ export class RawTable extends React.Component<RawTableProps, {}> {
     </table>;
   }
 
-  private toggleSortStateForColumn(index: number) {
+  private toggleSortStateForColumn(index: number): void {
     const sortState = this.props.sortState;
-    const prevDirection = sortState && sortState.columnIndex === index ? sortState.direction : undefined;
+    const prevDirection = sortState && sortState.columnIndex === index ? sortState.sortDirection : undefined;
     const nextDirection = nextSortDirection(prevDirection);
     const nextSortState = nextDirection === undefined ? undefined : {
       columnIndex: index,
-      direction: nextDirection
+      sortDirection: nextDirection
     };
     vscode.postMessage({
       t: 'changeSort',
@@ -84,7 +83,6 @@ export class RawTable extends React.Component<RawTableProps, {}> {
   }
 }
 
-
 /**
  * Render one column of a tuple.
  */
@@ -99,15 +97,3 @@ function renderTupleValue(v: ResultValue, databaseUri: string): JSX.Element {
     return renderLocation(v.location, v.label, databaseUri);
   }
 }
-
-function nextSortDirection(direction: SortDirection | undefined): SortDirection {
-  switch (direction) {
-    case SortDirection.asc:
-      return SortDirection.desc;
-    case SortDirection.desc:
-    case undefined:
-      return SortDirection.asc;
-    default:
-      return assertNever(direction);
-  }
-}

extensions/ql-vscode/src/view/result-table-utils.tsx

@@ -1,42 +1,52 @@
 import * as React from 'react';
 import { LocationValue, ResolvableLocationValue, tryGetResolvableLocation } from 'semmle-bqrs';
-import { SortState } from '../interface-types';
+import { RawResultsSortState, QueryMetadata, SortDirection } from '../interface-types';
 import { ResultSet, vscode } from './results';
+import { assertNever } from '../helpers-pure';
 
 export interface ResultTableProps {
   resultSet: ResultSet;
   databaseUri: string;
+  metadata?: QueryMetadata;
   resultsPath: string | undefined;
-  sortState?: SortState;
+  sortState?: RawResultsSortState;
 }
 
 export const className = 'vscode-codeql__result-table';
 export const tableSelectionHeaderClassName = 'vscode-codeql__table-selection-header';
+export const alertExtrasClassName = `${className}-alert-extras`;
 export const toggleDiagnosticsClassName = `${className}-toggle-diagnostics`;
 export const evenRowClassName = 'vscode-codeql__result-table-row--even';
 export const oddRowClassName = 'vscode-codeql__result-table-row--odd';
 export const pathRowClassName = 'vscode-codeql__result-table-row--path';
+export const selectedRowClassName = 'vscode-codeql__result-table-row--selected';
 
 export function jumpToLocationHandler(
   loc: ResolvableLocationValue,
-  databaseUri: string
+  databaseUri: string,
+  callback?: () => void
 ): (e: React.MouseEvent) => void {
   return (e) => {
-    vscode.postMessage({
+    jumpToLocation(loc, databaseUri);
-      t: 'viewSourceFile',
-      loc,
-      databaseUri
-    });
     e.preventDefault();
     e.stopPropagation();
+    if (callback) callback();
   };
 }
 
+export function jumpToLocation(loc: ResolvableLocationValue, databaseUri: string): void {
+  vscode.postMessage({
+    t: 'viewSourceFile',
+    loc,
+    databaseUri
+  });
+}
+
 /**
  * Render a location as a link which when clicked displays the original location.
  */
 export function renderLocation(loc: LocationValue | undefined, label: string | undefined,
-  databaseUri: string, title?: string): JSX.Element {
+  databaseUri: string, title?: string, callback?: () => void): JSX.Element {
 
   // If the label was empty, use a placeholder instead, so the link is still clickable.
   let displayLabel = label;
@@ -51,7 +61,7 @@ export function renderLocation(loc: LocationValue | undefined, label: string | u
       return <a href="#"
         className="vscode-codeql__result-table-location-link"
         title={title}
-        onClick={jumpToLocationHandler(resolvableLoc, databaseUri)}>{displayLabel}</a>;
+        onClick={jumpToLocationHandler(resolvableLoc, databaseUri, callback)}>{displayLabel}</a>;
     } else {
       return <span title={title}>{displayLabel}</span>;
     }
@@ -63,5 +73,32 @@ export function renderLocation(loc: LocationValue | undefined, label: string | u
  * Returns the attributes for a zebra-striped table row at position `index`.
  */
 export function zebraStripe(index: number, ...otherClasses: string[]): { className: string } {
-  return { className: [(index % 2) ? oddRowClassName : evenRowClassName, otherClasses].join(' ') };
+  return { className: [(index % 2) ? oddRowClassName : evenRowClassName, ...otherClasses].join(' ') };
+}
+
+/**
+ * Returns the attributes for a zebra-striped table row at position `index`,
+ * with highlighting if `isSelected` is true.
+ */
+export function selectableZebraStripe(isSelected: boolean, index: number, ...otherClasses: string[]): { className: string } {
+  return isSelected
+    ? { className: [selectedRowClassName, ...otherClasses].join(' ') }
+    : zebraStripe(index, ...otherClasses)
+}
+
+/**
+ * Returns the next sort direction when cycling through sort directions while clicking.
+ * if `includeUndefined` is true, include `undefined` in the cycle.
+ */
+export function nextSortDirection(direction: SortDirection | undefined, includeUndefined?: boolean): SortDirection | undefined {
+  switch (direction) {
+    case SortDirection.asc:
+      return SortDirection.desc;
+    case SortDirection.desc:
+      return includeUndefined ? undefined : SortDirection.asc;
+    case undefined:
+      return SortDirection.asc;
+    default:
+      return assertNever(direction);
+  }
 }

extensions/ql-vscode/src/view/result-tables.tsx

@@ -1,8 +1,8 @@
 import * as React from 'react';
-import { DatabaseInfo, Interpretation, SortState } from '../interface-types';
+import { DatabaseInfo, Interpretation, RawResultsSortState, QueryMetadata, ResultsPaths, InterpretedResultsSortState } from '../interface-types';
 import { PathTable } from './alert-table';
 import { RawTable } from './raw-results-table';
-import { ResultTableProps, tableSelectionHeaderClassName, toggleDiagnosticsClassName } from './result-table-utils';
+import { ResultTableProps, tableSelectionHeaderClassName, toggleDiagnosticsClassName, alertExtrasClassName } from './result-table-utils';
 import { ResultSet, vscode } from './results';
 
 /**
@@ -12,9 +12,11 @@ export interface ResultTablesProps {
   rawResultSets: readonly ResultSet[];
   interpretation: Interpretation | undefined;
   database: DatabaseInfo;
-  resultsPath: string | undefined;
+  metadata?: QueryMetadata;
-  kind: string | undefined;
+  resultsPath: string;
-  sortStates: Map<string, SortState>;
+  origResultsPaths: ResultsPaths;
+  sortStates: Map<string, RawResultsSortState>;
+  interpretedSortState?: InterpretedResultsSortState;
   isLoadingNewResults: boolean;
 }
 
@@ -29,6 +31,24 @@ const ALERTS_TABLE_NAME = 'alerts';
 const SELECT_TABLE_NAME = '#select';
 const UPDATING_RESULTS_TEXT_CLASS_NAME = "vscode-codeql__result-tables-updating-text";
 
+function getResultCount(resultSet: ResultSet): number {
+  switch (resultSet.t) {
+    case 'RawResultSet':
+      return resultSet.schema.tupleCount;
+    case 'SarifResultSet':
+      if (resultSet.sarif.runs.length === 0) return 0;
+      if (resultSet.sarif.runs[0].results === undefined) return 0;
+      return resultSet.sarif.runs[0].results.length + resultSet.numTruncatedResults;
+  }
+}
+
+function renderResultCountString(resultSet: ResultSet): JSX.Element {
+  const resultCount = getResultCount(resultSet);
+  return <span className="number-of-results">
+    {resultCount} {resultCount === 1 ? 'result' : 'results'}
+  </span>;
+}
+
 /**
  * Displays multiple `ResultTable` tables, where the table to be displayed is selected by a
  * dropdown.
@@ -70,35 +90,44 @@ export class ResultTables
     return [ALERTS_TABLE_NAME, SELECT_TABLE_NAME, resultSets[0].schema.name].filter(resultSetName => resultSetNames.includes(resultSetName))[0];
   }
 
-  private onChange = (event: React.ChangeEvent<HTMLSelectElement>): void => {
+  private onTableSelectionChange = (event: React.ChangeEvent<HTMLSelectElement>): void => {
     this.setState({ selectedTable: event.target.value });
   }
 
-  render(): React.ReactNode {
+  private alertTableExtras(): JSX.Element | undefined {
-    const { selectedTable } = this.state;
+    const { database, resultsPath, metadata, origResultsPaths } = this.props;
-    const resultSets = this.getResultSets();
-    const { database, resultsPath, kind } = this.props;
 
-    // Only show the Problems view display checkbox for the alerts table.
+    const displayProblemsAsAlertsToggle =
-    const diagnosticsCheckBox = selectedTable === ALERTS_TABLE_NAME ?
       <div className={toggleDiagnosticsClassName}>
         <input type="checkbox" id="toggle-diagnostics" name="toggle-diagnostics" onChange={(e) => {
           if (resultsPath !== undefined) {
             vscode.postMessage({
               t: 'toggleDiagnostics',
-              resultsPath: resultsPath,
+              origResultsPaths: origResultsPaths,
               databaseUri: database.databaseUri,
               visible: e.target.checked,
-              kind: kind
+              metadata: metadata
             });
           }
         }} />
         <label htmlFor="toggle-diagnostics">Show results in Problems view</label>
-      </div> : undefined;
+      </div>;
+
+    return <div className={alertExtrasClassName}>
+      {displayProblemsAsAlertsToggle}
+    </div>
+  }
+
+  render(): React.ReactNode {
+    const { selectedTable } = this.state;
+    const resultSets = this.getResultSets();
+
+    const resultSet = resultSets.find(resultSet => resultSet.schema.name == selectedTable);
+    const numberOfResults = resultSet && renderResultCountString(resultSet);
 
     return <div>
       <div className={tableSelectionHeaderClassName}>
-        <select value={selectedTable} onChange={this.onChange}>
+        <select value={selectedTable} onChange={this.onTableSelectionChange}>
           {
             resultSets.map(resultSet =>
               <option key={resultSet.schema.name} value={resultSet.schema.name}>
@@ -107,7 +136,8 @@ export class ResultTables
             )
           }
         </select>
-        {diagnosticsCheckBox}
+        {numberOfResults}
+        {selectedTable === ALERTS_TABLE_NAME ? this.alertTableExtras() : undefined}
         {
           this.props.isLoadingNewResults ?
             <span className={UPDATING_RESULTS_TEXT_CLASS_NAME}>Updating results…</span>
@@ -115,14 +145,11 @@ export class ResultTables
         }
       </div>
       {
-        resultSets.map(resultSet =>
+        resultSet &&
-          resultSet.schema.name === selectedTable ?
+        <ResultTable key={resultSet.schema.name} resultSet={resultSet}
-            <ResultTable key={resultSet.schema.name} resultSet={resultSet}
+          databaseUri={this.props.database.databaseUri}
-              databaseUri={this.props.database.databaseUri}
+          resultsPath={this.props.resultsPath}
-              resultsPath={this.props.resultsPath}
+          sortState={this.props.sortStates.get(resultSet.schema.name)} />
-              sortState={this.props.sortStates.get(resultSet.schema.name)} /> :
-            undefined
-        )
       }
     </div>;
   }
@@ -138,11 +165,9 @@ class ResultTable extends React.Component<ResultTableProps, {}> {
     const { resultSet } = this.props;
     switch (resultSet.t) {
       case 'RawResultSet': return <RawTable
-        resultSet={resultSet} databaseUri={this.props.databaseUri}
+        {...this.props} resultSet={resultSet} />;
-        resultsPath={this.props.resultsPath} sortState={this.props.sortState} />;
       case 'SarifResultSet': return <PathTable
-        resultSet={resultSet} databaseUri={this.props.databaseUri}
+        {...this.props} resultSet={resultSet} />;
-        resultsPath={this.props.resultsPath} />;
     }
   }
 }

extensions/ql-vscode/src/view/results.tsx

@@ -3,7 +3,8 @@ import * as Rdom from 'react-dom';
 import * as bqrs from 'semmle-bqrs';
 import { ElementBase, LocationValue, PrimitiveColumnValue, PrimitiveTypeKind, ResultSetSchema, tryGetResolvableLocation } from 'semmle-bqrs';
 import { assertNever } from '../helpers-pure';
-import { DatabaseInfo, FromResultsViewMsg, Interpretation, IntoResultsViewMsg, SortedResultSetInfo, SortState } from '../interface-types';
+import { DatabaseInfo, FromResultsViewMsg, Interpretation, IntoResultsViewMsg, SortedResultSetInfo, RawResultsSortState, NavigatePathMsg, QueryMetadata, ResultsPaths } from '../interface-types';
+import { EventHandlers as EventHandlerList } from './event-handler-list';
 import { ResultTables } from './result-tables';
 
 /**
@@ -23,8 +24,8 @@ declare const acquireVsCodeApi: () => VsCodeApi;
 export const vscode = acquireVsCodeApi();
 
 export interface ResultElement {
-  label: string,
+  label: string;
-  location?: LocationValue
+  location?: LocationValue;
 }
 
 export interface ResultUri {
@@ -36,7 +37,7 @@ export type ResultValue = ResultElement | ResultUri | string;
 export type ResultRow = ResultValue[];
 
 export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
-export type PathTableResultSet = { t: 'SarifResultSet', readonly schema: ResultSetSchema, name: string } & Interpretation;
+export type PathTableResultSet = { t: 'SarifResultSet'; readonly schema: ResultSetSchema; name: string } & Interpretation;
 
 export type ResultSet =
   | RawTableResultSet
@@ -57,12 +58,12 @@ async function* getChunkIterator(response: Response): AsyncIterableIterator<Uint
     if (done) {
       return;
     }
-    yield value;
+    yield value!;
   }
 }
 
 function translatePrimitiveValue(value: PrimitiveColumnValue, type: PrimitiveTypeKind):
-  ResultValue {
+ResultValue {
 
   switch (type) {
     case 'i':
@@ -126,7 +127,7 @@ async function parseResultSets(response: Response): Promise<readonly ResultSet[]
 
 interface ResultsInfo {
   resultsPath: string;
-  kind: string | undefined;
+  origResultsPaths: ResultsPaths;
   database: DatabaseInfo;
   interpretation: Interpretation | undefined;
   sortedResultsMap: Map<string, SortedResultSetInfo>;
@@ -134,11 +135,12 @@ interface ResultsInfo {
    * See {@link SetStateMsg.shouldKeepOldResultsWhileRendering}.
    */
   shouldKeepOldResultsWhileRendering: boolean;
+  metadata?: QueryMetadata;
 }
 
 interface Results {
   resultSets: readonly ResultSet[];
-  sortStates: Map<string, SortState>;
+  sortStates: Map<string, RawResultsSortState>;
   database: DatabaseInfo;
 }
 
@@ -156,6 +158,13 @@ interface ResultsViewState {
   isExpectingResultsUpdate: boolean;
 }
 
+export type NavigationEvent = NavigatePathMsg;
+
+/**
+ * Event handlers to be notified of navigation events coming from outside the webview.
+ */
+export const onNavigation = new EventHandlerList<NavigationEvent>();
+
 /**
  * A minimal state container for displaying results.
  */
@@ -178,11 +187,12 @@ class App extends React.Component<{}, ResultsViewState> {
       case 'setState':
         this.updateStateWithNewResultsInfo({
           resultsPath: msg.resultsPath,
-          kind: msg.kind,
+          origResultsPaths: msg.origResultsPaths,
           sortedResultsMap: new Map(Object.entries(msg.sortedResultsMap)),
           database: msg.database,
           interpretation: msg.interpretation,
-          shouldKeepOldResultsWhileRendering: msg.shouldKeepOldResultsWhileRendering
+          shouldKeepOldResultsWhileRendering: msg.shouldKeepOldResultsWhileRendering,
+          metadata: msg.metadata
         });
 
         this.loadResults();
@@ -192,6 +202,9 @@ class App extends React.Component<{}, ResultsViewState> {
           isExpectingResultsUpdate: true
         });
         break;
+      case 'navigatePath':
+        onNavigation.fire(msg);
+        break;
       default:
         assertNever(msg);
     }
@@ -199,7 +212,7 @@ class App extends React.Component<{}, ResultsViewState> {
 
   private updateStateWithNewResultsInfo(resultsInfo: ResultsInfo): void {
     this.setState(prevState => {
-      const stateWithDisplayedResults = (displayedResults: ResultsState) => ({
+      const stateWithDisplayedResults = (displayedResults: ResultsState): ResultsViewState => ({
         displayedResults,
         isExpectingResultsUpdate: prevState.isExpectingResultsUpdate,
         nextResultsInfo: resultsInfo
@@ -232,7 +245,7 @@ class App extends React.Component<{}, ResultsViewState> {
     }
 
     let results: Results | null = null;
-    let statusText: string = '';
+    let statusText = '';
     try {
       results = {
         resultSets: await this.getResultSets(resultsInfo),
@@ -285,21 +298,23 @@ class App extends React.Component<{}, ResultsViewState> {
     }));
   }
 
-  private getSortStates(resultsInfo: ResultsInfo): Map<string, SortState> {
+  private getSortStates(resultsInfo: ResultsInfo): Map<string, RawResultsSortState> {
     const entries = Array.from(resultsInfo.sortedResultsMap.entries());
     return new Map(entries.map(([key, sortedResultSetInfo]) =>
       [key, sortedResultSetInfo.sortState]));
   }
 
-  render() {
+  render(): JSX.Element {
     const displayedResults = this.state.displayedResults;
-    if (displayedResults.results !== null) {
+    if (displayedResults.results !== null && displayedResults.resultsInfo !== null) {
       return <ResultTables rawResultSets={displayedResults.results.resultSets}
         interpretation={displayedResults.resultsInfo ? displayedResults.resultsInfo.interpretation : undefined}
         database={displayedResults.results.database}
-        resultsPath={displayedResults.resultsInfo ? displayedResults.resultsInfo.resultsPath : undefined}
+        origResultsPaths={displayedResults.resultsInfo.origResultsPaths}
-        kind={displayedResults.resultsInfo ? displayedResults.resultsInfo.kind : undefined}
+        resultsPath={displayedResults.resultsInfo.resultsPath}
+        metadata={displayedResults.resultsInfo ? displayedResults.resultsInfo.metadata : undefined}
         sortStates={displayedResults.results.sortStates}
+        interpretedSortState={displayedResults.resultsInfo.interpretation?.sortState}
         isLoadingNewResults={this.state.isExpectingResultsUpdate || this.state.nextResultsInfo !== null} />;
     }
     else {
@@ -307,12 +322,12 @@ class App extends React.Component<{}, ResultsViewState> {
     }
   }
 
-  componentDidMount() {
+  componentDidMount(): void {
     this.vscodeMessageHandler = evt => this.handleMessage(evt.data as IntoResultsViewMsg);
     window.addEventListener('message', this.vscodeMessageHandler);
   }
 
-  componentWillUnmount() {
+  componentWillUnmount(): void {
     if (this.vscodeMessageHandler) {
       window.removeEventListener('message', this.vscodeMessageHandler);
     }
@@ -326,4 +341,4 @@ Rdom.render(
   document.getElementById('root')
 );
 
-vscode.postMessage({ t: "resultViewLoaded" })
+vscode.postMessage({ t: "resultViewLoaded" })

extensions/ql-vscode/src/view/resultsView.css

@@ -13,12 +13,16 @@
   border: 0;
 }
 
-.vscode-codeql__result-table-toggle-diagnostics {
+.vscode-codeql__result-table-alert-extras {
   display: inline-block;
   text-align: left;
   margin-left: auto;
 }
 
+.vscode-codeql__result-table-toggle-diagnostics  {
+  display: inline-block;
+}
+
 /* Keep the checkbox and its label in horizontal alignment. */
 .vscode-codeql__result-table-toggle-diagnostics label,
 .vscode-codeql__result-table-toggle-diagnostics input {
@@ -26,7 +30,7 @@
   vertical-align: middle;
 }
 .vscode-codeql__result-table-toggle-diagnostics input {
-  margin: 3px 3px 1px 3px;
+  margin: 3px 3px 1px 13px;
 }
 
 
@@ -41,6 +45,13 @@
   opacity: 0.6;
 }
 
+.vscode-codeql__result-table .sort-asc,
+.vscode-codeql__result-table .sort-desc,
+.vscode-codeql__result-table .sort-none {
+   cursor: pointer;
+   user-select: none;
+}
+
 .vscode-codeql__result-table .sort-none::after {
   /* Want to take up the same space as the other sort directions */
   content: " ▲";
@@ -87,6 +98,10 @@ select {
   background-color: var(--vscode-textBlockQuote-background);
 }
 
+.vscode-codeql__result-table-row--selected {
+  background-color: var(--vscode-editor-findMatchBackground);
+}
+
 td.vscode-codeql__icon-cell {
   text-align: center;
   position: relative;
@@ -104,8 +119,14 @@ td.vscode-codeql__path-index-cell {
   text-align: right;
 }
 
-td.vscode-codeql__location-cell {
+/* Both of these are !important to override the
-  text-align: right;
+   .vscode-codeql__result-table th { text-align: center } above */
+.vscode-codeql__alert-message-cell {
+  text-align: left !important;
+}
+
+.vscode-codeql__location-cell {
+  text-align: right !important;
 }
 
 .vscode-codeql__vertical-rule {
@@ -130,3 +151,7 @@ td.vscode-codeql__location-cell {
 .octicon-light {
   opacity: 0.6;
 }
+
+.number-of-results {
+  padding-left: 3em;
+}

extensions/ql-vscode/src/vscode-tests/.eslintrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  env: {
+    mocha: true
+  }
+}

extensions/ql-vscode/src/vscode-tests/index-template.ts

@@ -58,4 +58,4 @@ export function runTestsInDirectory(testsRoot: string): Promise<void> {
       }
     });
   });
-}
+}

extensions/ql-vscode/src/vscode-tests/minimal-workspace/activation.test.ts

@@ -1,6 +1,12 @@
 import * as assert from 'assert';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+import 'mocha';
 import * as path from 'path';
 import * as vscode from 'vscode';
+import * as determiningSelectedQueryTest from './determining-selected-query-test';
+
+chai.use(chaiAsPromised);
 
 describe('launching with a minimal workspace', async () => {
   const ext = vscode.extensions.getExtension('GitHub.vscode-codeql');
@@ -10,7 +16,7 @@ describe('launching with a minimal workspace', async () => {
   it('should not activate the extension at first', () => {
     assert(ext!.isActive === false);
   });
-  it('should activate the extension when a .ql file is opened', async function () {
+  it('should activate the extension when a .ql file is opened', async function() {
     const folders = vscode.workspace.workspaceFolders;
     assert(folders && folders.length === 1);
     const folderPath = folders![0].uri.fsPath;
@@ -23,4 +29,6 @@ describe('launching with a minimal workspace', async () => {
       assert(ext!.isActive);
     }, 1000);
   });
-});
+});
+
+determiningSelectedQueryTest.run();

extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts

@@ -0,0 +1,44 @@
+import { expect } from 'chai';
+import * as path from 'path';
+import * as vscode from 'vscode';
+import { Uri } from 'vscode';
+import { determineSelectedQuery } from '../../run-queries';
+
+async function showQlDocument(name: string): Promise<vscode.TextDocument> {
+  const folderPath = vscode.workspace.workspaceFolders![0].uri.fsPath;
+  const documentPath = path.resolve(folderPath, name);
+  const document = await vscode.workspace.openTextDocument(documentPath);
+  await vscode.window.showTextDocument(document!);
+  return document;
+}
+
+export function run() {
+  describe('Determining selected query', async () => {
+    it('should allow ql files to be queried', async () => {
+      const q = await determineSelectedQuery(Uri.parse('file:///tmp/queryname.ql'), false);
+      expect(q.queryPath).to.equal(path.join('/', 'tmp', 'queryname.ql'));
+      expect(q.quickEvalPosition).to.equal(undefined);
+    });
+
+    it('should allow ql files to be quick-evaled', async () => {
+      const doc = await showQlDocument('query.ql');
+      const q = await determineSelectedQuery(doc.uri, true);
+      expect(q.queryPath).to.satisfy((p: string) => p.endsWith(path.join('ql-vscode', 'test', 'data', 'query.ql')));
+    });
+
+    it('should allow qll files to be quick-evaled', async () => {
+      const doc = await showQlDocument('library.qll');
+      const q = await determineSelectedQuery(doc.uri, true);
+      expect(q.queryPath).to.satisfy((p: string) => p.endsWith(path.join('ql-vscode', 'test', 'data', 'library.qll')));
+    });
+
+    it('should reject non-ql files when running a query', async () => {
+      await expect(determineSelectedQuery(Uri.parse('file:///tmp/queryname.txt'), false)).to.be.rejectedWith(Error, 'The selected resource is not a CodeQL query file');
+      await expect(determineSelectedQuery(Uri.parse('file:///tmp/queryname.qll'), false)).to.be.rejectedWith(Error, 'The selected resource is not a CodeQL query file');
+    });
+
+    it('should reject non-ql[l] files when running a quick eval', async () => {
+      await expect(determineSelectedQuery(Uri.parse('file:///tmp/queryname.txt'), true)).to.be.rejectedWith(Error, 'The selected resource is not a CodeQL file');
+    });
+  });
+}

extensions/ql-vscode/src/vscode-tests/minimal-workspace/index.ts

@@ -1,4 +1,4 @@
 import { runTestsInDirectory } from '../index-template';
 export function run(): Promise<void> {
   return runTestsInDirectory(__dirname);
-}
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/activation.test.ts

@@ -10,4 +10,4 @@ describe('launching with no specified workspace', () => {
   it('should not activate the extension at first', () => {
     assert(ext!.isActive === false);
   });
-});
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/archive-filesystem-provider.test.ts

@@ -14,8 +14,8 @@ describe("archive filesystem provider", () => {
   });
 });
 
-describe('source archive uri encoding', function () {
+describe('source archive uri encoding', function() {
-  const testCases: { name: string, input: ZipFileReference }[] = [
+  const testCases: { name: string; input: ZipFileReference }[] = [
     {
       name: 'mixed case and unicode',
       input: { sourceArchiveZipPath: "/I-\u2665-codeql.zip", pathWithinSourceArchive: "/foo/bar" }
@@ -30,7 +30,7 @@ describe('source archive uri encoding', function () {
     }
   ];
   for (const testCase of testCases) {
-    it(`should work round trip with ${testCase.name}`, function () {
+    it(`should work round trip with ${testCase.name}`, function() {
       const output = decodeSourceArchiveUri(encodeSourceArchiveUri(testCase.input));
       expect(output).to.eql(testCase.input);
     });

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -1,8 +1,18 @@
-import { expect } from "chai";
+import * as chai from "chai";
+import * as path from "path";
 import * as fetch from "node-fetch";
+import 'chai/register-should';
+import * as sinonChai from 'sinon-chai';
+import * as sinon from 'sinon';
+import * as pq from "proxyquire";
 import "mocha";
+
 import { Version } from "../../cli-version";
-import { GithubRelease, GithubReleaseAsset, ReleasesApiConsumer, versionCompare } from "../../distribution"
+import { GithubRelease, GithubReleaseAsset, ReleasesApiConsumer, versionCompare } from "../../distribution";
+
+const proxyquire = pq.noPreserveCache();
+chai.use(sinonChai);
+const expect = chai.expect;
 
 describe("Releases API consumer", () => {
   const owner = "someowner";
@@ -48,7 +58,7 @@ describe("Releases API consumer", () => {
 
   it("picking latest release: is based on version", async () => {
     class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string, additionalHeaders: { [key: string]: string } = {}): Promise<fetch.Response> {
+      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
         if (apiPath === `/repos/${owner}/${repo}/releases`) {
           return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
         }
@@ -64,7 +74,7 @@ describe("Releases API consumer", () => {
 
   it("picking latest release: obeys version constraints", async () => {
     class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string, additionalHeaders: { [key: string]: string } = {}): Promise<fetch.Response> {
+      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
         if (apiPath === `/repos/${owner}/${repo}/releases`) {
           return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
         }
@@ -83,7 +93,7 @@ describe("Releases API consumer", () => {
 
   it("picking latest release: includes prereleases when option set", async () => {
     class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string, additionalHeaders: { [key: string]: string } = {}): Promise<fetch.Response> {
+      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
         if (apiPath === `/repos/${owner}/${repo}/releases`) {
           return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
         }
@@ -112,7 +122,7 @@ describe("Releases API consumer", () => {
     ];
 
     class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string, additionalHeaders: { [key: string]: string } = {}): Promise<fetch.Response> {
+      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
         if (apiPath === `/repos/${owner}/${repo}/releases`) {
           const responseBody: GithubRelease[] = [{
             "assets": expectedAssets,
@@ -151,8 +161,8 @@ describe("Release version ordering", () => {
       patchVersion,
       prereleaseVersion,
       rawString: `${majorVersion}.${minorVersion}.${patchVersion}` +
-      prereleaseVersion ? `-${prereleaseVersion}` : "" +
+        prereleaseVersion ? `-${prereleaseVersion}` : "" +
-      buildMetadata ? `+${buildMetadata}` : ""
+          buildMetadata ? `+${buildMetadata}` : ""
     };
   }
 
@@ -176,3 +186,104 @@ describe("Release version ordering", () => {
     expect(versionCompare(createVersion(2, 1, 0, "alpha.1", "abcdef0"), createVersion(2, 1, 0, "alpha.1", "bcdef01"))).to.equal(0);
   });
 });
+
+describe('Launcher path', () => {
+  let sandbox: sinon.SinonSandbox;
+  let warnSpy: sinon.SinonSpy;
+  let logSpy: sinon.SinonSpy;
+  let fsSpy: sinon.SinonSpy;
+  let platformSpy: sinon.SinonSpy;
+
+  let getExecutableFromDirectory: Function;
+
+  let launcherThatExists = '';
+
+  beforeEach(() => {
+    getExecutableFromDirectory = createModule().getExecutableFromDirectory;
+  });
+
+  beforeEach(() => {
+    sandbox.restore();
+  });
+
+  it('should not warn with proper launcher name', async () => {
+    launcherThatExists = 'codeql.exe';
+    const result = await getExecutableFromDirectory('abc');
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.exe`);
+
+    // correct launcher has been found, so alternate one not looked for
+    expect(fsSpy).not.to.have.been.calledWith(`abc${path.sep}codeql.cmd`);
+
+    // no warning message
+    expect(warnSpy).not.to.have.been.calledWith(sinon.match.string);
+    // No log message
+    expect(logSpy).not.to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(`abc${path.sep}codeql.exe`);
+  });
+
+  it('should warn when using a hard-coded deprecated launcher name', async () => {
+    launcherThatExists = 'codeql.cmd';
+    path.sep;
+    const result = await getExecutableFromDirectory('abc');
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.exe`);
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.cmd`);
+
+    // Should have opened a warning message
+    expect(warnSpy).to.have.been.calledWith(sinon.match.string);
+    // No log message
+    expect(logSpy).not.to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(`abc${path.sep}codeql.cmd`);
+  });
+
+  it('should avoid warn when no launcher is found', async () => {
+    launcherThatExists = 'xxx';
+    const result = await getExecutableFromDirectory('abc', false);
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.exe`);
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.cmd`);
+
+    // no warning message
+    expect(warnSpy).not.to.have.been.calledWith(sinon.match.string);
+    // log message sent out
+    expect(logSpy).not.to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(undefined);
+  });
+
+  it('should warn when no launcher is found', async () => {
+    launcherThatExists = 'xxx';
+    const result = await getExecutableFromDirectory('abc', true);
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.exe`);
+    expect(fsSpy).to.have.been.calledWith(`abc${path.sep}codeql.cmd`);
+
+    // no warning message
+    expect(warnSpy).not.to.have.been.calledWith(sinon.match.string);
+    // log message sent out
+    expect(logSpy).to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(undefined);
+  });
+
+  function createModule() {
+    sandbox = sinon.createSandbox();
+    warnSpy = sandbox.spy();
+    logSpy = sandbox.spy();
+    // pretend that only the .cmd file exists
+    fsSpy = sandbox.stub().callsFake(arg => arg.endsWith(launcherThatExists) ? true : false);
+    platformSpy = sandbox.stub().returns('win32');
+
+    return proxyquire('../../distribution', {
+      './helpers': {
+        showAndLogWarningMessage: warnSpy
+      },
+      './logging': {
+        'logger': {
+          log: logSpy
+        }
+      },
+      'fs-extra': {
+        pathExists: fsSpy
+      },
+      os: {
+        platform: platformSpy
+      }
+    });
+  }
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -0,0 +1,126 @@
+import { expect } from "chai";
+import "mocha";
+import { ExtensionContext, Memento } from "vscode";
+import { InvocationRateLimiter } from "../../helpers";
+
+describe("Invocation rate limiter", () => {
+  // 1 January 2020
+  let currentUnixTime = 1577836800;
+
+  function createDate(dateString?: string): Date {
+    if (dateString) {
+      return new Date(dateString);
+    }
+    const numMillisecondsPerSecond = 1000;
+    return new Date(currentUnixTime * numMillisecondsPerSecond);
+  }
+
+  function createInvocationRateLimiter<T>(funcIdentifier: string, func: () => Promise<T>): InvocationRateLimiter<T> {
+    return new InvocationRateLimiter(new MockExtensionContext(), funcIdentifier, func, s => createDate(s));
+  }
+
+  it("initially invokes function", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncCalled++;
+    });
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    expect(numTimesFuncCalled).to.equal(1);
+  });
+
+  it("doesn't invoke function again if no time has passed", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncCalled++;
+    });
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    expect(numTimesFuncCalled).to.equal(1);
+  });
+
+  it("doesn't invoke function again if requested time since last invocation hasn't passed", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncCalled++;
+    });
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+    currentUnixTime += 1;
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
+    expect(numTimesFuncCalled).to.equal(1);
+  });
+
+  it("invokes function again immediately if requested time since last invocation is 0 seconds", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncCalled++;
+    });
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
+    expect(numTimesFuncCalled).to.equal(2);
+  });
+
+  it("invokes function again after requested time since last invocation has elapsed", async () => {
+    let numTimesFuncCalled = 0;
+    const invocationRateLimiter = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncCalled++;
+    });
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
+    currentUnixTime += 1;
+    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
+    expect(numTimesFuncCalled).to.equal(2);
+  });
+
+  it("invokes functions with different rate limiters", async () => {
+    let numTimesFuncACalled = 0;
+    const invocationRateLimiterA = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncACalled++;
+    });
+    let numTimesFuncBCalled = 0;
+    const invocationRateLimiterB = createInvocationRateLimiter("funcid", async () => {
+      numTimesFuncBCalled++;
+    });
+    await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
+    await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
+    expect(numTimesFuncACalled).to.equal(1);
+    expect(numTimesFuncBCalled).to.equal(1);
+  });
+});
+
+class MockExtensionContext implements ExtensionContext {
+  subscriptions: { dispose(): unknown }[] = [];
+  workspaceState: Memento = new MockMemento();
+  globalState: Memento = new MockMemento();
+  extensionPath = "";
+  asAbsolutePath(_relativePath: string): string {
+    throw new Error("Method not implemented.");
+  }
+  storagePath = "";
+  globalStoragePath = "";
+  logPath = "";
+}
+
+class MockMemento implements Memento {
+  map = new Map<any, any>();
+
+  /**
+   * Return a value.
+   *
+   * @param key A string.
+   * @param defaultValue A value that should be returned when there is no
+   * value (`undefined`) with the given key.
+   * @return The stored value or the defaultValue.
+   */
+  get<T>(key: string, defaultValue?: T): T {
+    return this.map.has(key) ? this.map.get(key) : defaultValue;
+  }
+
+  /**
+   * Store a value. The value must be JSON-stringifyable.
+   *
+   * @param key A string.
+   * @param value A value. MUST not contain cyclic references.
+   */
+  async update(key: string, value: any): Promise<void> {
+    this.map.set(key, value);
+  }
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/index.ts

@@ -1,4 +1,4 @@
 import { runTestsInDirectory } from '../index-template';
 export function run(): Promise<void> {
   return runTestsInDirectory(__dirname);
-}
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/sarifMessageParsingTest.test.ts

@@ -0,0 +1,39 @@
+import 'mocha';
+import { expect } from "chai";
+
+import { parseSarifPlainTextMessage } from '../../sarif-utils';
+
+
+describe('parsing sarif', () => {
+  it('should be able to parse a simple message from the spec', async function() {
+    const message = "Tainted data was used. The data came from [here](3)."
+    const results = parseSarifPlainTextMessage(message);
+    expect(results).to.deep.equal([
+      "Tainted data was used. The data came from ",
+      { dest: 3, text: "here" }, "."
+    ]);
+  });
+
+  it('should be able to parse a complex message from the spec', async function() {
+    const message = "Prohibited term used in [para\\[0\\]\\\\spans\\[2\\]](1)."
+    const results = parseSarifPlainTextMessage(message);
+    expect(results).to.deep.equal([
+      "Prohibited term used in ",
+      { dest: 1, text: "para[0]\\spans[2]" }, "."
+    ]);
+  });
+  it('should be able to parse a broken complex message from the spec', async function() {
+    const message = "Prohibited term used in [para\\[0\\]\\\\spans\\[2\\](1)."
+    const results = parseSarifPlainTextMessage(message);
+    expect(results).to.deep.equal([
+      "Prohibited term used in [para[0]\\spans[2](1)."
+    ]);
+  });
+  it('should be able to parse a message with extra escaping the spec', async function() {
+    const message = "Tainted data was used. The data came from \\[here](3)."
+    const results = parseSarifPlainTextMessage(message);
+    expect(results).to.deep.equal([
+      "Tainted data was used. The data came from [here](3)."
+    ]);
+  });
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/webview-uri.test.ts

@@ -1,11 +1,14 @@
 import { expect } from "chai";
+import * as path from "path";
 import * as tmp from "tmp";
 import { window, ViewColumn, Uri } from "vscode";
 import { fileUriToWebviewUri, webviewUriToFileUri } from '../../interface';
 
-describe('webview uri conversion', function () {
+describe('webview uri conversion', function() {
-  it('should correctly round trip from filesystem to webview and back', function () {
+  const fileSuffix = '.bqrs';
-    const tmpFile = tmp.fileSync({ prefix: 'uri_test_', postfix: '.bqrs', keep: false });
+
+  function setupWebview(filePrefix: string) {
+    const tmpFile = tmp.fileSync({ prefix: `uri_test_${filePrefix}_`, postfix: fileSuffix, keep: false });
     const fileUriOnDisk = Uri.file(tmpFile.name);
     const panel = window.createWebviewPanel(
       'test panel',
@@ -18,7 +21,7 @@ describe('webview uri conversion', function () {
         ]
       }
     );
-    after(function () {
+    after(function() {
       panel.dispose();
       tmpFile.removeCallback();
     });
@@ -26,9 +29,23 @@ describe('webview uri conversion', function () {
     // CSP allowing nothing, to prevent warnings.
     const html = `<html><head><meta http-equiv="Content-Security-Policy" content="default-src 'none';"></head></html>`;
     panel.webview.html = html;
+    return {
+      fileUriOnDisk,
+      panel
+    }
+  }
 
+  it('should correctly round trip from filesystem to webview and back', function() {
+    const { fileUriOnDisk, panel } = setupWebview('');
     const webviewUri = fileUriToWebviewUri(panel, fileUriOnDisk);
     const reconstructedFileUri = webviewUriToFileUri(webviewUri);
     expect(reconstructedFileUri.toString(true)).to.equal(fileUriOnDisk.toString(true));
   });
+
+  it("does not double-encode # in URIs", function() {
+    const { fileUriOnDisk, panel } = setupWebview('#');
+    const webviewUri = fileUriToWebviewUri(panel, fileUriOnDisk);
+    const parsedUri = Uri.parse(webviewUri);
+    expect(path.basename(parsedUri.path, fileSuffix)).to.equal(path.basename(fileUriOnDisk.path, fileSuffix));
+  });
 });

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -1,6 +1,41 @@
 import * as path from 'path';
 import { runTests } from 'vscode-test';
 
+// A subset of the fields in TestOptions from vscode-test, which we
+// would simply use instead, but for the fact that it doesn't export
+// it.
+type Suite = {
+  extensionDevelopmentPath: string;
+  extensionTestsPath: string;
+  launchArgs: string[];
+};
+
+/**
+ * Run an integration test suite `suite` at most `tries` times, or
+ * until it succeeds, whichever comes first.
+ *
+ * TODO: Presently there is no way to distinguish a legitimately
+ * failed test run from the test runner being terminated by a signal.
+ * If in the future there arises a way to distinguish these cases
+ * (e.g. https://github.com/microsoft/vscode-test/pull/56) only retry
+ * in the terminated-by-signal case.
+ */
+async function runTestsWithRetry(suite: Suite, tries: number): Promise<void> {
+  for (let t = 0; t < tries; t++) {
+    try {
+      // Download and unzip VS Code if necessary, and run the integration test suite.
+      await runTests(suite);
+      return;
+    } catch (err) {
+      console.error(`Exception raised while running tests: ${err}`);
+      if (t < tries - 1)
+        console.log('Retrying...');
+    }
+  }
+  console.error(`Tried running suite ${tries} time(s), still failed, giving up.`);
+  process.exit(1);
+}
+
 /**
  * Integration test runner. Launches the VSCode Extension Development Host with this extension installed.
  * See https://github.com/microsoft/vscode-test/blob/master/sample/test/runTest.ts
@@ -32,11 +67,10 @@ async function main() {
     ];
 
     for (const integrationTestSuite of integrationTestSuites) {
-      // Download and unzip VS Code if necessary, and run the integration test suite.
+      await runTestsWithRetry(integrationTestSuite, 3);
-      await runTests(integrationTestSuite);
     }
   } catch (err) {
-    console.error('Failed to run tests');
+    console.error(`Unexpected exception while running tests: ${err}`);
     process.exit(1);
   }
 }

extensions/ql-vscode/test/.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+  env: {
+    mocha: true
+  },
+  parserOptions: {
+    project: 'tsconfig.json',
+  },
+}

extensions/ql-vscode/test/data/library.qll

@@ -0,0 +1,3 @@
+predicate foo() {
+ 1 == 1
+}

extensions/ql-vscode/test/pure-tests/location.test.ts

@@ -3,7 +3,6 @@ import 'mocha';
 import { LocationStyle, StringLocation, tryGetWholeFileLocation } from 'semmle-bqrs';
 
 describe('processing string locations', function () {
-  
   it('should detect Windows whole-file locations', function () {
     const loc: StringLocation = {
       t: LocationStyle.String,

extensions/ql-vscode/test/pure-tests/logging.test.ts

@@ -0,0 +1,154 @@
+import 'chai/register-should';
+import * as chai from 'chai';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import * as tmp from 'tmp';
+import 'mocha';
+import * as sinonChai from 'sinon-chai';
+import * as sinon from 'sinon';
+import * as pq from 'proxyquire';
+
+const proxyquire = pq.noPreserveCache().noCallThru();
+chai.use(sinonChai);
+const expect = chai.expect;
+
+describe('OutputChannelLogger tests', () => {
+  let OutputChannelLogger;
+  const tempFolders: Record<string, tmp.DirResult> = {};
+  let logger: any;
+  let mockOutputChannel: Record<string, sinon.SinonStub>;
+
+  beforeEach(async () => {
+    OutputChannelLogger = createModule().OutputChannelLogger;
+    tempFolders.globalStoragePath = tmp.dirSync({ prefix: 'logging-tests-global' });
+    tempFolders.storagePath = tmp.dirSync({ prefix: 'logging-tests-workspace' });
+    logger = new OutputChannelLogger('test-logger');
+  });
+
+  afterEach(() => {
+    tempFolders.globalStoragePath.removeCallback();
+    tempFolders.storagePath.removeCallback();
+  });
+
+  it('should log to the output channel', async () => {
+    await logger.log('xxx');
+    expect(mockOutputChannel.appendLine).to.have.been.calledWith('xxx');
+    expect(mockOutputChannel.append).not.to.have.been.calledWith('xxx');
+
+    await logger.log('yyy', { trailingNewline: false });
+    expect(mockOutputChannel.appendLine).not.to.have.been.calledWith('yyy');
+    expect(mockOutputChannel.append).to.have.been.calledWith('yyy');
+
+    // additionalLogLocation ignored since not initialized
+    await logger.log('zzz', { additionalLogLocation: 'hucairz' });
+
+    // should not have created any side logs
+    expect(fs.readdirSync(tempFolders.globalStoragePath.name).length).to.equal(0);
+    expect(fs.readdirSync(tempFolders.storagePath.name).length).to.equal(0);
+  });
+
+  it('should create a side log in the workspace area', async () => {
+    await sideLogTest('storagePath', 'globalStoragePath');
+  });
+
+  it('should create a side log in the global area', async () => {
+    await sideLogTest('globalStoragePath', 'storagePath');
+  });
+
+  async function sideLogTest(expectedArea: string, otherArea: string): Promise<void> {
+    logger.init({
+      [expectedArea]: tempFolders[expectedArea].name,
+      [otherArea]: undefined
+    });
+
+    await logger.log('xxx', { additionalLogLocation: 'first' });
+    await logger.log('yyy', { additionalLogLocation: 'second' });
+    await logger.log('zzz', { additionalLogLocation: 'first', trailingNewline: false });
+    await logger.log('aaa');
+
+    // expect 2 side logs
+    const testLoggerFolder = path.join(tempFolders[expectedArea].name, 'test-logger');
+    expect(fs.readdirSync(testLoggerFolder).length).to.equal(2);
+    expect(fs.readdirSync(tempFolders[otherArea].name).length).to.equal(0);
+
+    // contents
+    expect(fs.readFileSync(path.join(testLoggerFolder, 'first'), 'utf8')).to.equal('xxx\nzzz');
+    expect(fs.readFileSync(path.join(testLoggerFolder, 'second'), 'utf8')).to.equal('yyy\n');
+  }
+
+  it('should delete side logs on dispose', async () => {
+    logger.init({
+      storagePath: tempFolders.storagePath.name
+    });
+    await logger.log('xxx', { additionalLogLocation: 'first' });
+    await logger.log('yyy', { additionalLogLocation: 'second' });
+
+    const testLoggerFolder = path.join(tempFolders.storagePath.name, 'test-logger');
+    expect(fs.readdirSync(testLoggerFolder).length).to.equal(2);
+
+    await logger.dispose();
+    // need to wait for disposable-object to dispose
+    await waitABit();
+    expect(fs.readdirSync(testLoggerFolder).length).to.equal(0);
+    expect(mockOutputChannel.dispose).to.have.been.calledWith();
+  });
+
+  it('should remove an additional log location', async () => {
+    logger.init({
+      storagePath: tempFolders.storagePath.name
+    });
+    await logger.log('xxx', { additionalLogLocation: 'first' });
+    await logger.log('yyy', { additionalLogLocation: 'second' });
+
+    const testLoggerFolder = path.join(tempFolders.storagePath.name, 'test-logger');
+    expect(fs.readdirSync(testLoggerFolder).length).to.equal(2);
+
+    await logger.removeAdditionalLogLocation('first');
+    // need to wait for disposable-object to dispose
+    await waitABit();
+    expect(fs.readdirSync(testLoggerFolder).length).to.equal(1);
+    expect(fs.readFileSync(path.join(testLoggerFolder, 'second'), 'utf8')).to.equal('yyy\n');
+  });
+
+  it('should delete an existing folder on init', async () => {
+    fs.createFileSync(path.join(tempFolders.storagePath.name, 'test-logger', 'xxx'));
+    logger.init({
+      storagePath: tempFolders.storagePath.name
+    });
+    // should be empty dir
+
+    const testLoggerFolder = path.join(tempFolders.storagePath.name, 'test-logger');
+    expect(fs.readdirSync(testLoggerFolder).length).to.equal(1);
+  });
+
+  it('should show the output channel', () => {
+    logger.show(true);
+    expect(mockOutputChannel.show).to.have.been.calledWith(true);
+  });
+
+  function createModule(): any {
+    mockOutputChannel = {
+      append: sinon.stub(),
+      appendLine: sinon.stub(),
+      show: sinon.stub(),
+      dispose: sinon.stub(),
+    };
+
+    return proxyquire('../../src/logging', {
+      vscode: {
+        window: {
+          createOutputChannel: () => mockOutputChannel
+        },
+        Disposable: function() {
+          /**/
+        },
+        '@noCallThru': true,
+        '@global': true
+      }
+    });
+  }
+
+  function  waitABit(ms = 50): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+});

extensions/ql-vscode/test/pure-tests/query-test.ts

@@ -10,7 +10,7 @@ import { CancellationTokenSource } from 'vscode-jsonrpc';
 import * as messages from '../../src/messages';
 import * as qsClient from '../../src/queryserver-client';
 import * as cli from '../../src/cli';
-import { ProgressReporter } from '../../src/logging';
+import { ProgressReporter, Logger } from '../../src/logging';
 
 
 declare module "url" {
@@ -31,31 +31,31 @@ class Checkpoint<T> {
   private promise: Promise<T>;
 
   constructor() {
-    this.res = () => { };
+    this.res = () => { /**/ };
-    this.rej = () => { };
+    this.rej = () => { /**/ };
     this.promise = new Promise((res, rej) => { this.res = res; this.rej = rej; })
   }
 
-  async done() {
+  async done(): Promise<T> {
     return this.promise;
   }
 
-  async resolve() {
+  async resolve(): Promise<void> {
-    (this.res)();
+    await (this.res)();
   }
 
-  async reject(e: Error) {
+  async reject(e: Error): Promise<void> {
-    (this.rej)(e);
+    await (this.rej)(e);
   }
 }
 
 type ResultSets = {
-  [name: string]: bqrs.ColumnValue[][]
+  [name: string]: bqrs.ColumnValue[][];
 }
 
 type QueryTestCase = {
-  queryPath: string,
+  queryPath: string;
-  expectedResultSets: ResultSets
+  expectedResultSets: ResultSets;
 }
 
 // Test cases: queries to run and their expected results.
@@ -75,17 +75,22 @@ const queryTestCases: QueryTestCase[] = [
   }
 ];
 
-describe('using the query server', function () {
+describe('using the query server', function() {
-  before(function () {
+  before(function() {
     if (process.env["CODEQL_PATH"] === undefined) {
       console.log('The environment variable CODEQL_PATH is not set. The query server tests, which require the CodeQL CLI, will be skipped.');
       this.skip();
     }
   });
 
+  // Note this does not work with arrow functions as the test case bodies:
+  // ensure they are all written with standard anonymous functions.
+  this.timeout(10000);
+
   const codeQlPath = process.env["CODEQL_PATH"]!;
   let qs: qsClient.QueryServerClient;
   let cliServer: cli.CodeQLCliServer;
+  const queryServerStarted = new Checkpoint<void>();
   after(() => {
     if (qs) {
       qs.dispose();
@@ -94,13 +99,16 @@ describe('using the query server', function () {
       cliServer.dispose();
     }
   });
-  it('should be able to start the query server', async function () {
+
+  it('should be able to start the query server', async function() {
     const consoleProgressReporter: ProgressReporter = {
-      report: (v: {message: string}) => console.log(`progress reporter says ${v.message}`)
+      report: (v: { message: string }) => console.log(`progress reporter says ${v.message}`)
     };
-    const logger = {
+    const logger: Logger = {
-      log: (s: string) => console.log('logger says', s),
+      log: async (s: string) => console.log('logger says', s),
-      logWithoutTrailingNewline: (s: string) => { }
+      show: () => { /**/ },
+      removeAdditionalLogLocation: async () => { /**/ },
+      getBaseLocation: () => ''
     };
     cliServer = new cli.CodeQLCliServer({
       async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@@ -122,18 +130,17 @@ describe('using the query server', function () {
       task => task(consoleProgressReporter, token)
     );
     await qs.startQueryServer();
+    queryServerStarted.resolve();
   });
 
-  // Note this does not work with arrow functions as the test case bodies:
-  // ensure they are all written with standard anonymous functions.
-  this.timeout(5000);
-
   for (const queryTestCase of queryTestCases) {
     const queryName = path.basename(queryTestCase.queryPath);
     const compilationSucceeded = new Checkpoint<void>();
     const evaluationSucceeded = new Checkpoint<void>();
+    const parsedResults = new Checkpoint<void>();
 
-    it(`should be able to compile query ${queryName}`, async function () {
+    it(`should be able to compile query ${queryName}`, async function() {
+      await queryServerStarted.done();
       expect(fs.existsSync(queryTestCase.queryPath)).to.be.true;
       try {
         const qlProgram: messages.QlProgram = {
@@ -155,7 +162,7 @@ describe('using the query server', function () {
           resultPath: COMPILED_QUERY_PATH,
           target: { query: {} }
         };
-        const result = await qs.sendRequest(messages.compileQuery, params, token, () => { });
+        const result = await qs.sendRequest(messages.compileQuery, params, token, () => { /**/ });
         expect(result.messages!.length).to.equal(0);
         compilationSucceeded.resolve();
       }
@@ -164,10 +171,10 @@ describe('using the query server', function () {
       }
     });
 
-    it(`should be able to run query ${queryName}`, async function () {
+    it(`should be able to run query ${queryName}`, async function() {
       try {
         await compilationSucceeded.done();
-        const callbackId = qs.registerCallback(res => {
+        const callbackId = qs.registerCallback(_res => {
           evaluationSucceeded.resolve();
         });
         const queryToRun: messages.QueryToRun = {
@@ -188,7 +195,7 @@ describe('using the query server', function () {
           stopOnError: false,
           useSequenceHint: false
         };
-        await qs.sendRequest(messages.runQueries, params, token, () => { });
+        await qs.sendRequest(messages.runQueries, params, token, () => { /**/ });
       }
       catch (e) {
         evaluationSucceeded.reject(e);
@@ -196,7 +203,7 @@ describe('using the query server', function () {
     });
 
     const actualResultSets: ResultSets = {};
-    it(`should be able to parse results of query ${queryName}`, async function () {
+    it(`should be able to parse results of query ${queryName}`, async function() {
       let fileReader: FileReader | undefined;
       try {
         await evaluationSucceeded.done();
@@ -209,6 +216,7 @@ describe('using the query server', function () {
           }
           actualResultSets[reader.schema.name] = actualRows;
         }
+        parsedResults.resolve();
       } finally {
         if (fileReader) {
           fileReader.dispose();
@@ -216,7 +224,8 @@ describe('using the query server', function () {
       }
     });
 
-    it(`should have correct results for query ${queryName}`, async function () {
+    it(`should have correct results for query ${queryName}`, async function() {
+      await parsedResults.done();
       expect(actualResultSets!).not.to.be.empty;
       expect(Object.keys(actualResultSets!).sort()).to.eql(Object.keys(queryTestCase.expectedResultSets).sort());
       for (const name in queryTestCase.expectedResultSets) {

extensions/ql-vscode/tsconfig.json

@@ -1,3 +1,3 @@
 {
   "extends": "./node_modules/typescript-config/extension.tsconfig.json"
-}
+}

lib/semmle-bqrs/src/bqrs-custom.ts

@@ -166,7 +166,7 @@ type ParseTupleAction = (src: readonly ColumnValue[], dest: any) => void;
 type TupleParser<T> = (src: readonly ColumnValue[]) => T;
 
 export class CustomResultSet<TTuple> {
-  public constructor(private reader: ResultSetReader, private readonly type: { new(): TTuple },
+  public constructor(private reader: ResultSetReader,
     private readonly tupleParser: TupleParser<TTuple>) {
   }
 
@@ -192,7 +192,7 @@ class CustomResultSetBinder {
     const binder = new CustomResultSetBinder(rowType, reader.schema);
     const tupleParser = binder.bindRoot<TTuple>();
 
-    return new CustomResultSet<TTuple>(reader, rowType, tupleParser);
+    return new CustomResultSet<TTuple>(reader, tupleParser);
   }
 
   private bindRoot<TTuple>(): TupleParser<TTuple> {

lib/semmle-bqrs/src/bqrs-parse.ts

@@ -144,7 +144,14 @@ async function parsePrimitiveColumn(d: StreamDigester, type: PrimitiveTypeKind,
   switch (type) {
     case 's': return await parseString(d, pool);
     case 'b': return await d.readByte() !== 0;
-    case 'i': return await d.readLEB128UInt32();
+    case 'i': {
+      const unsignedValue = await d.readLEB128UInt32();
+      // `int` column values are encoded as 32-bit unsigned LEB128, but are really 32-bit two's
+      // complement signed integers. The easiest way to reinterpret from an unsigned int32 to a
+      // signed int32 in JavaScript is to use a bitwise operator, which does this coercion on its
+      // operands automatically.
+      return unsignedValue | 0;
+    }
     case 'f': return await d.readDoubleLE();
     case 'd': return await d.readDate();
     case 'u': return await parseString(d, pool);

lib/semmle-io/src/digester.ts

@@ -124,7 +124,7 @@ export class StreamDigester {
   }
 
   /**
-   * Implements an async read that span multple buffers.
+   * Implements an async read that spans multiple buffers.
    *
    * @param canReadFunc Callback function to determine how many bytes are required to complete the
    *  read operation.
@@ -186,7 +186,7 @@ export class StreamDigester {
   private readKnownSizeAcrossSeam<T>(byteCount: number,
     readFunc: (buffer: Buffer, offset: number) => T): Promise<T> {
 
-    return this.readAcrossSeam((buffer, offset, availableByteCount) => byteCount, readFunc);
+    return this.readAcrossSeam((_buffer, _offset, _availableByteCount) => byteCount, readFunc);
   }
 
   private readKnownSize<T>(byteCount: number, readFunc: (buffer: Buffer, offset: number) => T):
@@ -300,4 +300,4 @@ function canDecodeLEB128UInt32(buffer: Buffer, offset: number, byteCount: number
 function decodeLEB128UInt32(buffer: Buffer, offset: number): number {
   const { value } = leb.decodeUInt32(buffer, offset);
   return value;
-}
+}

lib/semmle-vscode-utils/src/index.ts

@@ -1 +1,4 @@
 export * from './disposable-object';
+export * from './multi-file-system-watcher';
+export * from './ui-service';
+

lib/semmle-vscode-utils/src/multi-file-system-watcher.ts

@@ -0,0 +1,65 @@
+import { DisposableObject } from './disposable-object';
+import { EventEmitter, Event, Uri, GlobPattern, workspace } from 'vscode';
+
+/**
+ * A collection of `FileSystemWatcher` objects. Disposing this object disposes all of the individual
+ * `FileSystemWatcher` objects and their event registrations.
+ */
+class WatcherCollection extends DisposableObject {
+  constructor() {
+    super();
+  }
+
+  /**
+   * Create a `FileSystemWatcher` and add it to the collection.
+   * @param pattern The pattern to watch.
+   * @param listener The event listener to be invoked when a watched file is created, changed, or
+   *   deleted.
+   * @param thisArgs The `this` argument for the event listener.
+   */
+  public addWatcher(pattern: GlobPattern, listener: (e: Uri) => any, thisArgs: any): void {
+    const watcher = workspace.createFileSystemWatcher(pattern);
+    this.push(watcher.onDidCreate(listener, thisArgs));
+    this.push(watcher.onDidChange(listener, thisArgs));
+    this.push(watcher.onDidDelete(listener, thisArgs));
+  }
+}
+
+/**
+ * A class to watch multiple patterns in the file system at the same time, reporting all
+ * notifications via a single event.
+ */
+export class MultiFileSystemWatcher extends DisposableObject {
+  private readonly _onDidChange = this.push(new EventEmitter<Uri>());
+  private watchers = this.track(new WatcherCollection());
+
+  constructor() {
+    super();
+  }
+
+  /**
+   * Event to be fired when any watched file is created, changed, or deleted.
+   */
+  public get onDidChange(): Event<Uri> { return this._onDidChange.event; }
+
+  /**
+   * Adds a new pattern to watch.
+   * @param pattern The pattern to watch.
+   */
+  public addWatch(pattern: GlobPattern): void {
+    this.watchers.addWatcher(pattern, this.handleDidChange, this);
+  }
+
+  /**
+   * Deletes all existing watchers.
+   */
+  public clear(): void {
+    this.disposeAndStopTracking(this.watchers);
+    this.watchers = this.track(new WatcherCollection());
+  }
+
+  private handleDidChange(uri: Uri): void {
+    this._onDidChange.fire(uri);
+  }
+}
+

lib/semmle-vscode-utils/src/ui-service.ts

@@ -0,0 +1,27 @@
+import { commands, TreeDataProvider, window } from 'vscode';
+import { DisposableObject } from './disposable-object';
+
+/**
+ * A VS Code service that interacts with the UI, including handling commands.
+ */
+export class UIService extends DisposableObject {
+  protected constructor() {
+    super();
+  }
+
+  /**
+   * Registers a command handler with Visual Studio Code.
+   * @param command The ID of the command to register.
+   * @param callback Callback function to implement the command.
+   * @remarks The command handler is automatically unregistered when the service is disposed.
+   */
+  protected registerCommand(command: string, callback: (...args: any[]) => any): void {
+    this.push(commands.registerCommand(command, callback, this));
+  }
+
+  protected registerTreeDataProvider<T>(viewId: string, treeDataProvider: TreeDataProvider<T>):
+    void {
+
+    this.push(window.registerTreeDataProvider<T>(viewId, treeDataProvider));
+  }
+}

rush.json

@@ -4,10 +4,10 @@
  */
 {
   "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush.schema.json",
-  "rushVersion": "5.11.2",
+  "rushVersion": "5.20.0",
-  "pnpmVersion": "2.15.1",
+  "pnpmVersion": "4.8.0",
   "pnpmOptions": {
-    "strictPeerDependencies": true,
+    "strictPeerDependencies": true
   },
   "nodeSupportedVersionRange": ">=10.13.0 <13.0.0",
   "ensureConsistentVersions": true,
@@ -55,4 +55,4 @@
       "shouldPublish": true
     }
   ]
-}
+}

syntaxes/README.md

@@ -0,0 +1,3 @@
+This folder contains a compiled version of the textmate grammar for use with systems that need a compiled version of the grammar in the repository such as linguist. It also contains a patch for the grammar to make it work with linguist.
+
+To update the grammar, first build the extension, then run "./updateSyntax".

syntaxes/updateSyntax

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+
+perl -0777 -pe 's/{\s*"include"\s*:\s*"text.html.markdown#[a-zA-Z_]+\"\s*}\s*,//igs' ../extensions/ql-vscode/out/syntaxes/ql.tmLanguage.json > ./ql.tmLanguage.json

tools/build-tasks/package.json

@@ -16,7 +16,7 @@
   },
   "dependencies": {
     "@microsoft/node-core-library": "~3.13.0",
-    "@microsoft/rush-lib": "~5.11.2",
+    "@microsoft/rush-lib": "~5.20.0",
     "ansi-colors": "^4.0.1",
     "child-process-promise": "^2.2.1",
     "fs-extra": "^8.1.0",
@@ -44,4 +44,4 @@
     "typescript-config": "^0.0.1",
     "typescript-formatter": "^7.2.2"
   }
-}
+}

tools/build-tasks/src/rush.ts

@@ -84,6 +84,8 @@ export class RushContext {
     }
     else {
       pkg = await this.getShrinkwrapPackage(name, version);
+      // Ensure a proper version number. pnpm uses syntax like 3.4.0_glob@7.1.6 for peer dependencies
+      version = version.split('_')[0];
       packagePath = path.join(this.packageRepository, name, version, 'package');
       if (!await fs.pathExists(packagePath)) {
         throw new Error(`Package '${name}:${version}' not found in package repository.`);

tools/build-tasks/src/textmate.ts

@@ -218,7 +218,7 @@ function transformFile(yaml: any) {
 }
 
 export function transpileTextMateGrammar() {
-  return through.obj((file: Vinyl, encoding: string, callback: Function): void => {
+  return through.obj((file: Vinyl, _encoding: string, callback: Function): void => {
     if (file.isNull()) {
       callback(null, file);
     }

tools/build-tasks/src/typescript.ts

@@ -43,7 +43,7 @@ export function compileTypeScript() {
   return tsProject.src()
     .pipe(sourcemaps.init())
     .pipe(tsProject(goodReporter()))
-    .pipe(sourcemaps.mapSources((sourcePath, file) => {
+    .pipe(sourcemaps.mapSources((sourcePath, _file) => {
       // The source path is kind of odd, because it's relative to the `tsconfig.json` file in the
       // `typescript-config` package, which lives in the `node_modules` directory of the package
       // that is being built. It starts out as something like '../../../src/foo.ts', and we need to

tsconfig.json

@@ -0,0 +1 @@
+{}

tsfmt.json

@@ -1,4 +1,5 @@
 {
+  "newLineCharacter": "\n",
   "indentStyle": 2,
   "insertSpaceAfterCommaDelimiter": true,
   "insertSpaceAfterSemicolonInForStatements": true,
@@ -15,4 +16,4 @@
   "insertSpaceBeforeFunctionParenthesis": false,
   "placeOpenBraceOnNewLineForFunctions": false,
   "placeOpenBraceOnNewLineForControlBlocks": false
-}
+}