Update changelog in preparation for v1.2.2 release (#433)

* Update changelog in preparation for v1.2.2 release

Co-authored-by: Shati Patel <42641846+shati-patel@users.noreply.github.com>

.editorconfig

@@ -4,3 +4,4 @@ indent_size = 2
 charset = utf-8
 end_of_line = lf
 trim_trailing_whitespace = true
+insert_final_newline = true

.github/pull_request_template.md

@@ -7,6 +7,6 @@ Replace this with a description of the changes your pull request makes.
 
 ## Checklist
 
-- [ ] [CHANGELOG.md](../extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
+- [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
 - [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
 - [ ] `@github/product-docs-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -0,0 +1,21 @@
+name: "Code Scanning - CodeQL"
+
+on:
+  push:
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  codeql:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

.github/workflows/main.yml

@@ -10,7 +10,7 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
@@ -18,11 +18,12 @@ jobs:
         with:
           node-version: '10.18.1'
 
+      - name: Install dependencies
+        run: node common/scripts/install-run-rush.js install
+        shell: bash
+
       - name: Build
-        run: |
-          cd build
-          npm install
-          npm run build-ci
+        run: node common/scripts/install-run-rush.js build
         shell: bash
 
       - name: Prepare artifacts
@@ -46,7 +47,7 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
@@ -55,18 +56,25 @@ jobs:
           node-version: '10.18.1'
 
       # We have to build the dependencies in `lib` before running any tests.
-      - name: Build
-        run: |
-          cd build
-          npm install
-          npm run build-ci
+      - name: Install dependencies
+        run: node common/scripts/install-run-rush.js install
         shell: bash
 
+      - name: Build
+        run: node common/scripts/install-run-rush.js build
+        shell: bash
+
+      - name: Lint
+        run: |
+          cd extensions/ql-vscode
+          npm run lint
+
       - name: Install CodeQL
         run: |
           mkdir codeql-home
           curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
           unzip -q -o codeql-home/codeql.zip -d codeql-home
+          unzip -q -o codeql-home/codeql.zip codeql/codeql.exe -d codeql-home
           rm codeql-home/codeql.zip
         shell: bash
 
@@ -80,7 +88,7 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: |
           cd extensions/ql-vscode
-          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.cmd')
+          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)

.github/workflows/release.yml

@@ -27,24 +27,18 @@ jobs:
     # TODO Share steps with the main workflow.
     steps:
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@v2
 
       - uses: actions/setup-node@v1
         with:
           node-version: '10.18.1'
 
-      # The checkout action does not fetch the master branch.
-      # Fetch the master branch so that we can base the version bump PR against master.
-      - name: Fetch master branch
-        run: |
-          git fetch --depth=1 origin master:master
+      - name: Install dependencies
+        run: node common/scripts/install-run-rush.js install
+        shell: bash
 
       - name: Build
-        run: |
-          cd build
-          npm install
-          # Release build instead of dev build.
-          npm run build-release
+        run: node common/scripts/install-run-rush.js build --release
         shell: bash
 
       - name: Prepare artifacts
@@ -57,8 +51,8 @@ jobs:
           VSIX_PATH="$(ls dist/*.vsix)"
           echo "::set-output name=vsix_path::$VSIX_PATH"
           # Transform the GitHub ref so it can be used in a filename.
-          # This is mainly needed for testing branches that modify this workflow.
-          REF_NAME="$(echo ${{ github.ref }} | sed -e 's:/:-:g')"
+          # The last sed invocation is used for testing branches that modify this workflow.
+          REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
       # Uploading artifacts is not necessary to create a release.
@@ -99,6 +93,13 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
+      # The checkout action does not fetch the master branch.
+      # Fetch the master branch so that we can base the version bump PR against master.
+      - name: Fetch master branch
+        run: |
+          git fetch --depth=1 origin master:master
+          git checkout master
+
       - name: Bump patch version
         id: bump-patch-version
         if: success()
@@ -110,7 +111,7 @@ jobs:
           echo "::set-output name=next_version::$NEXT_VERSION"
 
       - name: Create version bump PR
-        uses: peter-evans/create-pull-request@c202684c928d4c9f18394b2ad11df905c5d8b40c # v2.1.2
+        uses: peter-evans/create-pull-request@c7b64af0a489eae91f7890f2c1b63d13cc2d8ab7 # v2.4.2
         if: success()
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -18,3 +18,4 @@ artifacts/
 # Rush files
 /common/temp/**
 package-deps.json
+**/.rush/temp

.vscode/launch.json

@@ -8,8 +8,7 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--disable-extensions"
+        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -24,25 +23,30 @@
     },
     {
       "name": "Launch Unit Tests (vscode-codeql)",
-      "type": "extensionHost",
+      "type": "node",
       "request": "launch",
-      "runtimeExecutable": "${execPath}",
-      "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/test",
-        "--disable-extensions"
+      "program": "${workspaceFolder}/extensions/ql-vscode/node_modules/mocha/bin/_mocha",
+      "showAsyncStacks": true,
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
+      "runtimeArgs": [
+        "--inspect=9229"
       ],
+      "args": [
+        "--exit",
+        "-u",
+        "bdd",
+        "--colors",
+        "--diff",
+        "-r",
+        "ts-node/register",
+        "test/pure-tests/**/*.ts"
+      ],
+      "port": 9229,
       "stopOnEntry": false,
       "sourceMaps": true,
-      "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-bqrs/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io-node/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-vscode-utils/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/test/**/*.js"
-      ],
-      "preLaunchTask": "Build"
+      "preLaunchTask": "Build",
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
     },
     {
       "name": "Launch Integration Tests - No Workspace (vscode-codeql)",
@@ -51,8 +55,7 @@
       "runtimeExecutable": "${execPath}",
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
-        "--disable-extensions"
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -70,7 +73,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
-        "${workspaceRoot}/extensions/ql-vscode/test/data",
+        "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -81,4 +84,4 @@
       "preLaunchTask": "Build"
     }
   ]
-}
+}

.vscode/settings.json

@@ -1,14 +1,38 @@
 // Place your settings in this file to overwrite default and user settings.
 {
   "files.exclude": {
-    "out": false // set this to true to hide the "out" folder with the compiled JS files
+    "**/out": true, // set this to true to hide the "out" folder with the compiled JS files
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
   "files.watcherExclude": {
     "**/.git/**": true,
-    "**/node_modules/*/**": true
+    "**/out": true,
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
   "search.exclude": {
-    "out": true // set this to false to include "out" folder in search results
+    "**/out": true, // set this to false to include "out" folder in search results
+    "**/dist": true,
+    "**/node_modules": true,
+    "common/temp": true,
+    "**/.vscode-test": true
   },
-  "typescript.tsdk": "./common/temp/node_modules/typescript/lib" // we want to use the TS server from our node_modules folder to control its version
-}
+  "typescript.tsdk": "./common/temp/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
+  "eslint.validate": [
+    "javascript",
+    "javascriptreact",
+    "typescript",
+    "typescriptreact"
+  ],
+  "eslint.options": {
+    // This is necessary so that eslint can properly resolve its plugins
+    "resolvePluginsRelativeTo": "./extensions/ql-vscode"
+  },
+  // Force this to false since this will cause too many changes on each commit
+  "editor.formatOnSave": false
+}

.vscode/tasks.json

@@ -28,7 +28,7 @@
             "file": 1,
             "location": 2,
             "message": 3
-          },
+          }
         },
         "$ts-webpack"
       ]
@@ -100,6 +100,15 @@
         "clear": true
       },
       "problemMatcher": []
+    },
+    {
+      "type": "npm",
+      "script": "watch",
+      "path": "extensions/ql-vscode/",
+      "problemMatcher": [
+        "$gulp-tsc"
+      ],
+      "group": "build"
     }
   ]
 }

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-## Contributing
+# Contributing
 
 [fork]: https://github.com/github/vscode-codeql/fork
 [pr]: https://github.com/github/vscode-codeql/compare
@@ -13,19 +13,19 @@ Please note that this project is released with a [Contributor Code of Conduct][c
 
 ## Submitting a pull request
 
-0. [Fork][fork] and clone the repository
-0. Set up a local build
-0. Create a new branch: `git checkout -b my-branch-name`
-0. Make your change
-0. Push to your fork and [submit a pull request][pr]
-0. Pat yourself on the back and wait for your pull request to be reviewed and merged.
+1. [Fork][fork] and clone the repository
+1. Set up a local build
+1. Create a new branch: `git checkout -b my-branch-name`
+1. Make your change
+1. Push to your fork and [submit a pull request][pr]
+1. Pat yourself on the back and wait for your pull request to be reviewed and merged.
 
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 
-- Follow the [style guide][style].
-- Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
-- Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Follow the [style guide][style].
+* Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
+* Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
+* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
@@ -42,12 +42,22 @@ If you plan on building from the command line, it's easiest if Rush is installed
 npm install -g @microsoft/rush
 ```
 
+To get started, run:
+
+```shell
+rush update && rush build
+```
+
 Note that when you run the `rush` command from the globally installed version, it will examine the
 `rushVersion` property in the repo's `rush.json`, and if it differs from the globally installed
 version, it will download, cache, and run the version of Rush specified in the `rushVersion`
 property.
 
-If you plan on only building via VS Code tasks, you don't need Rush installed at all, since those
+A few more things to know about using rush:
+
+* Avoid running `npm` for any commands that install/link dependencies
+* Instead use the *rush* equivalent: `rush add <package>`, `rush update`, etc.
+* If you plan on only building via VS Code tasks, you don't need Rush installed at all, since those
 tasks run `common/scripts/install-run-rush.js` to bootstrap a locally installed and cached copy of
 Rush.
 
@@ -66,7 +76,7 @@ a single-project repo. With Rush, you need to do an "update" instead:
 ##### From the command line
 
 ```shell
-$ rush update
+rush update
 ```
 
 #### Building all projects (instead of `gulp`)
@@ -100,6 +110,8 @@ force a full rebuild of all projects:
 rush rebuild --verbose
 ```
 
+Note that `rush rebuild` performs a complete rebuild, whereas `rush build` performs an incremental build and in many cases will not need to do anything at all.
+
 ### Installing
 
 You can install the `.vsix` file from within VS Code itself, from the Extensions container in the sidebar:
@@ -118,18 +130,40 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
+### Running the unit/integration tests
+
+Ensure the `CODEQL_PATH` environment variable is set to point to the `codeql` cli executable.
+
+Outside of vscode, run:
+
+```shell
+npm run test && npm run integration
+```
+
+Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
+
 ## Releasing (write access required)
 
+1. Double-check the `CHANGELOG.md` contains all desired change comments
+   and has the version to be released with date at the top.
+1. Double-check that the extension `package.json` has the version you intend to release.
+   If you are doing a patch release (as opposed to minor or major version) this should already
+   be correct.
 1. Trigger a release build on Actions by adding a new tag on master of the format `vxx.xx.xx`
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
-1. Download the VSIX from the draft GitHub release that is created when the release build finishes.
+1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
+1. Optionally unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
+   or look at the source if there's any doubt the right code is being shipped.
 1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
 1. Click the `...` menu in the CodeQL row and click **Update**.
 1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Publish the draft GitHub release and confirm the new release is marked as the latest release at https://github.com/github/vscode-codeql/releases.
+1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
+1. If documentation changes need to be published, notify documentation team that release has been made.
+1. Review and merge the version bump PR that is automatically created by Actions.
 
 ## Resources
 
-- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
-- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
-- [GitHub Help](https://help.github.com)
+* [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+* [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+* [GitHub Help](https://help.github.com)

build/README.md

@@ -1,12 +0,0 @@
-GitHub Actions Build directory
-===
-
-The point of this directory is to allow us to do a local installation *of* the rush
-tool, since 
- - installing globally is not permitted on github actions
- - installing locally in the root directory of the repo creates `node_modules` there,
-   and rush itself gives error messages since it thinks `node_modules` is not supposed 
-   to exist, since rush is supposed to be managing subproject dependencies.
-
-Running rush from a subdirectory searches parent directories for `rush.json`
-and does the build starting from that file's location.

build/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "build",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "dependencies": {
-    "@microsoft/rush": "^5.10.3"
-  },
-  "devDependencies": {},
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "build": "rush update && rush build",
-    "build-ci": "rush install && rush build",
-    "build-release": "rush install && rush build --release"
-  },
-  "author": "GitHub"
-}

common/scripts/install-run-rush.js

@@ -1,52 +1,67 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where the Rush command may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
-// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it.
-// An example usage would be:
-//
-//    node common/scripts/install-run-rush.js install
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const path = require("path");
-const fs = require("fs");
-const install_run_1 = require("./install-run");
-const PACKAGE_NAME = '@microsoft/rush';
-function getRushVersion() {
-    const rushJsonFolder = install_run_1.findRushJsonFolder();
-    const rushJsonPath = path.join(rushJsonFolder, install_run_1.RUSH_JSON_FILENAME);
-    try {
-        const rushJsonContents = fs.readFileSync(rushJsonPath, 'utf-8');
-        // Use a regular expression to parse out the rushVersion value because rush.json supports comments,
-        // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script.
-        const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/);
-        return rushJsonMatches[1];
-    }
-    catch (e) {
-        throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` +
-            'The \'rushVersion\' field is either not assigned in rush.json or was specified ' +
-            'using an unexpected syntax.');
-    }
-}
-function run() {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ ...packageBinArgs /* [build, --to, myproject] */] = process.argv;
-    if (!nodePath || !scriptPath) {
-        throw new Error('Unexpected exception: could not detect node path or script path');
-    }
-    if (process.argv.length < 3) {
-        console.log('Usage: install-run-rush.js <command> [args...]');
-        console.log('Example: install-run-rush.js build --to myproject');
-        process.exit(1);
-    }
-    install_run_1.runWithErrorAndStatusCode(() => {
-        const version = getRushVersion();
-        console.log(`The rush.json configuration requests Rush version ${version}`);
-        return install_run_1.installAndRun(PACKAGE_NAME, version, 'rush', packageBinArgs);
-    });
-}
-run();
+"use strict";
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See the @microsoft/rush package's LICENSE file for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
+//
+// This script is intended for usage in an automated build environment where the Rush command may not have
+// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
+// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it.
+// An example usage would be:
+//
+//    node common/scripts/install-run-rush.js install
+//
+// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
+const path = require("path");
+const fs = require("fs");
+const install_run_1 = require("./install-run");
+const PACKAGE_NAME = '@microsoft/rush';
+const RUSH_PREVIEW_VERSION = 'RUSH_PREVIEW_VERSION';
+function _getRushVersion() {
+    const rushPreviewVersion = process.env[RUSH_PREVIEW_VERSION];
+    if (rushPreviewVersion !== undefined) {
+        console.log(`Using Rush version from environment variable ${RUSH_PREVIEW_VERSION}=${rushPreviewVersion}`);
+        return rushPreviewVersion;
+    }
+    const rushJsonFolder = install_run_1.findRushJsonFolder();
+    const rushJsonPath = path.join(rushJsonFolder, install_run_1.RUSH_JSON_FILENAME);
+    try {
+        const rushJsonContents = fs.readFileSync(rushJsonPath, 'utf-8');
+        // Use a regular expression to parse out the rushVersion value because rush.json supports comments,
+        // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script.
+        const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/);
+        return rushJsonMatches[1];
+    }
+    catch (e) {
+        throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` +
+            'The \'rushVersion\' field is either not assigned in rush.json or was specified ' +
+            'using an unexpected syntax.');
+    }
+}
+function _run() {
+    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ ...packageBinArgs /* [build, --to, myproject] */] = process.argv;
+    // Detect if this script was directly invoked, or if the install-run-rushx script was invokved to select the
+    // appropriate binary inside the rush package to run
+    const scriptName = path.basename(scriptPath);
+    const bin = scriptName.toLowerCase() === 'install-run-rushx.js' ? 'rushx' : 'rush';
+    if (!nodePath || !scriptPath) {
+        throw new Error('Unexpected exception: could not detect node path or script path');
+    }
+    if (process.argv.length < 3) {
+        console.log(`Usage: ${scriptName} <command> [args...]`);
+        if (scriptName === 'install-run-rush.js') {
+            console.log(`Example: ${scriptName} build --to myproject`);
+        }
+        else {
+            console.log(`Example: ${scriptName} custom-command`);
+        }
+        process.exit(1);
+    }
+    install_run_1.runWithErrorAndStatusCode(() => {
+        const version = _getRushVersion();
+        console.log(`The rush.json configuration requests Rush version ${version}`);
+        return install_run_1.installAndRun(PACKAGE_NAME, version, bin, packageBinArgs);
+    });
+}
+_run();
 //# sourceMappingURL=install-run-rush.js.map

common/scripts/install-run-rushx.js

@@ -0,0 +1,18 @@
+"use strict";
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See the @microsoft/rush package's LICENSE file for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
+//
+// This script is intended for usage in an automated build environment where the Rush command may not have
+// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
+// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the
+// rushx command.
+//
+// An example usage would be:
+//
+//    node common/scripts/install-run-rushx.js custom-command
+//
+// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
+require("./install-run-rush");
+//# sourceMappingURL=install-run-rushx.js.map

common/scripts/install-run.js

@@ -1,399 +1,433 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where a Node tool may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the specified
-// version of the specified tool (if not already installed), and then pass a command-line to it.
-// An example usage would be:
-//
-//    node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const childProcess = require("child_process");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-exports.RUSH_JSON_FILENAME = 'rush.json';
-const INSTALLED_FLAG_FILENAME = 'installed.flag';
-const NODE_MODULES_FOLDER_NAME = 'node_modules';
-const PACKAGE_JSON_FILENAME = 'package.json';
-/**
- * Parse a package specifier (in the form of name\@version) into name and version parts.
- */
-function parsePackageSpecifier(rawPackageSpecifier) {
-    rawPackageSpecifier = (rawPackageSpecifier || '').trim();
-    const separatorIndex = rawPackageSpecifier.lastIndexOf('@');
-    let name;
-    let version = undefined;
-    if (separatorIndex === 0) {
-        // The specifier starts with a scope and doesn't have a version specified
-        name = rawPackageSpecifier;
-    }
-    else if (separatorIndex === -1) {
-        // The specifier doesn't have a version
-        name = rawPackageSpecifier;
-    }
-    else {
-        name = rawPackageSpecifier.substring(0, separatorIndex);
-        version = rawPackageSpecifier.substring(separatorIndex + 1);
-    }
-    if (!name) {
-        throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`);
-    }
-    return { name, version };
-}
-/**
- * Resolve a package specifier to a static version
- */
-function resolvePackageVersion(rushCommonFolder, { name, version }) {
-    if (!version) {
-        version = '*'; // If no version is specified, use the latest version
-    }
-    if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) {
-        // If the version contains only characters that we recognize to be used in static version specifiers,
-        // pass the version through
-        return version;
-    }
-    else {
-        // version resolves to
-        try {
-            const rushTempFolder = ensureAndJoinPath(rushCommonFolder, 'temp');
-            const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-            syncNpmrc(sourceNpmrcFolder, rushTempFolder);
-            const npmPath = getNpmPath();
-            // This returns something that looks like:
-            //  @microsoft/rush@3.0.0 '3.0.0'
-            //  @microsoft/rush@3.0.1 '3.0.1'
-            //  ...
-            //  @microsoft/rush@3.0.20 '3.0.20'
-            //  <blank line>
-            const npmVersionSpawnResult = childProcess.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], {
-                cwd: rushTempFolder,
-                stdio: []
-            });
-            if (npmVersionSpawnResult.status !== 0) {
-                throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`);
-            }
-            const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString();
-            const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line);
-            const latestVersion = versionLines[versionLines.length - 1];
-            if (!latestVersion) {
-                throw new Error('No versions found for the specified version range.');
-            }
-            const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/);
-            if (!versionMatches) {
-                throw new Error(`Invalid npm output ${latestVersion}`);
-            }
-            return versionMatches[1];
-        }
-        catch (e) {
-            throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`);
-        }
-    }
-}
-let _npmPath = undefined;
-/**
- * Get the absolute path to the npm executable
- */
-function getNpmPath() {
-    if (!_npmPath) {
-        try {
-            if (os.platform() === 'win32') {
-                // We're on Windows
-                const whereOutput = childProcess.execSync('where npm', { stdio: [] }).toString();
-                const lines = whereOutput.split(os.EOL).filter((line) => !!line);
-                // take the last result, we are looking for a .cmd command
-                // see https://github.com/Microsoft/web-build-tools/issues/759
-                _npmPath = lines[lines.length - 1];
-            }
-            else {
-                // We aren't on Windows - assume we're on *NIX or Darwin
-                _npmPath = childProcess.execSync('which npm', { stdio: [] }).toString();
-            }
-        }
-        catch (e) {
-            throw new Error(`Unable to determine the path to the NPM tool: ${e}`);
-        }
-        _npmPath = _npmPath.trim();
-        if (!fs.existsSync(_npmPath)) {
-            throw new Error('The NPM executable does not exist');
-        }
-    }
-    return _npmPath;
-}
-exports.getNpmPath = getNpmPath;
-let _rushJsonFolder;
-/**
- * Find the absolute path to the folder containing rush.json
- */
-function findRushJsonFolder() {
-    if (!_rushJsonFolder) {
-        let basePath = __dirname;
-        let tempPath = __dirname;
-        do {
-            const testRushJsonPath = path.join(basePath, exports.RUSH_JSON_FILENAME);
-            if (fs.existsSync(testRushJsonPath)) {
-                _rushJsonFolder = basePath;
-                break;
-            }
-            else {
-                basePath = tempPath;
-            }
-        } while (basePath !== (tempPath = path.dirname(basePath))); // Exit the loop when we hit the disk root
-        if (!_rushJsonFolder) {
-            throw new Error('Unable to find rush.json.');
-        }
-    }
-    return _rushJsonFolder;
-}
-exports.findRushJsonFolder = findRushJsonFolder;
-/**
- * Create missing directories under the specified base directory, and return the resolved directory.
- *
- * Does not support "." or ".." path segments.
- * Assumes the baseFolder exists.
- */
-function ensureAndJoinPath(baseFolder, ...pathSegments) {
-    let joinedPath = baseFolder;
-    try {
-        for (let pathSegment of pathSegments) {
-            pathSegment = pathSegment.replace(/[\\\/]/g, '+');
-            joinedPath = path.join(joinedPath, pathSegment);
-            if (!fs.existsSync(joinedPath)) {
-                fs.mkdirSync(joinedPath);
-            }
-        }
-    }
-    catch (e) {
-        throw new Error(`Error building local installation folder (${path.join(baseFolder, ...pathSegments)}): ${e}`);
-    }
-    return joinedPath;
-}
-/**
- * As a workaround, _syncNpmrc() copies the .npmrc file to the target folder, and also trims
- * unusable lines from the .npmrc file.  If the source .npmrc file not exist, then _syncNpmrc()
- * will delete an .npmrc that is found in the target folder.
- *
- * Why are we trimming the .npmrc lines?  NPM allows environment variables to be specified in
- * the .npmrc file to provide different authentication tokens for different registry.
- * However, if the environment variable is undefined, it expands to an empty string, which
- * produces a valid-looking mapping with an invalid URL that causes an error.  Instead,
- * we'd prefer to skip that line and continue looking in other places such as the user's
- * home directory.
- *
- * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc()
- */
-function syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder) {
-    const sourceNpmrcPath = path.join(sourceNpmrcFolder, '.npmrc');
-    const targetNpmrcPath = path.join(targetNpmrcFolder, '.npmrc');
-    try {
-        if (fs.existsSync(sourceNpmrcPath)) {
-            let npmrcFileLines = fs.readFileSync(sourceNpmrcPath).toString().split('\n');
-            npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim());
-            const resultLines = [];
-            // Trim out lines that reference environment variables that aren't defined
-            for (const line of npmrcFileLines) {
-                // This finds environment variable tokens that look like "${VAR_NAME}"
-                const regex = /\$\{([^\}]+)\}/g;
-                const environmentVariables = line.match(regex);
-                let lineShouldBeTrimmed = false;
-                if (environmentVariables) {
-                    for (const token of environmentVariables) {
-                        // Remove the leading "${" and the trailing "}" from the token
-                        const environmentVariableName = token.substring(2, token.length - 1);
-                        if (!process.env[environmentVariableName]) {
-                            lineShouldBeTrimmed = true;
-                            break;
-                        }
-                    }
-                }
-                if (lineShouldBeTrimmed) {
-                    // Example output:
-                    // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}"
-                    resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line);
-                }
-                else {
-                    resultLines.push(line);
-                }
-            }
-            fs.writeFileSync(targetNpmrcPath, resultLines.join(os.EOL));
-        }
-        else if (fs.existsSync(targetNpmrcPath)) {
-            // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target
-            fs.unlinkSync(targetNpmrcPath);
-        }
-    }
-    catch (e) {
-        throw new Error(`Error syncing .npmrc file: ${e}`);
-    }
-}
-/**
- * Detects if the package in the specified directory is installed
- */
-function isPackageAlreadyInstalled(packageInstallFolder) {
-    try {
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        if (!fs.existsSync(flagFilePath)) {
-            return false;
-        }
-        const fileContents = fs.readFileSync(flagFilePath).toString();
-        return fileContents.trim() === process.version;
-    }
-    catch (e) {
-        return false;
-    }
-}
-/**
- * Removes the following files and directories under the specified folder path:
- *  - installed.flag
- *  -
- *  - node_modules
- */
-function cleanInstallFolder(rushCommonFolder, packageInstallFolder) {
-    try {
-        const flagFile = path.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        if (fs.existsSync(flagFile)) {
-            fs.unlinkSync(flagFile);
-        }
-        const packageLockFile = path.resolve(packageInstallFolder, 'package-lock.json');
-        if (fs.existsSync(packageLockFile)) {
-            fs.unlinkSync(packageLockFile);
-        }
-        const nodeModulesFolder = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME);
-        if (fs.existsSync(nodeModulesFolder)) {
-            const rushRecyclerFolder = ensureAndJoinPath(rushCommonFolder, 'temp', 'rush-recycler', `install-run-${Date.now().toString()}`);
-            fs.renameSync(nodeModulesFolder, rushRecyclerFolder);
-        }
-    }
-    catch (e) {
-        throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`);
-    }
-}
-function createPackageJson(packageInstallFolder, name, version) {
-    try {
-        const packageJsonContents = {
-            'name': 'ci-rush',
-            'version': '0.0.0',
-            'dependencies': {
-                [name]: version
-            },
-            'description': 'DON\'T WARN',
-            'repository': 'DON\'T WARN',
-            'license': 'MIT'
-        };
-        const packageJsonPath = path.join(packageInstallFolder, PACKAGE_JSON_FILENAME);
-        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2));
-    }
-    catch (e) {
-        throw new Error(`Unable to create package.json: ${e}`);
-    }
-}
-/**
- * Run "npm install" in the package install folder.
- */
-function installPackage(packageInstallFolder, name, version) {
-    try {
-        console.log(`Installing ${name}...`);
-        const npmPath = getNpmPath();
-        const result = childProcess.spawnSync(npmPath, ['install'], {
-            stdio: 'inherit',
-            cwd: packageInstallFolder,
-            env: process.env
-        });
-        if (result.status !== 0) {
-            throw new Error('"npm install" encountered an error');
-        }
-        console.log(`Successfully installed ${name}@${version}`);
-    }
-    catch (e) {
-        throw new Error(`Unable to install package: ${e}`);
-    }
-}
-/**
- * Get the ".bin" path for the package.
- */
-function getBinPath(packageInstallFolder, binName) {
-    const binFolderPath = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin');
-    const resolvedBinName = (os.platform() === 'win32') ? `${binName}.cmd` : binName;
-    return path.resolve(binFolderPath, resolvedBinName);
-}
-/**
- * Write a flag file to the package's install directory, signifying that the install was successful.
- */
-function writeFlagFile(packageInstallFolder) {
-    try {
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        fs.writeFileSync(flagFilePath, process.version);
-    }
-    catch (e) {
-        throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`);
-    }
-}
-function installAndRun(packageName, packageVersion, packageBinName, packageBinArgs) {
-    const rushJsonFolder = findRushJsonFolder();
-    const rushCommonFolder = path.join(rushJsonFolder, 'common');
-    const packageInstallFolder = ensureAndJoinPath(rushCommonFolder, 'temp', 'install-run', `${packageName}@${packageVersion}`);
-    if (!isPackageAlreadyInstalled(packageInstallFolder)) {
-        // The package isn't already installed
-        cleanInstallFolder(rushCommonFolder, packageInstallFolder);
-        const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-        syncNpmrc(sourceNpmrcFolder, packageInstallFolder);
-        createPackageJson(packageInstallFolder, packageName, packageVersion);
-        installPackage(packageInstallFolder, packageName, packageVersion);
-        writeFlagFile(packageInstallFolder);
-    }
-    const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`;
-    const statusMessageLine = new Array(statusMessage.length + 1).join('-');
-    console.log(os.EOL + statusMessage + os.EOL + statusMessageLine + os.EOL);
-    const binPath = getBinPath(packageInstallFolder, packageBinName);
-    const result = childProcess.spawnSync(binPath, packageBinArgs, {
-        stdio: 'inherit',
-        cwd: process.cwd(),
-        env: process.env
-    });
-    return result.status;
-}
-exports.installAndRun = installAndRun;
-function runWithErrorAndStatusCode(fn) {
-    process.exitCode = 1;
-    try {
-        const exitCode = fn();
-        process.exitCode = exitCode;
-    }
-    catch (e) {
-        console.error(os.EOL + os.EOL + e.toString() + os.EOL + os.EOL);
-    }
-}
-exports.runWithErrorAndStatusCode = runWithErrorAndStatusCode;
-function run() {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ rawPackageSpecifier, /* qrcode@^1.2.0 */ packageBinName, /* qrcode */ ...packageBinArgs /* [-f, myproject/lib] */] = process.argv;
-    if (!nodePath) {
-        throw new Error('Unexpected exception: could not detect node path');
-    }
-    if (path.basename(scriptPath).toLowerCase() !== 'install-run.js') {
-        // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control
-        // to the script that (presumably) imported this file
-        return;
-    }
-    if (process.argv.length < 4) {
-        console.log('Usage: install-run.js <package>@<version> <command> [args...]');
-        console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io');
-        process.exit(1);
-    }
-    runWithErrorAndStatusCode(() => {
-        const rushJsonFolder = findRushJsonFolder();
-        const rushCommonFolder = ensureAndJoinPath(rushJsonFolder, 'common');
-        const packageSpecifier = parsePackageSpecifier(rawPackageSpecifier);
-        const name = packageSpecifier.name;
-        const version = resolvePackageVersion(rushCommonFolder, packageSpecifier);
-        if (packageSpecifier.version !== version) {
-            console.log(`Resolved to ${name}@${version}`);
-        }
-        return installAndRun(name, version, packageBinName, packageBinArgs);
-    });
-}
-run();
+"use strict";
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See the @microsoft/rush package's LICENSE file for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
+//
+// This script is intended for usage in an automated build environment where a Node tool may not have
+// been preinstalled, or may have an unpredictable version.  This script will automatically install the specified
+// version of the specified tool (if not already installed), and then pass a command-line to it.
+// An example usage would be:
+//
+//    node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io
+//
+// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
+const childProcess = require("child_process");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+exports.RUSH_JSON_FILENAME = 'rush.json';
+const RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME = 'RUSH_TEMP_FOLDER';
+const INSTALLED_FLAG_FILENAME = 'installed.flag';
+const NODE_MODULES_FOLDER_NAME = 'node_modules';
+const PACKAGE_JSON_FILENAME = 'package.json';
+/**
+ * Parse a package specifier (in the form of name\@version) into name and version parts.
+ */
+function _parsePackageSpecifier(rawPackageSpecifier) {
+    rawPackageSpecifier = (rawPackageSpecifier || '').trim();
+    const separatorIndex = rawPackageSpecifier.lastIndexOf('@');
+    let name;
+    let version = undefined;
+    if (separatorIndex === 0) {
+        // The specifier starts with a scope and doesn't have a version specified
+        name = rawPackageSpecifier;
+    }
+    else if (separatorIndex === -1) {
+        // The specifier doesn't have a version
+        name = rawPackageSpecifier;
+    }
+    else {
+        name = rawPackageSpecifier.substring(0, separatorIndex);
+        version = rawPackageSpecifier.substring(separatorIndex + 1);
+    }
+    if (!name) {
+        throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`);
+    }
+    return { name, version };
+}
+/**
+ * As a workaround, copyAndTrimNpmrcFile() copies the .npmrc file to the target folder, and also trims
+ * unusable lines from the .npmrc file.
+ *
+ * Why are we trimming the .npmrc lines?  NPM allows environment variables to be specified in
+ * the .npmrc file to provide different authentication tokens for different registry.
+ * However, if the environment variable is undefined, it expands to an empty string, which
+ * produces a valid-looking mapping with an invalid URL that causes an error.  Instead,
+ * we'd prefer to skip that line and continue looking in other places such as the user's
+ * home directory.
+ *
+ * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._copyNpmrcFile()
+ */
+function _copyAndTrimNpmrcFile(sourceNpmrcPath, targetNpmrcPath) {
+    console.log(`Copying ${sourceNpmrcPath} --> ${targetNpmrcPath}`); // Verbose
+    let npmrcFileLines = fs.readFileSync(sourceNpmrcPath).toString().split('\n');
+    npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim());
+    const resultLines = [];
+    // Trim out lines that reference environment variables that aren't defined
+    for (const line of npmrcFileLines) {
+        // This finds environment variable tokens that look like "${VAR_NAME}"
+        const regex = /\$\{([^\}]+)\}/g;
+        const environmentVariables = line.match(regex);
+        let lineShouldBeTrimmed = false;
+        if (environmentVariables) {
+            for (const token of environmentVariables) {
+                // Remove the leading "${" and the trailing "}" from the token
+                const environmentVariableName = token.substring(2, token.length - 1);
+                if (!process.env[environmentVariableName]) {
+                    lineShouldBeTrimmed = true;
+                    break;
+                }
+            }
+        }
+        if (lineShouldBeTrimmed) {
+            // Example output:
+            // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}"
+            resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line);
+        }
+        else {
+            resultLines.push(line);
+        }
+    }
+    fs.writeFileSync(targetNpmrcPath, resultLines.join(os.EOL));
+}
+/**
+ * syncNpmrc() copies the .npmrc file to the target folder, and also trims unusable lines from the .npmrc file.
+ * If the source .npmrc file not exist, then syncNpmrc() will delete an .npmrc that is found in the target folder.
+ *
+ * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc()
+ */
+function _syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder, useNpmrcPublish) {
+    const sourceNpmrcPath = path.join(sourceNpmrcFolder, !useNpmrcPublish ? '.npmrc' : '.npmrc-publish');
+    const targetNpmrcPath = path.join(targetNpmrcFolder, '.npmrc');
+    try {
+        if (fs.existsSync(sourceNpmrcPath)) {
+            _copyAndTrimNpmrcFile(sourceNpmrcPath, targetNpmrcPath);
+        }
+        else if (fs.existsSync(targetNpmrcPath)) {
+            // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target
+            console.log(`Deleting ${targetNpmrcPath}`); // Verbose
+            fs.unlinkSync(targetNpmrcPath);
+        }
+    }
+    catch (e) {
+        throw new Error(`Error syncing .npmrc file: ${e}`);
+    }
+}
+let _npmPath = undefined;
+/**
+ * Get the absolute path to the npm executable
+ */
+function getNpmPath() {
+    if (!_npmPath) {
+        try {
+            if (os.platform() === 'win32') {
+                // We're on Windows
+                const whereOutput = childProcess.execSync('where npm', { stdio: [] }).toString();
+                const lines = whereOutput.split(os.EOL).filter((line) => !!line);
+                // take the last result, we are looking for a .cmd command
+                // see https://github.com/microsoft/rushstack/issues/759
+                _npmPath = lines[lines.length - 1];
+            }
+            else {
+                // We aren't on Windows - assume we're on *NIX or Darwin
+                _npmPath = childProcess.execSync('which npm', { stdio: [] }).toString();
+            }
+        }
+        catch (e) {
+            throw new Error(`Unable to determine the path to the NPM tool: ${e}`);
+        }
+        _npmPath = _npmPath.trim();
+        if (!fs.existsSync(_npmPath)) {
+            throw new Error('The NPM executable does not exist');
+        }
+    }
+    return _npmPath;
+}
+exports.getNpmPath = getNpmPath;
+function _ensureFolder(folderPath) {
+    if (!fs.existsSync(folderPath)) {
+        const parentDir = path.dirname(folderPath);
+        _ensureFolder(parentDir);
+        fs.mkdirSync(folderPath);
+    }
+}
+/**
+ * Create missing directories under the specified base directory, and return the resolved directory.
+ *
+ * Does not support "." or ".." path segments.
+ * Assumes the baseFolder exists.
+ */
+function _ensureAndJoinPath(baseFolder, ...pathSegments) {
+    let joinedPath = baseFolder;
+    try {
+        for (let pathSegment of pathSegments) {
+            pathSegment = pathSegment.replace(/[\\\/]/g, '+');
+            joinedPath = path.join(joinedPath, pathSegment);
+            if (!fs.existsSync(joinedPath)) {
+                fs.mkdirSync(joinedPath);
+            }
+        }
+    }
+    catch (e) {
+        throw new Error(`Error building local installation folder (${path.join(baseFolder, ...pathSegments)}): ${e}`);
+    }
+    return joinedPath;
+}
+function _getRushTempFolder(rushCommonFolder) {
+    const rushTempFolder = process.env[RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME];
+    if (rushTempFolder !== undefined) {
+        _ensureFolder(rushTempFolder);
+        return rushTempFolder;
+    }
+    else {
+        return _ensureAndJoinPath(rushCommonFolder, 'temp');
+    }
+}
+/**
+ * Resolve a package specifier to a static version
+ */
+function _resolvePackageVersion(rushCommonFolder, { name, version }) {
+    if (!version) {
+        version = '*'; // If no version is specified, use the latest version
+    }
+    if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) {
+        // If the version contains only characters that we recognize to be used in static version specifiers,
+        // pass the version through
+        return version;
+    }
+    else {
+        // version resolves to
+        try {
+            const rushTempFolder = _getRushTempFolder(rushCommonFolder);
+            const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
+            _syncNpmrc(sourceNpmrcFolder, rushTempFolder);
+            const npmPath = getNpmPath();
+            // This returns something that looks like:
+            //  @microsoft/rush@3.0.0 '3.0.0'
+            //  @microsoft/rush@3.0.1 '3.0.1'
+            //  ...
+            //  @microsoft/rush@3.0.20 '3.0.20'
+            //  <blank line>
+            const npmVersionSpawnResult = childProcess.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], {
+                cwd: rushTempFolder,
+                stdio: []
+            });
+            if (npmVersionSpawnResult.status !== 0) {
+                throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`);
+            }
+            const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString();
+            const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line);
+            const latestVersion = versionLines[versionLines.length - 1];
+            if (!latestVersion) {
+                throw new Error('No versions found for the specified version range.');
+            }
+            const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/);
+            if (!versionMatches) {
+                throw new Error(`Invalid npm output ${latestVersion}`);
+            }
+            return versionMatches[1];
+        }
+        catch (e) {
+            throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`);
+        }
+    }
+}
+let _rushJsonFolder;
+/**
+ * Find the absolute path to the folder containing rush.json
+ */
+function findRushJsonFolder() {
+    if (!_rushJsonFolder) {
+        let basePath = __dirname;
+        let tempPath = __dirname;
+        do {
+            const testRushJsonPath = path.join(basePath, exports.RUSH_JSON_FILENAME);
+            if (fs.existsSync(testRushJsonPath)) {
+                _rushJsonFolder = basePath;
+                break;
+            }
+            else {
+                basePath = tempPath;
+            }
+        } while (basePath !== (tempPath = path.dirname(basePath))); // Exit the loop when we hit the disk root
+        if (!_rushJsonFolder) {
+            throw new Error('Unable to find rush.json.');
+        }
+    }
+    return _rushJsonFolder;
+}
+exports.findRushJsonFolder = findRushJsonFolder;
+/**
+ * Detects if the package in the specified directory is installed
+ */
+function _isPackageAlreadyInstalled(packageInstallFolder) {
+    try {
+        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
+        if (!fs.existsSync(flagFilePath)) {
+            return false;
+        }
+        const fileContents = fs.readFileSync(flagFilePath).toString();
+        return fileContents.trim() === process.version;
+    }
+    catch (e) {
+        return false;
+    }
+}
+/**
+ * Removes the following files and directories under the specified folder path:
+ *  - installed.flag
+ *  -
+ *  - node_modules
+ */
+function _cleanInstallFolder(rushTempFolder, packageInstallFolder) {
+    try {
+        const flagFile = path.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME);
+        if (fs.existsSync(flagFile)) {
+            fs.unlinkSync(flagFile);
+        }
+        const packageLockFile = path.resolve(packageInstallFolder, 'package-lock.json');
+        if (fs.existsSync(packageLockFile)) {
+            fs.unlinkSync(packageLockFile);
+        }
+        const nodeModulesFolder = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME);
+        if (fs.existsSync(nodeModulesFolder)) {
+            const rushRecyclerFolder = _ensureAndJoinPath(rushTempFolder, 'rush-recycler', `install-run-${Date.now().toString()}`);
+            fs.renameSync(nodeModulesFolder, rushRecyclerFolder);
+        }
+    }
+    catch (e) {
+        throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`);
+    }
+}
+function _createPackageJson(packageInstallFolder, name, version) {
+    try {
+        const packageJsonContents = {
+            'name': 'ci-rush',
+            'version': '0.0.0',
+            'dependencies': {
+                [name]: version
+            },
+            'description': 'DON\'T WARN',
+            'repository': 'DON\'T WARN',
+            'license': 'MIT'
+        };
+        const packageJsonPath = path.join(packageInstallFolder, PACKAGE_JSON_FILENAME);
+        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2));
+    }
+    catch (e) {
+        throw new Error(`Unable to create package.json: ${e}`);
+    }
+}
+/**
+ * Run "npm install" in the package install folder.
+ */
+function _installPackage(packageInstallFolder, name, version) {
+    try {
+        console.log(`Installing ${name}...`);
+        const npmPath = getNpmPath();
+        const result = childProcess.spawnSync(npmPath, ['install'], {
+            stdio: 'inherit',
+            cwd: packageInstallFolder,
+            env: process.env
+        });
+        if (result.status !== 0) {
+            throw new Error('"npm install" encountered an error');
+        }
+        console.log(`Successfully installed ${name}@${version}`);
+    }
+    catch (e) {
+        throw new Error(`Unable to install package: ${e}`);
+    }
+}
+/**
+ * Get the ".bin" path for the package.
+ */
+function _getBinPath(packageInstallFolder, binName) {
+    const binFolderPath = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin');
+    const resolvedBinName = (os.platform() === 'win32') ? `${binName}.cmd` : binName;
+    return path.resolve(binFolderPath, resolvedBinName);
+}
+/**
+ * Write a flag file to the package's install directory, signifying that the install was successful.
+ */
+function _writeFlagFile(packageInstallFolder) {
+    try {
+        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
+        fs.writeFileSync(flagFilePath, process.version);
+    }
+    catch (e) {
+        throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`);
+    }
+}
+function installAndRun(packageName, packageVersion, packageBinName, packageBinArgs) {
+    const rushJsonFolder = findRushJsonFolder();
+    const rushCommonFolder = path.join(rushJsonFolder, 'common');
+    const rushTempFolder = _getRushTempFolder(rushCommonFolder);
+    const packageInstallFolder = _ensureAndJoinPath(rushTempFolder, 'install-run', `${packageName}@${packageVersion}`);
+    if (!_isPackageAlreadyInstalled(packageInstallFolder)) {
+        // The package isn't already installed
+        _cleanInstallFolder(rushTempFolder, packageInstallFolder);
+        const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
+        _syncNpmrc(sourceNpmrcFolder, packageInstallFolder);
+        _createPackageJson(packageInstallFolder, packageName, packageVersion);
+        _installPackage(packageInstallFolder, packageName, packageVersion);
+        _writeFlagFile(packageInstallFolder);
+    }
+    const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`;
+    const statusMessageLine = new Array(statusMessage.length + 1).join('-');
+    console.log(os.EOL + statusMessage + os.EOL + statusMessageLine + os.EOL);
+    const binPath = _getBinPath(packageInstallFolder, packageBinName);
+    const result = childProcess.spawnSync(binPath, packageBinArgs, {
+        stdio: 'inherit',
+        cwd: process.cwd(),
+        env: process.env
+    });
+    if (result.status !== null) {
+        return result.status;
+    }
+    else {
+        throw result.error || new Error('An unknown error occurred.');
+    }
+}
+exports.installAndRun = installAndRun;
+function runWithErrorAndStatusCode(fn) {
+    process.exitCode = 1;
+    try {
+        const exitCode = fn();
+        process.exitCode = exitCode;
+    }
+    catch (e) {
+        console.error(os.EOL + os.EOL + e.toString() + os.EOL + os.EOL);
+    }
+}
+exports.runWithErrorAndStatusCode = runWithErrorAndStatusCode;
+function _run() {
+    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ rawPackageSpecifier, /* qrcode@^1.2.0 */ packageBinName, /* qrcode */ ...packageBinArgs /* [-f, myproject/lib] */] = process.argv;
+    if (!nodePath) {
+        throw new Error('Unexpected exception: could not detect node path');
+    }
+    if (path.basename(scriptPath).toLowerCase() !== 'install-run.js') {
+        // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control
+        // to the script that (presumably) imported this file
+        return;
+    }
+    if (process.argv.length < 4) {
+        console.log('Usage: install-run.js <package>@<version> <command> [args...]');
+        console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io');
+        process.exit(1);
+    }
+    runWithErrorAndStatusCode(() => {
+        const rushJsonFolder = findRushJsonFolder();
+        const rushCommonFolder = _ensureAndJoinPath(rushJsonFolder, 'common');
+        const packageSpecifier = _parsePackageSpecifier(rawPackageSpecifier);
+        const name = packageSpecifier.name;
+        const version = _resolvePackageVersion(rushCommonFolder, packageSpecifier);
+        if (packageSpecifier.version !== version) {
+            console.log(`Resolved to ${name}@${version}`);
+        }
+        return installAndRun(name, version, packageBinName, packageBinArgs);
+    });
+}
+_run();
 //# sourceMappingURL=install-run.js.map

configs/typescript-config/common.tsconfig.json

@@ -27,4 +27,4 @@
     "../../test",
     "../../**/view"
   ]
-}
+}

extensions/ql-vscode/.eslintrc.js

@@ -0,0 +1,34 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    ecmaVersion: 2018,
+    sourceType: 'module',
+    project: ['tsconfig.json', './src/**/tsconfig.json'],
+  },
+  plugins: ['@typescript-eslint'],
+  env: {
+    node: true,
+    es6: true
+  },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+  rules: {
+    '@typescript-eslint/no-use-before-define': 0,
+    '@typescript-eslint/no-unused-vars': ["warn", {
+      "vars": "all",
+      "args": "none",
+      "ignoreRestSiblings": false
+    }],
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/no-non-null-assertion": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "prefer-const": ["warn", {"destructuring": "all"}],
+    "indent": "off",
+    "@typescript-eslint/indent": "off",
+    "@typescript-eslint/no-throw-literal": "error",
+    "no-useless-escape": 0,
+    "semi": 2
+  },
+};

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,79 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.2.2 - 8 June 2020
+
+- Fix auto-indentation rules.
+- Add ability to download platform-specific releases of the CodeQL CLI if they are available.
+- Fix handling of downloading prerelease versions of the CodeQL CLI.
+- Add pagination for displaying non-interpreted results.
+
+## 1.2.1 - 29 May 2020
+
+- Better formatting and autoindentation when adding QLDoc comments to `.ql` and `.qll` files.
+- Allow for more flexibility when opening a database in the workspace. A user can now choose the actual database folder, or the nested `db-*` folder.
+- Add query history menu command for viewing corresponding SARIF file.
+- Add ability for users to download databases directly from LGTM.com.
+
+## 1.2.0 - 19 May 2020
+
+- Enable 'Go to Definition' and 'Go to References' on source archive
+  files in CodeQL databases. This is handled by a CodeQL query.
+- Fix adding database archive files on Windows.
+- Enable adding remote and local database archive files from the
+  command palette.
+
+## 1.1.5 - 15 May 2020
+
+- Links in results are no longer underlined and monospaced.
+- Add the ability to choose a database either from an archive, a folder, or from the internet.
+- New icons for commands on the databases view.
+
+## 1.1.4 - 13 May 2020
+
+- Add the ability to download and install databases archives from the internet.
+
+## 1.1.3 - 8 May 2020
+
+- Add a suggestion in alerts view to view raw results, when there are
+  raw results but no alerts.
+- Add the ability to rename databases in the database view.
+- Add the ability to open the directory in the filesystem
+  of a database.
+
+## 1.1.2 - 28 April 2020
+
+- Implement syntax highlighting for the new `unique` aggregate.
+- Implement XML syntax highlighting for `.qhelp` files.
+- Add option to auto save queries before running them.
+- Add new command in query history to view the query text of the
+  selected query (note that this may be different from the current
+  contents of the query file if the file has been edited).
+- Add ability to sort CodeQL databases by name or by date added.
+
+## 1.1.1 - 23 March 2020
+
+- Fix quick evaluation in `.qll` files.
+- Add new command in query history view to view the log file of a
+  query.
+- Request user acknowledgment before updating the CodeQL binaries.
+- Warn when using the deprecated `codeql.cmd` launcher on Windows.
+
+## 1.1.0 - 17 March 2020
+
+- Add functionality for testing custom CodeQL queries by using the VS
+  Code Test Explorer extension and `codeql test`. See the documentation for
+  more details.
+- Add a "Show log" button to all information, error, and warning
+  popups that will display the CodeQL extension log.
+- Display a message when a query times out.
+- Show canceled queries in query history.
+- Improve error messages when attempting to run non-query files.
+
+## 1.0.6 - 28 February 2020
+
+- Add command to restart query server.
+- Enable support for future minor upgrades to the CodeQL CLI.
+
 ## 1.0.5 - 13 February 2020
 
 - Add an icon next to any failed query runs in the query history
@@ -15,7 +89,7 @@
 ## 1.0.3 - 13 January 2020
 
 - Reduce the frequency of CodeQL CLI update checks to help avoid hitting GitHub API limits of 60 requests per
-hour for unauthenticated IPs.
+  hour for unauthenticated IPs.
 - Fix sorting of result sets with names containing special characters.
 
 ## 1.0.2 - 13 December 2019
@@ -24,8 +98,7 @@ hour for unauthenticated IPs.
 - Allow customization of query history labels from settings and from
   query history view context menu.
 - Show number of results in results view.
-- Add commands `CodeQL: Show Next Step on Path` and `CodeQL: Show
-  Previous Step on Path` for navigating the steps on the currently
+- Add commands `CodeQL: Show Next Step on Path` and `CodeQL: Show Previous Step on Path` for navigating the steps on the currently
   shown path result.
 
 ## 1.0.1 - 21 November 2019
@@ -36,7 +109,6 @@ hour for unauthenticated IPs.
 - Fix the automatic upgrading of CodeQL databases when using upgrade scripts from the workspace.
 - Allow removal of items from the CodeQL Query History view.
 
-
 ## 1.0.0 - 14 November 2019
 
 Initial release of CodeQL for Visual Studio Code.

extensions/ql-vscode/README.md

@@ -2,10 +2,10 @@
 
 This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
 
-* Enables you to use CodeQL to query databases generated from source code.
-* Shows the flow of data through the results of path queries, which is essential for triaging security results.
-* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/Semmle/ql).
-* Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
+- Enables you to use CodeQL to query databases generated from source code.
+- Shows the flow of data through the results of path queries, which is essential for triaging security results.
+- Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
+- Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
 To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
 
@@ -14,18 +14,18 @@ To see what has changed in the last few versions of the extension, see the [Chan
 The information in this `README` file describes the quickest way to start using CodeQL.
 For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
 
-**Quick start: Installing and configuring the extension**
+### Quick start: Installing and configuring the extension
 
 1. [Install the extension](#installing-the-extension).
 1. [Check access to the CodeQL CLI](#checking-access-to-the-codeql-cli).
 1. [Clone the CodeQL starter workspace](#cloning-the-codeql-starter-workspace).
 
-**Quick start: Using CodeQL**
+### Quick start: Using CodeQL
 
 1. [Import a database from LGTM](#importing-a-database-from-lgtm).
 1. [Run a query](#running-a-query).
 
------
+---
 
 ## Quick start: Installing and configuring the extension
 
@@ -49,11 +49,26 @@ If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Lo
 ### Cloning the CodeQL starter workspace
 
 When you're working with CodeQL, you need access to the standard CodeQL libraries and queries.
-Initially, we recommend that you clone and use the ready-to-use starter workspace, https://github.com/github/vscode-codeql-starter/.
+Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
 For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
 
+## Upgrading CodeQL standard libraries
+
+You can easily keep up-to-date with the latest changes to the [CodeQL standard libraries](https://github.com/github/codeql).
+
+If you're using the [CodeQL starter workspace](https://github.com/github/vscode-codeql-starter/), you can pull in the latest standard libraries by running:
+
+```shell
+git pull
+git submodule update --recursive
+```
+
+in the starter workspace directory.
+
+If you're using your own clone of the CodeQL standard libraries, you can do a `git pull` from where you have the libraries checked out.
+
 ## Quick start: Using CodeQL
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.
@@ -62,16 +77,13 @@ You can find all the commands contributed by the extension in the Command Palett
 
 While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
 
-1. Log in to LGTM.com.
-1. Find a project you're interested in and display the **Integrations** tab (for example, [Apache Kafka](https://lgtm.com/projects/g/apache/kafka/ci/)).
-1. Scroll to the **CodeQL databases for local analysis** section at the bottom of the page.
-1. Download databases for the languages that you want to explore.
-1. Unzip the databases.
-1. For each database that you want to import:
-    1. In the VS Code sidebar, go to **CodeQL** > **Databases** and click **+**.
-    1. Browse to the unzipped database folder (the parent folder that contains `db-<language>` and `src`) and select **Choose database** to add it.
-
-When the import is complete, each CodeQL database is displayed in the CodeQL sidebar under **Databases**.
+1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
+1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
+1. Copy the link to that project, for example `https://lgtm.com/projects/g/apache/kafka`.
+1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from LGTM** command.
+1. Paste the link you copied earlier.
+1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
+1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
 ### Running a query
 
@@ -79,7 +91,7 @@ The instructions below assume that you're using the CodeQL starter workspace, or
 
 1. Expand the `ql` folder and locate a query to run. The standard queries are grouped by target language and then type, for example: `ql/java/ql/src/Likely Bugs`.
 1. Open a query (`.ql`) file.
-3. Right-click in the query window and select **CodeQL: Run Query**. Alternatively, open the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**), type `Run Query`, then select **CodeQL: Run Query**.
+1. Right-click in the query window and select **CodeQL: Run Query**. Alternatively, open the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**), type `Run Query`, then select **CodeQL: Run Query**.
 
 The CodeQL extension runs the query on the current database using the CLI and reports progress in the bottom right corner of the application.
 When the results are ready, they're displayed in the CodeQL Query Results view. Use the dropdown menu to choose between different forms of result output.
@@ -90,10 +102,10 @@ If there are any problems running a query, a notification is displayed in the bo
 
 For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
 
-* [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
-* [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-* [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
-* [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
+- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
+- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
 

extensions/ql-vscode/gulpfile.js/webpack.config.ts

@@ -10,7 +10,7 @@ export const config: webpack.Configuration = {
     path: path.resolve(__dirname, '..', 'out'),
     filename: "[name].js"
   },
-  devtool: 'source-map',
+  devtool: "inline-source-map",
   resolve: {
     extensions: ['.js', '.ts', '.tsx', '.json']
   },

extensions/ql-vscode/gulpfile.js/webpack.ts

@@ -15,7 +15,8 @@ export function compileView(cb: (err?: Error) => void) {
       hash: false,
       entrypoints: false,
       timings: false,
-      modules: false
+      modules: false,
+      errors: true
     }));
     if (stats.hasErrors()) {
       cb(new Error('Compilation errors detected.'));

extensions/ql-vscode/language-configuration.json

@@ -1,72 +1,34 @@
 {
-    "comments": {
-        // symbol used for single line comment. Remove this entry if your language does not support line comments
-        "lineComment": "//",
-        // symbols used for start and end a block comment. Remove this entry if your language does not support block comments
-        "blockComment": [
-            "/*",
-            "*/"
-        ]
-    },
-    // symbols used as brackets
-    "brackets": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ]
-    ],
-    // symbols that are auto closed when typing
-    "autoClosingPairs": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ],
-        [
-            "\"",
-            "\""
-        ],
-        [
-            "'",
-            "'"
-        ]
-    ],
-    // symbols that that can be used to surround a selection
-    "surroundingPairs": [
-        [
-            "{",
-            "}"
-        ],
-        [
-            "[",
-            "]"
-        ],
-        [
-            "(",
-            ")"
-        ],
-        [
-            "\"",
-            "\""
-        ],
-        [
-            "'",
-            "'"
-        ]
-    ]
-}
+  "comments": {
+    "lineComment": "//",
+    "blockComment": ["/*", "*/"]
+  },
+  "brackets": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"]
+  ],
+  "autoClosingPairs": [
+    { "open": "{", "close": "}" },
+    { "open": "[", "close": "]" },
+    { "open": "(", "close": ")" },
+    { "open": "'", "close": "'", "notIn": ["string", "comment"] },
+    { "open": "\"", "close": "\"", "notIn": ["string"] },
+    { "open": "/**", "close": " */", "notIn": ["string"] }
+  ],
+  "autoCloseBefore": ";:.=}])> \n\t",
+  "surroundingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    ["'", "'"],
+    ["\"", "\""]
+  ],
+  "folding": {
+    "markers": {
+      "start": "^\\s*//\\s*#?region\\b",
+      "end": "^\\s*//\\s*#?endregion\\b"
+    }
+  },
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+}

extensions/ql-vscode/media/dark/archive-plus.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M14.5 1H1.5L1 1.5V4.5L1.5 5H2V13.5L2.5 14H13.5L14 13.5V5H14.5L15 4.5V1.5L14.5 1ZM13.5 4H2.5H2V2H14V4H13.5ZM3 13V5H13V13H3ZM11 7H5V8H11V7Z" fill="#C5C5C5"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/dark/cloud-download.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9565 6H12.0064C12.8004 6 13.5618 6.31607 14.1232 6.87868C14.6846 7.44129 15 8.20435 15 9C15 9.79565 14.6846 10.5587 14.1232 11.1213C13.5618 11.6839 12.8004 12 12.0064 12V11C12.5357 11 13.0434 10.7893 13.4176 10.4142C13.7919 10.0391 14.0021 9.53044 14.0021 9C14.0021 8.46957 13.7919 7.96086 13.4176 7.58579C13.0434 7.21072 12.5357 7 12.0064 7H11.0924L10.9687 6.143C10.8938 5.60541 10.6456 5.10711 10.2618 4.72407C9.87801 4.34103 9.37977 4.09427 8.84303 4.02143C8.30629 3.94859 7.76051 4.05365 7.2889 4.3206C6.81729 4.58754 6.44573 5.00173 6.23087 5.5L5.89759 6.262L5.08933 6.073C4.90382 6.02699 4.71364 6.0025 4.52255 6C3.86093 6 3.22641 6.2634 2.75858 6.73224C2.29075 7.20108 2.02792 7.83696 2.02792 8.5C2.02792 9.16304 2.29075 9.79893 2.75858 10.2678C3.22641 10.7366 3.86093 11 4.52255 11H5.02148V12H4.52255C4.02745 12.0043 3.5371 11.903 3.08403 11.7029C2.63096 11.5028 2.22553 11.2084 1.89461 10.8394C1.5637 10.4703 1.31488 10.0349 1.16465 9.56211C1.01442 9.08932 0.966217 8.58992 1.02324 8.09704C1.08026 7.60416 1.24121 7.12906 1.4954 6.70326C1.74959 6.27745 2.09121 5.91068 2.49762 5.62727C2.90402 5.34385 3.36591 5.15027 3.85264 5.05937C4.33938 4.96847 4.83984 4.98232 5.32083 5.1C5.6241 4.40501 6.14511 3.82799 6.80496 3.45635C7.4648 3.08472 8.22753 2.9387 8.9776 3.04044C9.72768 3.14217 10.4242 3.4861 10.9618 4.02014C11.4993 4.55418 11.8485 5.24923 11.9565 6ZM6.70719 11.1214L8.0212 12.4354V7H9.01506V12.3992L10.2929 11.1214L11 11.8285L8.85356 13.9749H8.14645L6.00008 11.8285L6.70719 11.1214Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/folder-opened-plus.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0.5 14H12.5L12.98 13.63L15.61 6.63L15.13 6H13V3.5L12.5 3H6.70996L5.84998 2.15002L5.5 2H0.5L0 2.5V13.5L0.5 14ZM1 3H5.29004L6.15002 3.84998L6.5 4H12V6H8.5L8.15002 6.15002L7.29004 7H2.5L2.03003 7.33997L1.03003 10.42L1 3ZM12.13 13H1.18994L2.85999 8H7.5L7.84998 7.84998L8.70996 7H14.5L12.13 13Z" fill="#C5C5C5"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/dark/lgtm-plus.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none">
+  <path d="M16.010 6.49c-3.885 0-7.167 0.906-9.328 2.813-0.063-0.12-0.109-0.219-0.188-0.339-0.224-0.365-0.438-0.776-1.104-1.188-0.411-0.26-0.87-0.438-1.349-0.516-0.208-0.021-0.422-0.021-0.63 0l0.135-0.016c-1.214 0-1.922 0.724-2.385 1.354-0.458 0.625-0.755 1.328-0.948 2.099-0.38 1.542-0.385 3.536 1.083 5.026 0.766 0.781 1.667 1.151 2.484 1.37 0.156 0.042 0.297 0.052 0.448 0.083 0.531 2.521 2.104 4.656 4.208 5.839v0.005c1.24 0.693 2.417 1.010 3.297 1.349 1.234 0.479 2.536 1 4.052 1.135l0.078 0.005h0.198c1.745 0 3.063-0.703 4.203-1.141 0.875-0.333 2.052-0.641 3.302-1.344 0.578-0.323 1.115-0.719 1.594-1.172 1.318-1.234 2.229-2.839 2.625-4.599 1.115-0.182 2.141-0.719 2.922-1.536 1.464-1.484 1.458-3.479 1.078-5.021-0.193-0.771-0.49-1.474-0.948-2.099-0.458-0.63-1.172-1.354-2.385-1.354l0.135 0.016c-0.208-0.021-0.422-0.021-0.63 0-0.479 0.078-0.938 0.255-1.344 0.516-0.667 0.411-0.88 0.823-1.104 1.182-0.073 0.12-0.12 0.219-0.188 0.333-2.156-1.901-5.432-2.802-9.313-2.802zM16.042 8.313c4.745 0 8.016 1.422 9.411 3.964 0.839-0.323 1.453-2.521 2.146-2.948 0.563-0.344 0.885-0.26 0.885-0.26 1.271 0 2.578 3.729 0.953 5.38-0.859 0.875-2.443 1.12-3.229 1.057-0.063 2.542-1.542 4.833-3.5 5.932-1 0.563-2.068 0.854-3.063 1.234-1.229 0.469-2.38 1.016-3.547 1.016h-0.125c-1.161-0.099-2.318-0.542-3.547-1.016-0.995-0.38-2.068-0.682-3.063-1.24-1.948-1.099-3.427-3.391-3.49-5.927-0.781 0.068-2.385-0.177-3.245-1.057-1.625-1.651-0.318-5.38 0.948-5.38 0 0 0.328-0.083 0.885 0.26 0.698 0.427 1.318 2.646 2.161 2.953 1.391-2.547 4.667-3.969 9.417-3.969zM10.875 11.422c-2.276-0.042-4.146 1.792-4.146 4.068 0 2.281 1.87 4.115 4.146 4.073 5.328-0.099 5.328-8.047 0-8.141zM21.208 11.422c-5.427 0-5.427 8.141 0 8.141s5.427-8.141 0-8.141zM11.453 13.708c2.349 0.063 2.349 3.552 0 3.615-1.182 0-2.042-1.115-1.75-2.255 0.318 0.771 1.469 0.547 1.464-0.292 0-0.406-0.318-0.745-0.729-0.76 0.302-0.203 0.656-0.313 1.016-0.307zM20.641 13.708c2.344 0.063 2.344 3.552 0 3.615-1.182 0-2.047-1.115-1.755-2.255 0.229 0.552 0.979 0.641 1.328 0.146 0.344-0.49 0.010-1.167-0.589-1.193 0.297-0.208 0.651-0.313 1.016-0.313zM15.359 19.906c-0.318 0.026-0.5 0.193-0.5 0.635 0 0.281 0.182 0.484 0.5 0.484 0.229 0 0.266-0.323 0.047-0.375-0.031-0.005-0.172-0.057-0.172-0.182 0-0.12 0-0.167 0.24-0.198 0.104-0.016 0.156-0.141 0.125-0.24s-0.125-0.135-0.24-0.125zM16.724 19.906c-0.115-0.005-0.208 0.026-0.24 0.125s0.021 0.224 0.125 0.24c0.24 0.031 0.24 0.078 0.24 0.198 0 0.125-0.141 0.177-0.172 0.182-0.219 0.052-0.182 0.375 0.042 0.375 0.323 0 0.51-0.203 0.51-0.484 0-0.443-0.188-0.609-0.505-0.635z" fill="#C5C5C5"/>
+  <line y2="24" x2="16" y1="26" x1="32" stroke-width="2" stroke="green" fill="none"/>
+  <line y2="16" x2="24" y1="32" x1="24" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/dark/sort-alpha.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
+			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72
+				250.027,338.24 432,338.24 432,304.32" fill="#C5C5C5"/>
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
+			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
+				L113.28,140.48l41.387,110.507H71.893z" fill="#C5C5C5"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/dark/sort-date.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/light/archive-plus.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M14.5 1H1.5L1 1.5V4.5L1.5 5H2V13.5L2.5 14H13.5L14 13.5V5H14.5L15 4.5V1.5L14.5 1ZM13.5 4H2.5H2V2H14V4H13.5ZM3 13V5H13V13H3ZM11 7H5V8H11V7Z" fill="#424242"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/light/cloud-download.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9565 6H12.0064C12.8004 6 13.5618 6.31607 14.1232 6.87868C14.6846 7.44129 15 8.20435 15 9C15 9.79565 14.6846 10.5587 14.1232 11.1213C13.5618 11.6839 12.8004 12 12.0064 12V11C12.5357 11 13.0434 10.7893 13.4176 10.4142C13.7919 10.0391 14.0021 9.53044 14.0021 9C14.0021 8.46957 13.7919 7.96086 13.4176 7.58579C13.0434 7.21072 12.5357 7 12.0064 7H11.0924L10.9687 6.143C10.8938 5.60541 10.6456 5.10711 10.2618 4.72407C9.87801 4.34103 9.37977 4.09427 8.84303 4.02143C8.30629 3.94859 7.76051 4.05365 7.2889 4.3206C6.81729 4.58754 6.44573 5.00173 6.23087 5.5L5.89759 6.262L5.08933 6.073C4.90382 6.02699 4.71364 6.0025 4.52255 6C3.86093 6 3.22641 6.2634 2.75858 6.73224C2.29075 7.20108 2.02792 7.83696 2.02792 8.5C2.02792 9.16304 2.29075 9.79893 2.75858 10.2678C3.22641 10.7366 3.86093 11 4.52255 11H5.02148V12H4.52255C4.02745 12.0043 3.5371 11.903 3.08403 11.7029C2.63096 11.5028 2.22553 11.2084 1.89461 10.8394C1.5637 10.4703 1.31488 10.0349 1.16465 9.56211C1.01442 9.08932 0.966217 8.58992 1.02324 8.09704C1.08026 7.60416 1.24121 7.12906 1.4954 6.70326C1.74959 6.27745 2.09121 5.91068 2.49762 5.62727C2.90402 5.34385 3.36591 5.15027 3.85264 5.05937C4.33938 4.96847 4.83984 4.98232 5.32083 5.1C5.6241 4.40501 6.14511 3.82799 6.80496 3.45635C7.4648 3.08472 8.22753 2.9387 8.9776 3.04044C9.72768 3.14217 10.4242 3.4861 10.9618 4.02014C11.4993 4.55418 11.8485 5.24923 11.9565 6ZM6.70719 11.1214L8.0212 12.4354V7H9.01506V12.3992L10.2929 11.1214L11 11.8285L8.85356 13.9749H8.14645L6.00008 11.8285L6.70719 11.1214Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/folder-opened-plus.svg

@@ -0,0 +1,12 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0)">
+<path d="M0.499817 14H12.4998L12.9798 13.63L15.6098 6.63L15.1298 6H12.9998V3.5L12.4998 3H6.70978L5.84979 2.15002L5.49982 2H0.499817L-0.000183105 2.5V13.5L0.499817 14ZM0.999817 3H5.28986L6.14984 3.84998L6.49982 4H11.9998V6H8.49982L8.14984 6.15002L7.28986 7H2.49982L2.02985 7.33997L1.02985 10.42L0.999817 3ZM12.1298 13H1.18976L2.8598 8H7.49982L7.84979 7.84998L8.70978 7H14.4998L12.1298 13Z" fill="#424242"/>
+<line y2="12" x2="8" y1="12" x1="16" stroke-width="1" stroke="green" fill="none"/>
+<line y2="8" x2="12" y1="16" x1="12" stroke-width="1" stroke="green" fill="none"/>
+</g>
+<defs>
+<clipPath id="clip0">
+<path d="M-0.000183105 0H15.9998V16H-0.000183105V0Z" fill="white"/>
+</clipPath>
+</defs>
+</svg>

extensions/ql-vscode/media/light/lgtm-plus.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32" fill="none">
+  <path d="M16.010 6.49c-3.885 0-7.167 0.906-9.328 2.813-0.063-0.12-0.109-0.219-0.188-0.339-0.224-0.365-0.438-0.776-1.104-1.188-0.411-0.26-0.87-0.438-1.349-0.516-0.208-0.021-0.422-0.021-0.63 0l0.135-0.016c-1.214 0-1.922 0.724-2.385 1.354-0.458 0.625-0.755 1.328-0.948 2.099-0.38 1.542-0.385 3.536 1.083 5.026 0.766 0.781 1.667 1.151 2.484 1.37 0.156 0.042 0.297 0.052 0.448 0.083 0.531 2.521 2.104 4.656 4.208 5.839v0.005c1.24 0.693 2.417 1.010 3.297 1.349 1.234 0.479 2.536 1 4.052 1.135l0.078 0.005h0.198c1.745 0 3.063-0.703 4.203-1.141 0.875-0.333 2.052-0.641 3.302-1.344 0.578-0.323 1.115-0.719 1.594-1.172 1.318-1.234 2.229-2.839 2.625-4.599 1.115-0.182 2.141-0.719 2.922-1.536 1.464-1.484 1.458-3.479 1.078-5.021-0.193-0.771-0.49-1.474-0.948-2.099-0.458-0.63-1.172-1.354-2.385-1.354l0.135 0.016c-0.208-0.021-0.422-0.021-0.63 0-0.479 0.078-0.938 0.255-1.344 0.516-0.667 0.411-0.88 0.823-1.104 1.182-0.073 0.12-0.12 0.219-0.188 0.333-2.156-1.901-5.432-2.802-9.313-2.802zM16.042 8.313c4.745 0 8.016 1.422 9.411 3.964 0.839-0.323 1.453-2.521 2.146-2.948 0.563-0.344 0.885-0.26 0.885-0.26 1.271 0 2.578 3.729 0.953 5.38-0.859 0.875-2.443 1.12-3.229 1.057-0.063 2.542-1.542 4.833-3.5 5.932-1 0.563-2.068 0.854-3.063 1.234-1.229 0.469-2.38 1.016-3.547 1.016h-0.125c-1.161-0.099-2.318-0.542-3.547-1.016-0.995-0.38-2.068-0.682-3.063-1.24-1.948-1.099-3.427-3.391-3.49-5.927-0.781 0.068-2.385-0.177-3.245-1.057-1.625-1.651-0.318-5.38 0.948-5.38 0 0 0.328-0.083 0.885 0.26 0.698 0.427 1.318 2.646 2.161 2.953 1.391-2.547 4.667-3.969 9.417-3.969zM10.875 11.422c-2.276-0.042-4.146 1.792-4.146 4.068 0 2.281 1.87 4.115 4.146 4.073 5.328-0.099 5.328-8.047 0-8.141zM21.208 11.422c-5.427 0-5.427 8.141 0 8.141s5.427-8.141 0-8.141zM11.453 13.708c2.349 0.063 2.349 3.552 0 3.615-1.182 0-2.042-1.115-1.75-2.255 0.318 0.771 1.469 0.547 1.464-0.292 0-0.406-0.318-0.745-0.729-0.76 0.302-0.203 0.656-0.313 1.016-0.307zM20.641 13.708c2.344 0.063 2.344 3.552 0 3.615-1.182 0-2.047-1.115-1.755-2.255 0.229 0.552 0.979 0.641 1.328 0.146 0.344-0.49 0.010-1.167-0.589-1.193 0.297-0.208 0.651-0.313 1.016-0.313zM15.359 19.906c-0.318 0.026-0.5 0.193-0.5 0.635 0 0.281 0.182 0.484 0.5 0.484 0.229 0 0.266-0.323 0.047-0.375-0.031-0.005-0.172-0.057-0.172-0.182 0-0.12 0-0.167 0.24-0.198 0.104-0.016 0.156-0.141 0.125-0.24s-0.125-0.135-0.24-0.125zM16.724 19.906c-0.115-0.005-0.208 0.026-0.24 0.125s0.021 0.224 0.125 0.24c0.24 0.031 0.24 0.078 0.24 0.198 0 0.125-0.141 0.177-0.172 0.182-0.219 0.052-0.182 0.375 0.042 0.375 0.323 0 0.51-0.203 0.51-0.484 0-0.443-0.188-0.609-0.505-0.635z" fill="#424242"/>
+  <line y2="24" x2="16" y1="26" x1="32" stroke-width="2" stroke="green" fill="none"/>
+  <line y2="16" x2="24" y1="32" x1="24" stroke-width="1" stroke="green" fill="none"/>
+</svg>

extensions/ql-vscode/media/light/sort-alpha.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413 			"/>
+			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72 
+				250.027,338.24 432,338.24 432,304.32 			"/>
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227 			"/>
+			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
+				L113.28,140.48l41.387,110.507H71.893z"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/light/sort-date.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.0.5",
+  "version": "1.2.2",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -18,16 +18,28 @@
   "categories": [
     "Programming Languages"
   ],
+  "extensionDependencies": [
+    "hbenl.vscode-test-explorer"
+  ],
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
+    "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
-    "onCommand:codeQL.chooseDatabase",
+    "onCommand:codeQLDatabases.chooseDatabaseFolder",
+    "onCommand:codeQLDatabases.chooseDatabaseArchive",
+    "onCommand:codeQLDatabases.chooseDatabaseInternet",
+    "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
+    "onCommand:codeQL.chooseDatabaseFolder",
+    "onCommand:codeQL.chooseDatabaseArchive",
+    "onCommand:codeQL.chooseDatabaseInternet",
+    "onCommand:codeQL.chooseDatabaseLgtm",
     "onCommand:codeQLDatabases.chooseDatabase",
     "onCommand:codeQLDatabases.setCurrentDatabase",
     "onCommand:codeQL.quickQuery",
+    "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
     "onFileSystem:codeql-zip-archive"
   ],
@@ -72,6 +84,12 @@
           ".dbscheme"
         ],
         "configuration": "./language-configuration.json"
+      },
+      {
+        "id": "xml",
+        "extensions": [
+          ".qhelp"
+        ]
       }
     ],
     "grammars": [
@@ -94,7 +112,7 @@
           "scope": "machine",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.cmd` on Windows. This overrides all other CodeQL CLI settings."
+          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -127,10 +145,23 @@
           "default": false,
           "description": "Enable debug logging and tuple counting when running CodeQL queries. This information is useful for debugging query performance."
         },
+        "codeQL.runningQueries.autoSave": {
+          "type": "boolean",
+          "default": false,
+          "description": "Enable automatically saving a modified query file when running a query."
+        },
         "codeQL.queryHistory.format": {
           "type": "string",
           "default": "[%t] %q on %d - %s",
           "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+        },
+        "codeQL.runningTests.numberOfThreads": {
+          "scope": "window",
+          "type": "integer",
+          "default": 1,
+          "minimum": 1,
+          "maximum": 1024,
+          "description": "Number of threads for running CodeQL tests."
         }
       }
     },
@@ -148,11 +179,35 @@
         "title": "CodeQL: Quick Query"
       },
       {
-        "command": "codeQL.chooseDatabase",
-        "title": "CodeQL: Choose Database",
+        "command": "codeQLDatabases.chooseDatabaseFolder",
+        "title": "Choose Database from Folder",
         "icon": {
-          "light": "media/black-plus.svg",
-          "dark": "media/white-plus.svg"
+          "light": "media/light/folder-opened-plus.svg",
+          "dark": "media/dark/folder-opened-plus.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseArchive",
+        "title": "Choose Database from Archive",
+        "icon": {
+          "light": "media/light/archive-plus.svg",
+          "dark": "media/dark/archive-plus.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseInternet",
+        "title": "Download Database",
+        "icon": {
+          "light": "media/light/cloud-download.svg",
+          "dark": "media/dark/cloud-download.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.chooseDatabaseLgtm",
+        "title": "Download from LGTM",
+        "icon": {
+          "light": "media/light/lgtm-plus.svg",
+          "dark": "media/dark/lgtm-plus.svg"
         }
       },
       {
@@ -179,6 +234,46 @@
         "command": "codeQLDatabases.upgradeDatabase",
         "title": "Upgrade Database"
       },
+      {
+        "command": "codeQLDatabases.renameDatabase",
+        "title": "Rename Database"
+      },
+      {
+        "command": "codeQLDatabases.openDatabaseFolder",
+        "title": "Show Database Directory"
+      },
+      {
+        "command": "codeQL.chooseDatabaseFolder",
+        "title": "CodeQL: Choose Database from Folder"
+      },
+      {
+        "command": "codeQL.chooseDatabaseArchive",
+        "title": "CodeQL: Choose Database from Archive"
+      },
+      {
+        "command": "codeQL.chooseDatabaseInternet",
+        "title": "CodeQL: Download Database"
+      },
+      {
+        "command": "codeQL.chooseDatabaseLgtm",
+        "title": "CodeQL: Download Database from LGTM"
+      },
+      {
+        "command": "codeQLDatabases.sortByName",
+        "title": "Sort by Name",
+        "icon": {
+          "light": "media/light/sort-alpha.svg",
+          "dark": "media/dark/sort-alpha.svg"
+        }
+      },
+      {
+        "command": "codeQLDatabases.sortByDateAdded",
+        "title": "Sort by Date Added",
+        "icon": {
+          "light": "media/light/sort-date.svg",
+          "dark": "media/dark/sort-date.svg"
+        }
+      },
       {
         "command": "codeQL.checkForUpdatesToCLI",
         "title": "CodeQL: Check for CLI Updates"
@@ -195,6 +290,18 @@
         "command": "codeQLQueryHistory.itemClicked",
         "title": "Query History Item"
       },
+      {
+        "command": "codeQLQueryHistory.showQueryLog",
+        "title": "Show Query Log"
+      },
+      {
+        "command": "codeQLQueryHistory.showQueryText",
+        "title": "Show Query Text"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarif",
+        "title": "View SARIF"
+      },
       {
         "command": "codeQLQueryResults.nextPathStep",
         "title": "CodeQL: Show Next Step on Path"
@@ -206,12 +313,49 @@
       {
         "command": "codeQLQueryHistory.setLabel",
         "title": "Set Label"
+      },
+      {
+        "command": "codeQL.restartQueryServer",
+        "title": "CodeQL: Restart Query Server"
+      },
+      {
+        "command": "codeQLTests.showOutputDifferences",
+        "title": "CodeQL: Show Test Output Differences"
+      },
+      {
+        "command": "codeQLTests.acceptOutput",
+        "title": "CodeQL: Accept Test Output"
       }
     ],
     "menus": {
       "view/title": [
         {
-          "command": "codeQL.chooseDatabase",
+          "command": "codeQLDatabases.sortByName",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.sortByDateAdded",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseFolder",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseArchive",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "view == codeQLDatabases",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         }
@@ -232,6 +376,16 @@
           "group": "9_qlCommands",
           "when": "view == codeQLDatabases"
         },
+        {
+          "command": "codeQLDatabases.renameDatabase",
+          "group": "9_qlCommands",
+          "when": "view == codeQLDatabases"
+        },
+        {
+          "command": "codeQLDatabases.openDatabaseFolder",
+          "group": "9_qlCommands",
+          "when": "view == codeQLDatabases"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "group": "9_qlCommands",
@@ -246,13 +400,38 @@
           "command": "codeQLQueryHistory.setLabel",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryLog",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryText",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarif",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "group": "qltest@1",
+          "when": "view == test-explorer && viewItem == testWithSource"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "group": "qltest@2",
+          "when": "view == test-explorer && viewItem == testWithSource"
         }
       ],
       "explorer/context": [
         {
           "command": "codeQL.setCurrentDatabase",
           "group": "9_qlCommands",
-          "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder"
+          "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
         },
         {
           "command": "codeQL.runQuery",
@@ -277,10 +456,46 @@
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.renameDatabase",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.openDatabaseFolder",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.sortByName",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.sortByDateAdded",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.removeDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.chooseDatabaseFolder",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseArchive",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.upgradeDatabase",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "when": "false"
@@ -293,6 +508,18 @@
           "command": "codeQLQueryHistory.itemClicked",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.showQueryLog",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.showQueryText",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarif",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.setLabel",
           "when": "false"
@@ -339,10 +566,13 @@
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
     "integration": "node ./out/vscode-tests/run-integration-tests.js",
     "update-vscode": "node ./node_modules/vscode/bin/install",
-    "postinstall": "node ./node_modules/vscode/bin/install",
-    "format": "tsfmt -r"
+    "postinstall": "npm rebuild && node ./node_modules/vscode/bin/install",
+    "format": "tsfmt -r",
+    "lint": "eslint src test --ext .ts,.tsx",
+    "format-staged": "lint-staged"
   },
   "dependencies": {
+    "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
     "fs-extra": "^8.1.0",
     "glob-promise": "^3.4.0",
@@ -354,12 +584,19 @@
     "semmle-io-node": "^0.0.1",
     "semmle-vscode-utils": "^0.0.1",
     "tmp": "^0.1.0",
+    "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
-    "vscode-jsonrpc": "^4.0.0",
-    "vscode-languageclient": "^5.2.1"
+    "vscode-jsonrpc": "^5.0.1",
+    "vscode-languageclient": "^6.1.3",
+    "vscode-test-adapter-api": "~1.7.0",
+    "vscode-test-adapter-util": "~0.7.0",
+    "minimist": "~1.2.5",
+    "semver": "~7.3.2",
+    "@types/semver": "~7.2.0"
   },
   "devDependencies": {
     "@types/chai": "^4.1.7",
+    "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
     "@types/fs-extra": "^8.0.0",
     "@types/glob": "^7.1.1",
@@ -380,14 +617,15 @@
     "@types/xml2js": "~0.4.4",
     "build-tasks": "^0.0.1",
     "chai": "^4.2.0",
-    "child-process-promise": "^2.2.1",
     "css-loader": "~3.1.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
     "mocha": "~6.2.1",
+    "mocha-sinon": "~2.1.0",
     "npm-run-all": "^4.1.5",
+    "sinon": "~9.0.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
     "ts-loader": "^5.4.5",
@@ -397,8 +635,36 @@
     "typescript-config": "^0.0.1",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
-    "vscode-test": "^1.0.0",
+    "vscode-test": "^1.4.0",
     "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2"
+    "webpack-cli": "^3.3.2",
+    "eslint": "~6.8.0",
+    "@typescript-eslint/eslint-plugin": "~2.23.0",
+    "@typescript-eslint/parser": "~2.23.0",
+    "chai-as-promised": "~7.1.1",
+    "@types/chai-as-promised": "~7.1.2",
+    "@types/sinon": "~7.5.2",
+    "sinon-chai": "~3.5.0",
+    "@types/sinon-chai": "~3.2.3",
+    "proxyquire": "~2.1.3",
+    "@types/proxyquire": "~1.3.28",
+    "eslint-plugin-react": "~7.19.0",
+    "husky": "~4.2.5",
+    "lint-staged": "~10.2.2",
+    "prettier": "~2.0.5"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "npm run format-staged",
+      "pre-push": "npm run lint"
+    }
+  },
+  "lint-staged": {
+    "./**/*.{json,css,scss,md}": [
+      "prettier --write"
+    ],
+    "./**/*.{ts,tsx}": [
+      "tsfmt -r"
+    ]
   }
 }

extensions/ql-vscode/src/adapt.ts

@@ -0,0 +1,134 @@
+import { DecodedBqrsChunk, ResultSetSchema, ColumnKind, Column, ColumnValue } from "./bqrs-cli-types";
+import { LocationValue, ResultSetSchema as AdaptedSchema, ColumnSchema, ColumnType, LocationStyle } from 'semmle-bqrs';
+
+// FIXME: This is a temporary bit of impedance matching to convert
+// from the types provided by ./bqrs-cli-types, to the types used by
+// the view layer.
+//
+// The reason that it is benign for now is that it is only used by
+// feature-flag-guarded codepaths that won't be encountered by normal
+// users. It is not yet guaranteed to produce correct output for raw
+// results.
+//
+// Eventually, the view layer should be refactored to directly accept data
+// of types coming from bqrs-cli-types, and this file can be deleted.
+
+export type ResultRow = ResultValue[];
+
+export interface ResultElement {
+  label: string;
+  location?: LocationValue;
+}
+
+export interface ResultUri {
+  uri: string;
+}
+
+export type ResultValue = ResultElement | ResultUri | string;
+
+export interface RawResultSet {
+  readonly schema: AdaptedSchema;
+  readonly rows: readonly ResultRow[];
+}
+
+function adaptKind(kind: ColumnKind): ColumnType {
+  // XXX what about 'u'?
+  if (kind === 'e') {
+    return { type: 'e', primitiveType: 's', locationStyle: LocationStyle.FivePart, hasLabel: true };
+  }
+  else {
+    return { type: kind };
+  }
+}
+
+function adaptColumn(col: Column): ColumnSchema {
+  return { name: col.name!, type: adaptKind(col.kind) };
+}
+
+export function adaptSchema(schema: ResultSetSchema): AdaptedSchema {
+  return {
+    columns: schema.columns.map(adaptColumn),
+    name: schema.name,
+    tupleCount: schema.rows,
+    version: 0,
+  };
+}
+
+export function adaptValue(val: ColumnValue): ResultValue {
+  // XXX taking a lot of incorrect shortcuts here
+
+  if (typeof val === 'string') {
+    return val;
+  }
+
+  if (typeof val === 'number' || typeof val === 'boolean') {
+    return val + '';
+  }
+
+  const url = val.url;
+
+  if (typeof url === 'string') {
+    return url;
+  }
+
+  if (url === undefined) {
+    return 'none';
+  }
+
+  return {
+    label: val.label || '',
+    location: {
+      t: LocationStyle.FivePart,
+      lineStart: url.startLine,
+      lineEnd: url.endLine,
+      colStart: url.startColumn,
+      colEnd: url.endColumn,
+      // FIXME: This seems definitely wrong. Should we be using
+      // something like the code in sarif-utils.ts?
+      file: url.uri.replace(/file:/, ''),
+    }
+  };
+
+}
+
+export function adaptRow(row: ColumnValue[]): ResultRow {
+  return row.map(adaptValue);
+}
+
+export function adaptBqrs(schema: AdaptedSchema, page: DecodedBqrsChunk): RawResultSet {
+  return {
+    schema,
+    rows: page.tuples.map(adaptRow),
+  };
+}
+
+/**
+ * This type has two branches; we are in the process of changing from
+ * one to the other. The old way is to parse them inside the webview,
+ * the new way is to parse them in the extension. The main motivation
+ * for this transition is to make pagination possible in such a way
+ * that only one page needs to be sent from the extension to the webview.
+ */
+export type ParsedResultSets = ExtensionParsedResultSets | WebviewParsedResultSets;
+
+/**
+ * The old method doesn't require any nontrivial information to be included here,
+ * just a tag to indicate that it is being used.
+ */
+export interface WebviewParsedResultSets {
+  t: 'WebviewParsed';
+  selectedTable?: string; // when undefined, means 'show default table'
+}
+
+/**
+ * The new method includes which bqrs page is being sent, and the
+ * actual results parsed on the extension side.
+ */
+export interface ExtensionParsedResultSets {
+  t: 'ExtensionParsed';
+  pageNumber: number;
+  numPages: number;
+  selectedTable?: string; // when undefined, means 'show default table'
+  resultSetNames: string[];
+  resultSet: RawResultSet;
+}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -51,7 +51,10 @@ export type Entry = File | Directory;
  */
 export type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
 
-export type ZipFileReference = { sourceArchiveZipPath: string, pathWithinSourceArchive: string };
+export type ZipFileReference = {
+  sourceArchiveZipPath: string;
+  pathWithinSourceArchive: string;
+};
 
 /** Encodes a reference to a source file within a zipped source archive into a single URI. */
 export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
@@ -87,7 +90,7 @@ export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
   });
 }
 
-const sourceArchiveUriAuthorityPattern = /^(\d+)\-(\d+)$/;
+const sourceArchiveUriAuthorityPattern = /^(\d+)-(\d+)$/;
 
 class InvalidSourceArchiveUriError extends Error {
   constructor(uri: vscode.Uri) {
@@ -139,8 +142,8 @@ function ensureDir(map: DirectoryHierarchyMap, dir: string) {
 }
 
 type Archive = {
-  unzipped: unzipper.CentralDirectory,
-  dirMap: DirectoryHierarchyMap,
+  unzipped: unzipper.CentralDirectory;
+  dirMap: DirectoryHierarchyMap;
 };
 
 export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
@@ -169,8 +172,8 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
   async readDirectory(uri: vscode.Uri): Promise<[string, vscode.FileType][]> {
     const ref = decodeSourceArchiveUri(uri);
     const archive = await this.getArchive(ref.sourceArchiveZipPath);
-    let contents = archive.dirMap.get(ref.pathWithinSourceArchive);
-    const result = contents === undefined ? [] : Array.from(contents.entries());
+    const contents = archive.dirMap.get(ref.pathWithinSourceArchive);
+    const result = contents === undefined ? undefined : Array.from(contents.entries());
     if (result === undefined) {
       throw vscode.FileSystemError.FileNotFound(uri);
     }
@@ -189,7 +192,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
 
   // write operations, all disabled
 
-  writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { create: boolean, overwrite: boolean }): void {
+  writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { create: boolean; overwrite: boolean }): void {
     throw this.readOnlyError;
   }
 
@@ -235,11 +238,11 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
     if (archive.dirMap.has(reqPath)) {
       return new Directory(reqPath);
     }
-    throw vscode.FileSystemError.FileNotFound(uri);
+    throw vscode.FileSystemError.FileNotFound(`uri '${uri.toString()}', interpreted as '${reqPath}' in archive '${ref.sourceArchiveZipPath}'`);
   }
 
   private async _lookupAsFile(uri: vscode.Uri): Promise<File> {
-    let entry = await this._lookup(uri);
+    const entry = await this._lookup(uri);
     if (entry instanceof File) {
       return entry;
     }
@@ -254,7 +257,7 @@ export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
 
   watch(_resource: vscode.Uri): vscode.Disposable {
     // ignore, fires for all changes...
-    return new vscode.Disposable(() => { });
+    return new vscode.Disposable(() => { /**/ });
   }
 }
 

extensions/ql-vscode/src/bqrs-cli-types.ts

@@ -0,0 +1,99 @@
+
+export const PAGE_SIZE = 1000;
+
+/**
+ * The single-character codes used in the bqrs format for the the kind
+ * of a result column. This namespace is intentionally not an enum, see
+ * the "for the sake of extensibility" comment in messages.ts.
+ */
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace ColumnKindCode {
+  export const FLOAT = "f";
+  export const INTEGER = "i";
+  export const STRING = "s";
+  export const BOOLEAN = "b";
+  export const DATE = "d";
+  export const ENTITY = "e";
+}
+
+export type ColumnKind =
+  | typeof ColumnKindCode.FLOAT
+  | typeof ColumnKindCode.INTEGER
+  | typeof ColumnKindCode.STRING
+  | typeof ColumnKindCode.BOOLEAN
+  | typeof ColumnKindCode.DATE
+  | typeof ColumnKindCode.ENTITY;
+
+export interface Column {
+  name?: string;
+  kind: ColumnKind;
+}
+
+export interface ResultSetSchema {
+  name: string;
+  rows: number;
+  columns: Column[];
+  pagination?: PaginationInfo;
+}
+
+export function getResultSetSchema(resultSetName: string, resultSets: BQRSInfo): ResultSetSchema | undefined {
+  for (const schema of resultSets["result-sets"]) {
+    if (schema.name === resultSetName) {
+      return schema;
+    }
+  }
+  return undefined;
+}
+export interface PaginationInfo {
+  "step-size": number;
+  offsets: number[];
+}
+
+export interface BQRSInfo {
+  "result-sets": ResultSetSchema[];
+}
+
+export interface EntityValue {
+  url?: UrlValue;
+  label?: string;
+}
+
+export interface LineColumnLocation {
+  uri: string;
+  startLine: number;
+  startColumn: number;
+  endLine: number;
+  endColumn: number;
+  charOffset: never;
+  charLength: never;
+}
+
+export interface OffsetLengthLocation {
+  uri: string;
+  startLine: never;
+  startColumn: never;
+  endLine: never;
+  endColumn: never;
+  charOffset: number;
+  charLength: number;
+}
+
+export interface WholeFileLocation {
+  uri: string;
+  startLine: never;
+  startColumn: never;
+  endLine: never;
+  endColumn: never;
+  charOffset: never;
+  charLength: never;
+}
+
+export type UrlValue = LineColumnLocation | OffsetLengthLocation | WholeFileLocation | string;
+
+
+export type ColumnValue = EntityValue | number | string | boolean;
+
+export interface DecodedBqrsChunk {
+  tuples: ColumnValue[][];
+  next?: number;
+}

extensions/ql-vscode/src/cli-version.ts

@@ -1,10 +1,11 @@
+import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
 import { Logger } from "./logging";
 
 /**
  * Get the version of a CodeQL CLI.
  */
-export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<Version | undefined> {
+export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<semver.SemVer | undefined> {
   const output: string = await runCodeQlCliCommand(
     codeQlPath,
     ["version"],
@@ -12,85 +13,5 @@ export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): P
     "Checking CodeQL version",
     logger
   );
-  return tryParseVersionString(output.trim());
-}
-
-/**
- * Try to parse a version string, returning undefined if we can't parse it.
- * 
- * Version strings must contain a major, minor, and patch version.  They may optionally
- * start with "v" and may optionally contain some "tail" string after the major, minor, and
- * patch versions, for example as in `v2.1.0+baf5bff`.
- */
-export function tryParseVersionString(versionString: string): Version | undefined {
-  const match = versionString.match(versionRegex);
-  if (match === null) {
-    return undefined;
-  }
-  return {
-    buildMetadata: match[5],
-    majorVersion: Number.parseInt(match[1], 10),
-    minorVersion: Number.parseInt(match[2], 10),
-    patchVersion: Number.parseInt(match[3], 10),
-    prereleaseVersion: match[4],
-    rawString: versionString,
-  }
-}
-
-/**
- * Regex for parsing semantic versions
- * 
- * From the semver spec https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
- */
-const versionRegex = new RegExp(String.raw`^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)` +
-  String.raw`(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?` +
-  String.raw`(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`);
-
-/**
- * A version of the CodeQL CLI.
- */
-export interface Version {
-  /**
-   * Build metadata
-   * 
-   * For example, this will be `abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   * Build metadata must be ignored when comparing versions. 
-   */
-  buildMetadata: string | undefined;
-
-  /**
-   * Major version number
-   * 
-   * For example, this will be `2` for version 2.1.0-alpha.1+abcdef0.
-   */
-  majorVersion: number;
-
-  /**
-   * Minor version number
-   * 
-   * For example, this will be `1` for version 2.1.0-alpha.1+abcdef0.
-   */
-  minorVersion: number;
-
-  /**
-   * Patch version number
-   * 
-   * For example, this will be `0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  patchVersion: number;
-
-  /**
-   * Prerelease version
-   * 
-   * For example, this will be `alpha.1` for version 2.1.0-alpha.1+abcdef0.
-   * The prerelease version must be considered when comparing versions.
-   */
-  prereleaseVersion: string | undefined;
-
-  /**
-   * Raw version string
-   * 
-   * For example, this will be `2.1.0-alpha.1+abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  rawString: string;
+  return semver.parse(output.trim()) || undefined;
 }

extensions/ql-vscode/src/cli.ts

@@ -1,13 +1,19 @@
-import * as child_process from "child_process";
+/* eslint-disable @typescript-eslint/camelcase */
+import * as cpp from 'child-process-promise';
+import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
+import { Readable } from 'stream';
+import { StringDecoder } from 'string_decoder';
+import * as tk from 'tree-kill';
 import * as util from 'util';
-import { Logger, ProgressReporter } from "./logging";
-import { Disposable } from "vscode";
+import { CancellationToken, Disposable } from 'vscode';
+import { BQRSInfo, DecodedBqrsChunk } from "./bqrs-cli-types";
 import { DistributionProvider } from "./distribution";
-import { SortDirection, QueryMetadata } from "./interface-types";
 import { assertNever } from "./helpers-pure";
+import { QueryMetadata, SortDirection } from "./interface-types";
+import { Logger, ProgressReporter } from "./logging";
 
 /**
  * The version of the SARIF format that we are using.
@@ -23,10 +29,10 @@ const LOGGING_FLAGS = ['-v', '--log-to-stderr'];
  * The expected output of `codeql resolve library-path`.
  */
 export interface QuerySetup {
-  libraryPath: string[],
-  dbscheme: string,
-  relativeName?: string,
-  compilationCache?: string
+  libraryPath: string[];
+  dbscheme: string;
+  relativeName?: string;
+  compilationCache?: string;
 }
 
 /**
@@ -62,6 +68,31 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve tests`.
+ */
+export type ResolvedTests = string[];
+
+/**
+ * Options for `codeql test run`.
+ */
+export interface TestRunOptions {
+  cancellationToken?: CancellationToken;
+  logger?: Logger;
+}
+
+/**
+ * Event fired by `codeql test run`.
+ */
+export interface TestCompleted {
+  test: string;
+  pass: boolean;
+  messages: string[];
+  compilationMs: number;
+  evaluationMs: number;
+  expected: string;
+}
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -91,11 +122,11 @@ export class CodeQLCliServer implements Disposable {
   }
 
 
-  dispose() {
+  dispose(): void {
     this.killProcessIfRunning();
   }
 
-  killProcessIfRunning() {
+  killProcessIfRunning(): void {
     if (this.process) {
       // Tell the Java CLI server process to shut down.
       this.logger.log('Sending shutdown request');
@@ -122,8 +153,8 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Restart the server when the current command terminates
    */
-  private restartCliServer() {
-    let callback = () => {
+  private restartCliServer(): void {
+    const callback = (): void => {
       try {
         this.killProcessIfRunning();
       } finally {
@@ -134,39 +165,47 @@ export class CodeQLCliServer implements Disposable {
     // If the server is not running a command run this immediately
     // otherwise add to the front of the queue (as we want to run this after the next command()).
     if (this.commandInProcess) {
-      this.commandQueue.unshift(callback)
+      this.commandQueue.unshift(callback);
     } else {
       callback();
     }
 
   }
 
+  /**
+   * Get the path to the CodeQL CLI distribution, or throw an exception if not found.
+   */
+  private async getCodeQlPath(): Promise<string> {
+    const codeqlPath = await this.config.getCodeQlPathWithoutVersionCheck();
+    if (!codeqlPath) {
+      throw new Error('Failed to find CodeQL distribution.');
+    }
+    return codeqlPath;
+  }
+
   /**
    * Launch the cli server
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
-    const config = await this.config.getCodeQlPathWithoutVersionCheck();
-    if (!config) {
-      throw new Error("Failed to find codeql distribution")
-    }
-    return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, _data => { })
+    const config = await this.getCodeQlPath();
+    return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, _data => { /**/ });
   }
 
   private async runCodeQlCliInternal(command: string[], commandArgs: string[], description: string): Promise<string> {
-    let stderrBuffers: Buffer[] = [];
+    const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
-      throw new Error("runCodeQlCliInternal called while cli was running")
+      throw new Error("runCodeQlCliInternal called while cli was running");
     }
     this.commandInProcess = true;
     try {
       //Launch the process if it doesn't exist
       if (!this.process) {
-        this.process = await this.launchProcess()
+        this.process = await this.launchProcess();
       }
       // Grab the process so that typescript know that it is always defined.
       const process = this.process;
       // The array of fragments of stdout
-      let stdoutBuffers: Buffer[] = [];
+      const stdoutBuffers: Buffer[] = [];
 
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
@@ -191,29 +230,30 @@ export class CodeQLCliServer implements Disposable {
           // Listen for process exit.
           process.addListener("close", (code) => reject(code));
           // Write the command followed by a null terminator.
-          process.stdin.write(JSON.stringify(args), "utf8")
-          process.stdin.write(this.nullBuffer)
+          process.stdin.write(JSON.stringify(args), "utf8");
+          process.stdin.write(this.nullBuffer);
         });
         // Join all the data together
-        let fullBuffer = Buffer.concat(stdoutBuffers);
+        const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
-        let data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
+        const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
         this.logger.log(`CLI command succeeded.`);
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
         // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
-        if (stderrBuffers.length == 0) {
-          throw new Error(`${description} failed: ${err}`)
-        } else {
-          throw new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString("utf8")}`);
-        }
+        const newError =
+          stderrBuffers.length == 0
+            ? new Error(`${description} failed: ${err}`)
+            : new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString("utf8")}`);
+        newError.stack += (err.stack || '');
+        throw newError;
       } finally {
         this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
         // Remove the listeners we set up.
-        process.stdout.removeAllListeners('data')
-        process.stderr.removeAllListeners('data')
+        process.stdout.removeAllListeners('data');
+        process.stderr.removeAllListeners('data');
         process.removeAllListeners("close");
       }
     } finally {
@@ -226,13 +266,94 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Run the next command in the queue
    */
-  private runNext() {
+  private runNext(): void {
     const callback = this.commandQueue.shift();
     if (callback) {
       callback();
     }
   }
 
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param cancellationToken CancellationToken to terminate the test process.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  private async* runAsyncCodeQlCliCommandInternal(
+    command: string[],
+    commandArgs: string[],
+    cancellationToken?: CancellationToken,
+    logger?: Logger
+  ): AsyncGenerator<string, void, unknown> {
+    // Add format argument first, in case commandArgs contains positional parameters.
+    const args = [
+      ...command,
+      '--format', 'jsonz',
+      ...commandArgs
+    ];
+
+    // Spawn the CodeQL process
+    const codeqlPath = await this.getCodeQlPath();
+    const childPromise = cpp.spawn(codeqlPath, args);
+    const child = childPromise.childProcess;
+
+    let cancellationRegistration: Disposable | undefined = undefined;
+    try {
+      if (cancellationToken !== undefined) {
+        cancellationRegistration = cancellationToken.onCancellationRequested(_e => {
+          tk(child.pid);
+        });
+      }
+      if (logger !== undefined) {
+        // The human-readable output goes to stderr.
+        logStream(child.stderr!, logger);
+      }
+
+      for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
+        yield event;
+      }
+
+      await childPromise;
+    }
+    finally {
+      if (cancellationRegistration !== undefined) {
+        cancellationRegistration.dispose();
+      }
+    }
+  }
+
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param description Description of the action being run, to be shown in log and error messages.
+   * @param cancellationToken CancellationToken to terminate the test process.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  public async* runAsyncCodeQlCliCommand<EventType>(
+    command: string[],
+    commandArgs: string[],
+    description: string,
+    cancellationToken?: CancellationToken,
+    logger?: Logger
+  ): AsyncGenerator<EventType, void, unknown> {
+    for await (const event of await this.runAsyncCodeQlCliCommandInternal(command, commandArgs,
+      cancellationToken, logger)) {
+      try {
+        yield JSON.parse(event) as EventType;
+      } catch (err) {
+        throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`);
+      }
+    }
+  }
+
   /**
    * Runs a CodeQL CLI command on the server, returning the output as a string.
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
@@ -248,17 +369,17 @@ export class CodeQLCliServer implements Disposable {
 
     return new Promise((resolve, reject) => {
       // Construct the command that actually does the work
-      const callback = () => {
+      const callback = (): void => {
         try {
           this.runCodeQlCliInternal(command, commandArgs, description).then(resolve, reject);
         } catch (err) {
           reject(err);
         }
-      }
+      };
       // If the server is not running a command, then run the given command immediately,
       // otherwise add to the queue
       if (this.commandInProcess) {
-        this.commandQueue.push(callback)
+        this.commandQueue.push(callback);
       } else {
         callback();
       }
@@ -280,7 +401,7 @@ export class CodeQLCliServer implements Disposable {
     try {
       return JSON.parse(result) as OutputType;
     } catch (err) {
-      throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`)
+      throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`);
     }
   }
 
@@ -298,6 +419,40 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, "Resolving library paths");
   }
 
+  /**
+   * Finds all available QL tests in a given directory.
+   * @param testPath Root of directory tree to search for tests.
+   * @returns The list of tests that were found.
+   */
+  public async resolveTests(testPath: string): Promise<ResolvedTests> {
+    const subcommandArgs = [
+      testPath
+    ];
+    return await this.runJsonCodeQlCliCommand<ResolvedTests>(['resolve', 'tests'], subcommandArgs, 'Resolving tests');
+  }
+
+  /**
+   * Runs QL tests.
+   * @param testPaths Full paths of the tests to run.
+   * @param workspaces Workspace paths to use as search paths for QL packs.
+   * @param options Additional options.
+   */
+  public async* runTests(
+    testPaths: string[], workspaces: string[], options: TestRunOptions
+  ): AsyncGenerator<TestCompleted, void, unknown> {
+
+    const subcommandArgs = [
+      '--additional-packs', workspaces.join(path.delimiter),
+      '--threads', '8',
+      ...testPaths
+    ];
+
+    for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
+      subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
+      yield event;
+    }
+  }
+
   /**
    * Gets the metadata for a query.
    * @param queryPath The path to the query.
@@ -310,6 +465,7 @@ export class CodeQLCliServer implements Disposable {
    * Gets the RAM setting for the query server.
    * @param queryMemoryMb The maximum amount of RAM to use, in MB.
    * Leave `undefined` for CodeQL to choose a limit based on the available system memory.
+   * @param progressReporter The progress reporter to send progress information to.
    * @returns String arguments that can be passed to the CodeQL query server,
    * indicating how to split the given RAM limit between heap and off-heap memory.
    */
@@ -320,9 +476,41 @@ export class CodeQLCliServer implements Disposable {
     }
     return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, "Resolving RAM settings", progressReporter);
   }
+  /**
+   * Gets the headers (and optionally pagination info) of a bqrs.
+   * @param bqrsPath The path to the bqrs.
+   * @param pageSize The page size to precompute offsets into the binary file for.
+   */
+  async bqrsInfo(bqrsPath: string, pageSize?: number): Promise<BQRSInfo> {
+    const subcommandArgs = (
+      pageSize ? ["--paginate-rows", pageSize.toString()] : []
+    ).concat(
+      bqrsPath
+    );
+    return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, "Reading bqrs header");
+  }
+
+  /**
+  * Gets the results from a bqrs.
+  * @param bqrsPath The path to the bqrs.
+  * @param resultSet The result set to get.
+  * @param pageSize How many results to get.
+  * @param offset The 0-based index of the first result to get.
+  */
+  async bqrsDecode(bqrsPath: string, resultSet: string, pageSize?: number, offset?: number): Promise<DecodedBqrsChunk> {
+    const subcommandArgs = [
+      "--entities=url,string",
+      "--result-set", resultSet,
+    ].concat(
+      pageSize ? ["--rows", pageSize.toString()] : []
+    ).concat(
+      offset ? ["--start-at", offset.toString()] : []
+    ).concat([bqrsPath]);
+    return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, "Reading bqrs data");
+  }
 
 
-  async interpretBqrs(metadata: { kind: string, id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     const args = [
       `-t=kind=${metadata.kind}`,
       `-t=id=${metadata.id}`,
@@ -347,12 +535,12 @@ export class CodeQLCliServer implements Disposable {
     try {
       output = await fs.readFile(interpretedResultsPath, 'utf8');
     } catch (err) {
-      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`)
+      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
     }
     try {
       return JSON.parse(output) as sarif.Log;
     } catch (err) {
-      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`)
+      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
     }
   }
 
@@ -409,11 +597,16 @@ export class CodeQLCliServer implements Disposable {
 
   /**
    * Gets information about available qlpacks
-   * @param searchPath A list of directories to search for qlpacks
+   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
+   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
+   *   the default CLI search path is used.
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
-  resolveQlpacks(searchPath: string[]): Promise<QlpacksInfo> {
-    const args = ['--additional-packs', searchPath.join(path.delimiter)];
+  resolveQlpacks(additionalPacks: string[], searchPath?: string[]): Promise<QlpacksInfo> {
+    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    if (searchPath !== undefined) {
+      args.push('--search-path', path.join(...searchPath));
+    }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(
       ['resolve', 'qlpacks'],
@@ -421,6 +614,27 @@ export class CodeQLCliServer implements Disposable {
       "Resolving qlpack information",
     );
   }
+
+  /**
+   * Gets information about queries in a query suite.
+   * @param suite The suite to resolve.
+   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
+   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
+   *   the default CLI search path is used.
+   * @returns A list of query files found.
+   */
+  resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
+    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    if (searchPath !== undefined) {
+      args.push('--search-path', path.join(...searchPath));
+    }
+    args.push(suite);
+    return this.runJsonCodeQlCliCommand<string[]>(
+      ['resolve', 'queries'],
+      args,
+      "Resolving queries",
+    );
+  }
 }
 
 /**
@@ -500,6 +714,104 @@ export async function runCodeQlCliCommand(codeQlPath: string, command: string[],
     logger.log(`CLI command succeeded.`);
     return result.stdout;
   } catch (err) {
-    throw new Error(`${description} failed: ${err.stderr || err}`)
+    throw new Error(`${description} failed: ${err.stderr || err}`);
+  }
+}
+
+/**
+ * Buffer to hold state used when splitting a text stream into lines.
+ */
+class SplitBuffer {
+  private readonly decoder = new StringDecoder('utf8');
+  private readonly maxSeparatorLength: number;
+  private buffer = '';
+  private searchIndex = 0;
+
+  constructor(private readonly separators: readonly string[]) {
+    this.maxSeparatorLength = separators.map(s => s.length).reduce((a, b) => Math.max(a, b), 0);
+  }
+
+  /**
+   * Append new text data to the buffer.
+   * @param chunk The chunk of data to append.
+   */
+  public addChunk(chunk: Buffer): void {
+    this.buffer += this.decoder.write(chunk);
+  }
+
+  /**
+   * Signal that the end of the input stream has been reached.
+   */
+  public end(): void {
+    this.buffer += this.decoder.end();
+    this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
+  }
+
+  /**
+   * Extract the next full line from the buffer, if one is available.
+   * @returns The text of the next available full line (without the separator), or `undefined` if no
+   * line is available.
+   */
+  public getNextLine(): string | undefined {
+    while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
+      for (const separator of this.separators) {
+        if (this.buffer.startsWith(separator, this.searchIndex)) {
+          const line = this.buffer.substr(0, this.searchIndex);
+          this.buffer = this.buffer.substr(this.searchIndex + separator.length);
+          this.searchIndex = 0;
+          return line;
+        }
+      }
+      this.searchIndex++;
+    }
+
+    return undefined;
+  }
+}
+
+/**
+ * Splits a text stream into lines based on a list of valid line separators.
+ * @param stream The text stream to split. This stream will be fully consumed.
+ * @param separators The list of strings that act as line separators.
+ * @returns A sequence of lines (not including separators).
+ */
+async function* splitStreamAtSeparators(
+  stream: Readable, separators: string[]
+): AsyncGenerator<string, void, unknown> {
+
+  const buffer = new SplitBuffer(separators);
+  for await (const chunk of stream) {
+    buffer.addChunk(chunk);
+    let line: string | undefined;
+    do {
+      line = buffer.getNextLine();
+      if (line !== undefined) {
+        yield line;
+      }
+    } while (line !== undefined);
+  }
+  buffer.end();
+  let line: string | undefined;
+  do {
+    line = buffer.getNextLine();
+    if (line !== undefined) {
+      yield line;
+    }
+  } while (line !== undefined);
+}
+
+/**
+ *  Standard line endings for splitting human-readable text.
+ */
+const lineEndings = ['\r\n', '\r', '\n'];
+
+/**
+ * Log a text stream to a `Logger` interface.
+ * @param stream The stream to log.
+ * @param logger The logger that will consume the stream output.
+ */
+async function logStream(stream: Readable, logger: Logger): Promise<void> {
+  for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
+    logger.log(line);
   }
 }

extensions/ql-vscode/src/config.ts

@@ -1,5 +1,5 @@
 import { DisposableObject } from 'semmle-vscode-utils';
-import { workspace, Event, EventEmitter, ConfigurationChangeEvent } from 'vscode';
+import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
 
@@ -27,10 +27,30 @@ class Setting {
     }
     return workspace.getConfiguration(this.parent.qualifiedName).get<T>(this.name)!;
   }
+
+  updateValue<T>(value: T, target: ConfigurationTarget): Thenable<void> {
+    if (this.parent === undefined) {
+      throw new Error('Cannot update the value of a root setting.');
+    }
+    return workspace.getConfiguration(this.parent.qualifiedName).update(this.name, value, target);
+  }
+
 }
 
 const ROOT_SETTING = new Setting('codeQL');
 
+// Enable experimental features
+
+/**
+ * Any settings below are deliberately not in package.json so that
+ * they do not appear in the settings ui in vscode itself. If users
+ * want to enable experimental features, they can add them directly in
+ * their vscode settings json file.
+ */
+
+/* Advanced setting: used to enable bqrs parsing in the cli instead of in the webview. */
+export const EXPERIMENTAL_BQRS_SETTING = new Setting('experimentalBqrsParsing', ROOT_SETTING);
+
 // Distribution configuration
 
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
@@ -59,16 +79,17 @@ const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
+export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
 const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
 
 export interface QueryServerConfig {
-  codeQlPath: string,
-  debug: boolean,
-  numThreads: number,
-  queryMemoryMb?: number,
-  timeoutSecs: number,
+  codeQlPath: string;
+  debug: boolean;
+  numThreads: number;
+  queryMemoryMb?: number;
+  timeoutSecs: number;
   onDidChangeQueryServerConfiguration?: Event<void>;
 }
 
@@ -76,7 +97,7 @@ export interface QueryServerConfig {
 const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING];
 
 export interface QueryHistoryConfig {
-  format: string,
+  format: string;
   onDidChangeQueryHistoryConfiguration: Event<void>;
 }
 
@@ -111,7 +132,7 @@ abstract class ConfigListener extends DisposableObject {
 
 export class DistributionConfigListener extends ConfigListener implements DistributionConfig {
   public get customCodeQlPath(): string | undefined {
-    return CUSTOM_CODEQL_PATH_SETTING.getValue() ? CUSTOM_CODEQL_PATH_SETTING.getValue() : undefined;
+    return CUSTOM_CODEQL_PATH_SETTING.getValue() || undefined;
   }
 
   public get includePrerelease(): boolean {
@@ -119,7 +140,7 @@ export class DistributionConfigListener extends ConfigListener implements Distri
   }
 
   public get personalAccessToken(): string | undefined {
-    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() ? PERSONAL_ACCESS_TOKEN_SETTING.getValue() : undefined;
+    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
   public get onDidChangeDistributionConfiguration(): Event<void> {

extensions/ql-vscode/src/databaseFetcher.ts

@@ -0,0 +1,442 @@
+import fetch, { Response } from "node-fetch";
+import * as unzipper from "unzipper";
+import {
+  Uri,
+  ProgressOptions,
+  ProgressLocation,
+  commands,
+  window,
+} from "vscode";
+import * as fs from "fs-extra";
+import * as path from "path";
+import { DatabaseManager, DatabaseItem } from "./databases";
+import {
+  ProgressCallback,
+  showAndLogErrorMessage,
+  withProgress,
+  showAndLogInformationMessage,
+} from "./helpers";
+import { logger } from "./logging";
+
+/**
+ * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
+ *
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function promptImportInternetDatabase(
+  databasesManager: DatabaseManager,
+  storagePath: string
+): Promise<DatabaseItem | undefined> {
+  let item: DatabaseItem | undefined = undefined;
+
+  try {
+    const databaseUrl = await window.showInputBox({
+      prompt: "Enter URL of zipfile of database to download",
+    });
+    if (databaseUrl) {
+      validateHttpsUrl(databaseUrl);
+
+      const progressOptions: ProgressOptions = {
+        location: ProgressLocation.Notification,
+        title: "Adding database from URL",
+        cancellable: false,
+      };
+      await withProgress(
+        progressOptions,
+        async (progress) =>
+          (item = await databaseArchiveFetcher(
+            databaseUrl,
+            databasesManager,
+            storagePath,
+            progress
+          ))
+      );
+      commands.executeCommand("codeQLDatabases.focus");
+    }
+    showAndLogInformationMessage(
+      "Database downloaded and imported successfully."
+    );
+  } catch (e) {
+    showAndLogErrorMessage(e.message);
+  }
+
+  return item;
+}
+
+/**
+ * Prompts a user to fetch a database from lgtm.
+ * User enters a project url and then the user is asked which language
+ * to download (if there is more than one)
+ *
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function promptImportLgtmDatabase(
+  databasesManager: DatabaseManager,
+  storagePath: string
+): Promise<DatabaseItem | undefined> {
+  let item: DatabaseItem | undefined = undefined;
+
+  try {
+    const lgtmUrl = await window.showInputBox({
+      prompt:
+        "Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)",
+    });
+    if (!lgtmUrl) {
+      return;
+    }
+    if (looksLikeLgtmUrl(lgtmUrl)) {
+      const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
+      if (databaseUrl) {
+        const progressOptions: ProgressOptions = {
+          location: ProgressLocation.Notification,
+          title: "Adding database from LGTM",
+          cancellable: false,
+        };
+        await withProgress(
+          progressOptions,
+          async (progress) =>
+            (item = await databaseArchiveFetcher(
+              databaseUrl,
+              databasesManager,
+              storagePath,
+              progress
+            ))
+        );
+        commands.executeCommand("codeQLDatabases.focus");
+      }
+    } else {
+      throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+    }
+    if (item) {
+      showAndLogInformationMessage(
+        "Database downloaded and imported successfully."
+      );
+    }
+  } catch (e) {
+    showAndLogErrorMessage(e.message);
+  }
+
+  return item;
+}
+
+/**
+ * Imports a database from a local archive.
+ *
+ * @param databaseUrl the file url of the archive to import
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function importArchiveDatabase(
+  databaseUrl: string,
+  databasesManager: DatabaseManager,
+  storagePath: string
+): Promise<DatabaseItem | undefined> {
+  let item: DatabaseItem | undefined = undefined;
+  try {
+    const progressOptions: ProgressOptions = {
+      location: ProgressLocation.Notification,
+      title: "Importing database from archive",
+      cancellable: false,
+    };
+    await withProgress(
+      progressOptions,
+      async (progress) =>
+        (item = await databaseArchiveFetcher(
+          databaseUrl,
+          databasesManager,
+          storagePath,
+          progress
+        ))
+    );
+    commands.executeCommand("codeQLDatabases.focus");
+
+    if (item) {
+      showAndLogInformationMessage(
+        "Database unzipped and imported successfully."
+      );
+    }
+  } catch (e) {
+    showAndLogErrorMessage(e.message);
+  }
+  return item;
+}
+
+/**
+ * Fetches an archive database. The database might be on the internet
+ * or in the local filesystem.
+ *
+ * @param databaseUrl URL from which to grab the database
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ * @param progressCallback optional callback to send progress messages to
+ */
+async function databaseArchiveFetcher(
+  databaseUrl: string,
+  databasesManager: DatabaseManager,
+  storagePath: string,
+  progressCallback?: ProgressCallback
+): Promise<DatabaseItem> {
+  progressCallback?.({
+    maxStep: 3,
+    message: "Getting database",
+    step: 1,
+  });
+  if (!storagePath) {
+    throw new Error("No storage path specified.");
+  }
+  await fs.ensureDir(storagePath);
+  const unzipPath = await getStorageFolder(storagePath, databaseUrl);
+
+  if (isFile(databaseUrl)) {
+    await readAndUnzip(databaseUrl, unzipPath);
+  } else {
+    await fetchAndUnzip(databaseUrl, unzipPath, progressCallback);
+  }
+
+  progressCallback?.({
+    maxStep: 3,
+    message: "Opening database",
+    step: 3,
+  });
+
+  // find the path to the database. The actual database might be in a sub-folder
+  const dbPath = await findDirWithFile(
+    unzipPath,
+    ".dbinfo",
+    "codeql-database.yml"
+  );
+  if (dbPath) {
+    const item = await databasesManager.openDatabase(Uri.file(dbPath));
+    databasesManager.setCurrentDatabaseItem(item);
+    return item;
+  } else {
+    throw new Error("Database not found in archive.");
+  }
+}
+
+async function getStorageFolder(storagePath: string, urlStr: string) {
+  // we need to generate a folder name for the unzipped archive,
+  // this needs to be human readable since we may use this name as the initial
+  // name for the database
+  const url = Uri.parse(urlStr);
+  // MacOS has a max filename length of 255
+  // and remove a few extra chars in case we need to add a counter at the end.
+  let lastName = path.basename(url.path).substring(0, 250);
+  if (lastName.endsWith(".zip")) {
+    lastName = lastName.substring(0, lastName.length - 4);
+  }
+
+  const realpath = await fs.realpath(storagePath);
+  let folderName = path.join(realpath, lastName);
+
+  // avoid overwriting existing folders
+  let counter = 0;
+  while (await fs.pathExists(folderName)) {
+    counter++;
+    folderName = path.join(realpath, `${lastName}-${counter}`);
+    if (counter > 100) {
+      throw new Error("Could not find a unique name for downloaded database.");
+    }
+  }
+  return folderName;
+}
+
+function validateHttpsUrl(databaseUrl: string) {
+  let uri;
+  try {
+    uri = Uri.parse(databaseUrl, true);
+  } catch (e) {
+    throw new Error(`Invalid url: ${databaseUrl}`);
+  }
+
+  if (uri.scheme !== "https") {
+    throw new Error("Must use https for downloading a database.");
+  }
+}
+
+async function readAndUnzip(databaseUrl: string, unzipPath: string) {
+  const unzipStream = unzipper.Extract({
+    path: unzipPath,
+  });
+
+  await new Promise((resolve, reject) => {
+    // we already know this is a file scheme
+    const databaseFile = Uri.parse(databaseUrl).fsPath;
+    const stream = fs.createReadStream(databaseFile);
+    stream.on("error", reject);
+    unzipStream.on("error", reject);
+    unzipStream.on("close", resolve);
+    stream.pipe(unzipStream);
+  });
+}
+
+async function fetchAndUnzip(
+  databaseUrl: string,
+  unzipPath: string,
+  progressCallback?: ProgressCallback
+) {
+  const response = await fetch(databaseUrl);
+
+  await checkForFailingResponse(response);
+
+  const unzipStream = unzipper.Extract({
+    path: unzipPath,
+  });
+  progressCallback?.({
+    maxStep: 3,
+    message: "Unzipping database",
+    step: 2,
+  });
+  await new Promise((resolve, reject) => {
+    const handler = (err: Error) => {
+      if (err.message.startsWith('invalid signature')) {
+        reject(new Error('Not a valid archive.'));
+      } else {
+        reject(err);
+      }
+    };
+    response.body.on("error", handler);
+    unzipStream.on("error", handler);
+    unzipStream.on("close", resolve);
+    response.body.pipe(unzipStream);
+  });
+}
+
+async function checkForFailingResponse(response: Response): Promise<void | never> {
+  if (response.ok) {
+    return;
+  }
+
+  // An error downloading the database. Attempt to extract the resaon behind it.
+  const text = await response.text();
+  let msg: string;
+  try {
+    const obj = JSON.parse(text);
+    msg = obj.error || obj.message || obj.reason || JSON.stringify(obj, null, 2);
+  } catch (e) {
+    msg = text;
+  }
+  throw new Error(`Error downloading database.\n\nReason: ${msg}`);
+}
+
+function isFile(databaseUrl: string) {
+  return Uri.parse(databaseUrl).scheme === "file";
+}
+
+/**
+ * Recursively looks for a file in a directory. If the file exists, then returns the directory containing the file.
+ *
+ * @param dir The directory to search
+ * @param toFind The file to recursively look for in this directory
+ *
+ * @returns the directory containing the file, or undefined if not found.
+ */
+// exported for testing
+export async function findDirWithFile(
+  dir: string,
+  ...toFind: string[]
+): Promise<string | undefined> {
+  if (!(await fs.stat(dir)).isDirectory()) {
+    return;
+  }
+  const files = await fs.readdir(dir);
+  if (toFind.some((file) => files.includes(file))) {
+    return dir;
+  }
+  for (const file of files) {
+    const newPath = path.join(dir, file);
+    const result = await findDirWithFile(newPath, ...toFind);
+    if (result) {
+      return result;
+    }
+  }
+  return;
+}
+
+/**
+ * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
+ * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
+ *
+ * After the {provider}/{org}/{name} path components, there may be the components
+ * related to sub pages.
+ *
+ * This function accepts any url that matches the patter above
+ *
+ * @param lgtmUrl The URL to the lgtm project
+ *
+ * @return true if this looks like an LGTM project url
+ */
+// exported for testing
+export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string {
+  if (!lgtmUrl) {
+    return false;
+  }
+
+  try {
+    const uri = Uri.parse(lgtmUrl, true);
+    if (uri.scheme !== "https") {
+      return false;
+    }
+
+    if (uri.authority !== "lgtm.com" && uri.authority !== "www.lgtm.com") {
+      return false;
+    }
+
+    const paths = uri.path.split("/").filter((segment) => segment);
+    return paths.length >= 4 && paths[0] === "projects";
+  } catch (e) {
+    return false;
+  }
+}
+
+// exported for testing
+export async function convertToDatabaseUrl(lgtmUrl: string) {
+  try {
+    const uri = Uri.parse(lgtmUrl, true);
+    const paths = ["api", "v1.0"].concat(
+      uri.path.split("/").filter((segment) => segment)
+    ).slice(0, 6);
+    const projectUrl = `https://lgtm.com/${paths.join("/")}`;
+    const projectResponse = await fetch(projectUrl);
+    const projectJson = await projectResponse.json();
+
+    if (projectJson.code === 404) {
+      throw new Error();
+    }
+
+    const language = await promptForLanguage(projectJson);
+    if (!language) {
+      return;
+    }
+    return `https://lgtm.com/${[
+      "api",
+      "v1.0",
+      "snapshots",
+      projectJson.id,
+      language,
+    ].join("/")}`;
+  } catch (e) {
+    logger.log(`Error: ${e.message}`);
+    throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+  }
+}
+
+async function promptForLanguage(
+  projectJson: any
+): Promise<string | undefined> {
+  if (!projectJson?.languages?.length) {
+    return;
+  }
+  if (projectJson.languages.length === 1) {
+    return projectJson.languages[0].language;
+  }
+
+  return await window.showQuickPick(
+    projectJson.languages.map((lang: { language: string }) => lang.language), {
+    placeHolder: "Select the database language to download:"
+  }
+  );
+}

extensions/ql-vscode/src/databases-ui.ts

@@ -1,22 +1,24 @@
 import * as path from 'path';
 import { DisposableObject } from 'semmle-vscode-utils';
-import { commands, Event, EventEmitter, ExtensionContext, ProviderResult, TreeDataProvider, TreeItem, Uri, window } from 'vscode';
+import { commands, Event, EventEmitter, ExtensionContext, ProviderResult, TreeDataProvider, TreeItem, Uri, window, env } from 'vscode';
 import * as cli from './cli';
 import { DatabaseItem, DatabaseManager, getUpgradesDirectories } from './databases';
-import { getOnDiskWorkspaceFolders } from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
 import { logger } from './logging';
 import { clearCacheInDatabase, UserCancellationException } from './run-queries';
 import * as qsClient from './queryserver-client';
 import { upgradeDatabase } from './upgrades';
+import { importArchiveDatabase, promptImportInternetDatabase, promptImportLgtmDatabase } from './databaseFetcher';
+import * as fs from 'fs-extra';
 
-type ThemableIconPath = { light: string, dark: string } | string;
+type ThemableIconPath = { light: string; dark: string } | string;
 
 /**
  * Path to icons to display next to currently selected database.
  */
 const SELECTED_DATABASE_ICON: ThemableIconPath = {
-  light: 'media/check-light-mode.svg',
-  dark: 'media/check-dark-mode.svg',
+  light: 'media/light/check.svg',
+  dark: 'media/dark/check.svg',
 };
 
 /**
@@ -34,12 +36,21 @@ function joinThemableIconPath(base: string, iconPath: ThemableIconPath): Themabl
     return path.join(base, iconPath);
 }
 
+enum SortOrder {
+  NameAsc = 'NameAsc',
+  NameDesc = 'NameDesc',
+  DateAddedAsc = 'DateAddedAsc',
+  DateAddedDesc = 'DateAddedDesc'
+}
+
 /**
  * Tree data provider for the databases view.
  */
 class DatabaseTreeDataProvider extends DisposableObject
   implements TreeDataProvider<DatabaseItem> {
 
+  private _sortOrder = SortOrder.NameAsc;
+
   private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
   private currentDatabaseItem: DatabaseItem | undefined;
 
@@ -84,7 +95,18 @@ class DatabaseTreeDataProvider extends DisposableObject
 
   public getChildren(element?: DatabaseItem): ProviderResult<DatabaseItem[]> {
     if (element === undefined) {
-      return this.databaseManager.databaseItems.slice(0);
+      return this.databaseManager.databaseItems.slice(0).sort((db1, db2) => {
+        switch (this.sortOrder) {
+          case SortOrder.NameAsc:
+            return db1.name.localeCompare(db2.name);
+          case SortOrder.NameDesc:
+            return db2.name.localeCompare(db1.name);
+          case SortOrder.DateAddedAsc:
+            return (db1.dateAdded || 0) - (db2.dateAdded || 0);
+          case SortOrder.DateAddedDesc:
+            return (db2.dateAdded || 0) - (db1.dateAdded || 0);
+        }
+      });
     }
     else {
       return [];
@@ -98,6 +120,15 @@ class DatabaseTreeDataProvider extends DisposableObject
   public getCurrent(): DatabaseItem | undefined {
     return this.currentDatabaseItem;
   }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire();
+  }
 }
 
 /** Gets the first element in the given list, if any, or undefined if the list is empty or undefined. */
@@ -118,40 +149,93 @@ function getFirst(list: Uri[] | undefined): Uri | undefined {
  * XXX: no validation is done other than checking the directory name
  * to make sure it really is a database directory.
  */
-async function chooseDatabaseDir(): Promise<Uri | undefined> {
+async function chooseDatabaseDir(byFolder: boolean): Promise<Uri | undefined> {
   const chosen = await window.showOpenDialog({
-    openLabel: 'Choose Database',
-    canSelectFiles: true,
-    canSelectFolders: true,
-    canSelectMany: false
+    openLabel: byFolder ? 'Choose Database folder' : 'Choose Database archive',
+    canSelectFiles: !byFolder,
+    canSelectFolders: byFolder,
+    canSelectMany: false,
+    filters: byFolder ? {} : { Archives: ['zip'] }
   });
   return getFirst(chosen);
 }
 
 export class DatabaseUI extends DisposableObject {
-  public constructor(ctx: ExtensionContext, private cliserver: cli.CodeQLCliServer, private databaseManager: DatabaseManager,
-    private readonly queryServer: qsClient.QueryServerClient | undefined) {
+  private treeDataProvider: DatabaseTreeDataProvider;
 
+  public constructor(
+    ctx: ExtensionContext,
+    private cliserver: cli.CodeQLCliServer,
+    private databaseManager: DatabaseManager,
+    private readonly queryServer: qsClient.QueryServerClient | undefined,
+    private readonly storagePath: string
+  ) {
     super();
 
-    const treeDataProvider = this.push(new DatabaseTreeDataProvider(ctx, databaseManager));
-    this.push(window.createTreeView('codeQLDatabases', { treeDataProvider }));
+    this.treeDataProvider = this.push(new DatabaseTreeDataProvider(ctx, databaseManager));
+    this.push(window.createTreeView('codeQLDatabases', { treeDataProvider: this.treeDataProvider }));
 
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabase', this.handleChooseDatabase));
+    logger.log('Registering database panel commands.');
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseFolder', this.handleChooseDatabaseFolder));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseArchive', this.handleChooseDatabaseArchive));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseInternet', this.handleChooseDatabaseInternet));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseLgtm', this.handleChooseDatabaseLgtm));
     ctx.subscriptions.push(commands.registerCommand('codeQL.setCurrentDatabase', this.handleSetCurrentDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQL.upgradeCurrentDatabase', this.handleUpgradeCurrentDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQL.clearCache', this.handleClearCache));
     ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.setCurrentDatabase', this.handleMakeCurrentDatabase));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.sortByName', this.handleSortByName));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.sortByDateAdded', this.handleSortByDateAdded));
     ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.removeDatabase', this.handleRemoveDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.upgradeDatabase', this.handleUpgradeDatabase));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.renameDatabase', this.handleRenameDatabase));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.openDatabaseFolder', this.handleOpenFolder));
   }
 
   private handleMakeCurrentDatabase = async (databaseItem: DatabaseItem): Promise<void> => {
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
   }
 
-  private handleChooseDatabase = async (): Promise<DatabaseItem | undefined> => {
-    return await this.chooseAndSetDatabase();
+  handleChooseDatabaseFolder = async (): Promise<DatabaseItem | undefined> => {
+    try {
+      return await this.chooseAndSetDatabase(true);
+    } catch (e) {
+      showAndLogErrorMessage(e.message);
+      return undefined;
+    }
+  }
+
+  handleChooseDatabaseArchive = async (): Promise<DatabaseItem | undefined> => {
+    try {
+      return await this.chooseAndSetDatabase(false);
+    } catch (e) {
+      showAndLogErrorMessage(e.message);
+      return undefined;
+    }
+  }
+
+  handleChooseDatabaseInternet = async (): Promise<DatabaseItem | undefined> => {
+    return await promptImportInternetDatabase(this.databaseManager, this.storagePath);
+  }
+
+  handleChooseDatabaseLgtm = async (): Promise<DatabaseItem | undefined> => {
+    return await promptImportLgtmDatabase(this.databaseManager, this.storagePath);
+  }
+
+  private handleSortByName = async () => {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  }
+
+  private handleSortByDateAdded = async () => {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAddedAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateAddedDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAddedAsc;
+    }
   }
 
   private handleUpgradeCurrentDatabase = async (): Promise<void> => {
@@ -213,6 +297,11 @@ export class DatabaseUI extends DisposableObject {
   }
 
   private handleSetCurrentDatabase = async (uri: Uri): Promise<DatabaseItem | undefined> => {
+    // Assume user has selected an archive if the file has a .zip extension
+    if (uri.path.endsWith('.zip')) {
+      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
+    }
+
     return await this.setCurrentDatabase(uri);
   }
 
@@ -220,6 +309,29 @@ export class DatabaseUI extends DisposableObject {
     this.databaseManager.removeDatabaseItem(databaseItem);
   }
 
+  private handleRenameDatabase = async (databaseItem: DatabaseItem): Promise<void> => {
+    try {
+      const newName = await window.showInputBox({
+        prompt: 'Choose new database name',
+        value: databaseItem.name
+      });
+
+      if (newName) {
+        this.databaseManager.renameDatabaseItem(databaseItem, newName);
+      }
+    } catch (e) {
+      showAndLogErrorMessage(e.message);
+    }
+  }
+
+  private handleOpenFolder = async (databaseItem: DatabaseItem): Promise<void> => {
+    try {
+      await env.openExternal(databaseItem.databaseUri);
+    } catch (e) {
+      showAndLogErrorMessage(e.message);
+    }
+  }
+
   /**
    * Return the current database directory. If we don't already have a
    * current database, ask the user for one, and return that, or
@@ -227,7 +339,7 @@ export class DatabaseUI extends DisposableObject {
    */
   public async getDatabaseItem(): Promise<DatabaseItem | undefined> {
     if (this.databaseManager.currentDatabaseItem === undefined) {
-      await this.chooseAndSetDatabase();
+      await this.chooseAndSetDatabase(false);
     }
 
     return this.databaseManager.currentDatabaseItem;
@@ -247,13 +359,44 @@ export class DatabaseUI extends DisposableObject {
    * Ask the user for a database directory. Returns the chosen database, or `undefined` if the
    * operation was canceled.
    */
-  private async chooseAndSetDatabase(): Promise<DatabaseItem | undefined> {
-    const uri = await chooseDatabaseDir();
-    if (uri !== undefined) {
-      return await this.setCurrentDatabase(uri);
-    }
-    else {
+  private async chooseAndSetDatabase(byFolder: boolean): Promise<DatabaseItem | undefined> {
+    const uri = await chooseDatabaseDir(byFolder);
+
+    if (!uri) {
       return undefined;
     }
+
+    if (byFolder) {
+      const fixedUri = await this.fixDbUri(uri);
+      // we are selecting a database folder
+      return await this.setCurrentDatabase(fixedUri);
+    }
+    else {
+      // we are selecting a database archive. Must unzip into a workspace-controlled area
+      // before importing.
+      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
+    }
+  }
+
+  /**
+   * Perform some heuristics to ensure a proper database location is chosen.
+   *
+   * 1. If the selected URI to add is a file, choose the containing directory
+   * 2. If the selected URI is a directory matching db-*, choose the containing directory
+   * 3. choose the current directory
+   *
+   * @param uri a URI that is a datbase folder or inside it
+   *
+   * @return the actual database folder found by using the heuristics above.
+   */
+  private async fixDbUri(uri: Uri): Promise<Uri> {
+    let dbPath = uri.fsPath;
+    if ((await fs.stat(dbPath)).isFile()) {
+      dbPath = path.dirname(dbPath);
+    }
+    if (path.basename(dbPath).startsWith('db-')) {
+      dbPath = path.dirname(dbPath);
+    }
+    return Uri.file(dbPath);
   }
 }

extensions/ql-vscode/src/databases.ts

@@ -24,21 +24,23 @@ import { Logger, logger } from './logging';
  * The name of the key in the workspaceState dictionary in which we
  * persist the current database across sessions.
  */
-const CURRENT_DB: string = 'currentDatabase';
+const CURRENT_DB = 'currentDatabase';
 
 /**
  * The name of the key in the workspaceState dictionary in which we
  * persist the list of databases across sessions.
  */
-const DB_LIST: string = 'databaseList';
+const DB_LIST = 'databaseList';
 
 export interface DatabaseOptions {
   displayName?: string;
   ignoreSourceArchive?: boolean;
+  dateAdded?: number | undefined;
 }
 
 interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
+  dateAdded: number | undefined;
 }
 
 interface PersistedDatabaseItem {
@@ -107,10 +109,11 @@ async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
   return vscode.Uri.file(dbAbsolutePath);
 }
 
-async function findSourceArchive(databasePath: string, silent: boolean = false):
-  Promise<vscode.Uri | undefined> {
+async function findSourceArchive(
+  databasePath: string, silent = false
+): Promise<vscode.Uri | undefined> {
 
-  const relativePaths = ['src', 'output/src_archive']
+  const relativePaths = ['src', 'output/src_archive'];
 
   for (const relativePath of relativePaths) {
     const basePath = path.join(databasePath, relativePath);
@@ -128,8 +131,9 @@ async function findSourceArchive(databasePath: string, silent: boolean = false):
   return undefined;
 }
 
-async function resolveDatabase(databasePath: string):
-  Promise<DatabaseContents | undefined> {
+async function resolveDatabase(
+  databasePath: string
+): Promise<DatabaseContents | undefined> {
 
   const name = path.basename(databasePath);
 
@@ -200,7 +204,7 @@ export interface DatabaseItem {
   /** The URI of the database */
   readonly databaseUri: vscode.Uri;
   /** The name of the database to be displayed in the UI */
-  readonly name: string;
+  name: string;
   /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
   readonly sourceArchive: vscode.Uri | undefined;
   /**
@@ -208,6 +212,12 @@ export interface DatabaseItem {
    * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`.
    */
   readonly contents: DatabaseContents | undefined;
+
+  /**
+   * The date this database was added as a unix timestamp. Or undefined if we don't know.
+   */
+  readonly dateAdded: number | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
   /**
@@ -278,6 +288,10 @@ class DatabaseItemImpl implements DatabaseItem {
     }
   }
 
+  public set name(newName: string) {
+    this.options.displayName = newName;
+  }
+
   public get sourceArchive(): vscode.Uri | undefined {
     if (this.options.ignoreSourceArchive || (this._contents === undefined)) {
       return undefined;
@@ -291,6 +305,10 @@ class DatabaseItemImpl implements DatabaseItem {
     return this._contents;
   }
 
+  public get dateAdded(): number | undefined {
+    return this.options.dateAdded;
+  }
+
   public get error(): Error | undefined {
     return this._error;
   }
@@ -427,31 +445,30 @@ class DatabaseItemImpl implements DatabaseItem {
  * `event` fires. If waiting for the event takes too long (by default
  * >1000ms) log a warning, and resolve to undefined.
  */
-function eventFired<T>(event: vscode.Event<T>, timeoutMs: number = 1000): Promise<T | undefined> {
+function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | undefined> {
   return new Promise((res, _rej) => {
-    let timeout: NodeJS.Timeout | undefined;
-    let disposable: vscode.Disposable | undefined;
-    function dispose() {
-      if (timeout !== undefined) clearTimeout(timeout);
-      if (disposable !== undefined) disposable.dispose();
-    }
-    disposable = event(e => {
-      res(e); dispose();
-    });
-    timeout = setTimeout(() => {
+    const timeout = setTimeout(() => {
       logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
-      res(undefined); dispose();
+      res(undefined);
+      dispose();
     }, timeoutMs);
+    const disposable = event(e => {
+      res(e);
+      dispose();
+    });
+    function dispose() {
+      clearTimeout(timeout);
+      disposable.dispose();
+    }
   });
 }
 
 export class DatabaseManager extends DisposableObject {
-  private readonly _onDidChangeDatabaseItem =
-    this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  private readonly _onDidChangeDatabaseItem = this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+
   readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
 
-  private readonly _onDidChangeCurrentDatabaseItem =
-    this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  private readonly _onDidChangeCurrentDatabaseItem = this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
   readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event;
 
   private readonly _databaseItems: DatabaseItemImpl[] = [];
@@ -465,8 +482,9 @@ export class DatabaseManager extends DisposableObject {
     this.loadPersistedState();  // Let this run async.
   }
 
-  public async openDatabase(uri: vscode.Uri, options?: DatabaseOptions):
-    Promise<DatabaseItem> {
+  public async openDatabase(
+    uri: vscode.Uri, options?: DatabaseOptions
+  ): Promise<DatabaseItem> {
 
     const contents = await resolveDatabaseContents(uri);
     const realOptions = options || {};
@@ -475,7 +493,8 @@ export class DatabaseManager extends DisposableObject {
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ?
         realOptions.ignoreSourceArchive : isQLTestDatabase,
-      displayName: realOptions.displayName
+      displayName: realOptions.displayName,
+      dateAdded: realOptions.dateAdded || Date.now()
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (item) => {
       this._onDidChangeDatabaseItem.fire(item);
@@ -525,11 +544,13 @@ export class DatabaseManager extends DisposableObject {
     }
   }
 
-  private async createDatabaseItemFromPersistedState(state: PersistedDatabaseItem):
-    Promise<DatabaseItem> {
+  private async createDatabaseItemFromPersistedState(
+    state: PersistedDatabaseItem
+  ): Promise<DatabaseItem> {
 
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
+    let dateAdded = undefined;
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -537,14 +558,18 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.ignoreSourceArchive === 'boolean') {
         ignoreSourceArchive = state.options.ignoreSourceArchive;
       }
+      if (typeof state.options.dateAdded === 'number') {
+        dateAdded = state.options.dateAdded;
+      }
     }
     const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: ignoreSourceArchive,
-      displayName: displayName
+      ignoreSourceArchive,
+      displayName,
+      dateAdded
     };
     const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri), undefined, fullOptions,
       (item) => {
-        this._onDidChangeDatabaseItem.fire(item)
+        this._onDidChangeDatabaseItem.fire(item);
       });
     await this.addDatabaseItem(item);
 
@@ -571,7 +596,7 @@ export class DatabaseManager extends DisposableObject {
       }
     } catch (e) {
       // database list had an unexpected type - nothing to be done?
-      showAndLogErrorMessage('Database list loading failed: ${}', e.message);
+      showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
     }
   }
 
@@ -584,7 +609,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   public async setCurrentDatabaseItem(item: DatabaseItem | undefined,
-    skipRefresh: boolean = false): Promise<void> {
+    skipRefresh = false): Promise<void> {
 
     if (!skipRefresh && (item !== undefined)) {
       await item.refresh();  // Will throw on invalid database.
@@ -610,12 +635,23 @@ export class DatabaseManager extends DisposableObject {
     return this._databaseItems.find(item => item.databaseUri.toString(true) === uriString);
   }
 
+  public findDatabaseItemBySourceArchive(uri: vscode.Uri): DatabaseItem | undefined {
+    const uriString = uri.toString(true);
+    return this._databaseItems.find(item => item.sourceArchive && item.sourceArchive.toString(true) === uriString);
+  }
+
   private async addDatabaseItem(item: DatabaseItemImpl) {
     this._databaseItems.push(item);
     this.updatePersistedDatabaseList();
     this._onDidChangeDatabaseItem.fire(undefined);
   }
 
+  public async renameDatabaseItem(item: DatabaseItem, newName: string) {
+    item.name = newName;
+    this.updatePersistedDatabaseList();
+    this._onDidChangeDatabaseItem.fire(item);
+  }
+
   public removeDatabaseItem(item: DatabaseItem) {
     if (this._currentDatabaseItem == item)
       this._currentDatabaseItem = undefined;
@@ -632,6 +668,14 @@ export class DatabaseManager extends DisposableObject {
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
+    // Delete folder from file system only if it is controlled by the extension
+    if (this.isExtensionControlledLocation(item.databaseUri)) {
+      logger.log(`Deleting database from filesystem.`);
+      fs.remove(item.databaseUri.path).then(
+        () => logger.log(`Deleted '${item.databaseUri.path}'`),
+        e => logger.log(`Failed to delete '${item.databaseUri.path}'. Reason: ${e.message}`));
+    }
+
     this._onDidChangeDatabaseItem.fire(undefined);
   }
 
@@ -643,6 +687,11 @@ export class DatabaseManager extends DisposableObject {
   private updatePersistedDatabaseList(): void {
     this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
   }
+
+  private isExtensionControlledLocation(uri: vscode.Uri) {
+    const storagePath = this.ctx.storagePath || this.ctx.globalStoragePath;
+    return uri.path.startsWith(storagePath);
+  }
 }
 
 /**

extensions/ql-vscode/src/definitions.ts

@@ -0,0 +1,214 @@
+import * as fs from 'fs-extra';
+import * as yaml from 'js-yaml';
+import * as tmp from 'tmp';
+import * as vscode from "vscode";
+import { decodeSourceArchiveUri, zipArchiveScheme } from "./archive-filesystem-provider";
+import { ColumnKindCode, EntityValue, getResultSetSchema, LineColumnLocation, UrlValue } from "./bqrs-cli-types";
+import { CodeQLCliServer } from "./cli";
+import { DatabaseItem, DatabaseManager } from "./databases";
+import * as helpers from './helpers';
+import { CachedOperation } from './helpers';
+import * as messages from "./messages";
+import { QueryServerClient } from "./queryserver-client";
+import { compileAndRunQueryAgainstDatabase, QueryWithResults } from "./run-queries";
+
+/**
+ * Run templated CodeQL queries to find definitions and references in
+ * source-language files. We may eventually want to find a way to
+ * generalize this to other custom queries, e.g. showing dataflow to
+ * or from a selected identifier.
+ */
+
+const TEMPLATE_NAME = "selectedSourceFile";
+const SELECT_QUERY_NAME = "#select";
+
+enum KeyType {
+  DefinitionQuery = 'DefinitionQuery',
+  ReferenceQuery = 'ReferenceQuery',
+}
+
+function tagOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery: return "ide-contextual-queries/local-definitions";
+    case KeyType.ReferenceQuery: return "ide-contextual-queries/local-references";
+  }
+}
+
+function nameOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery: return "definitions";
+    case KeyType.ReferenceQuery: return "references";
+  }
+}
+
+async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
+  const suiteFile = tmp.fileSync({ postfix: '.qls' }).name;
+  const suiteYaml = { qlpack, include: { kind: 'definitions', 'tags contain': tagOfKeyType(keyType) } };
+  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
+
+  const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
+  if (queries.length === 0) {
+    vscode.window.showErrorMessage(
+      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. It might be necessary to upgrade the CodeQL libraries.`
+    );
+    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
+  }
+  return queries;
+}
+
+async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string | undefined> {
+  if (db.contents === undefined)
+    return undefined;
+  const datasetPath = db.contents.datasetUri.fsPath;
+  const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath);
+  return qlpack;
+}
+
+interface FullLocationLink extends vscode.LocationLink {
+  originUri: vscode.Uri;
+}
+
+export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
+  private cache: CachedOperation<vscode.LocationLink[]>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+  }
+
+  async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+    return getLinksForUriString(this.cli, this.qs, this.dbm, uriString, KeyType.DefinitionQuery, (src, _dest) => src === uriString);
+  }
+
+  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+    const fileLinks = await this.cache.get(document.uri.toString());
+    const locLinks: vscode.LocationLink[] = [];
+    for (const link of fileLinks) {
+      if (link.originSelectionRange!.contains(position)) {
+        locLinks.push(link);
+      }
+    }
+    return locLinks;
+  }
+}
+
+export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+  private cache: CachedOperation<FullLocationLink[]>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<FullLocationLink[]>(this.getReferences.bind(this));
+  }
+
+  async getReferences(uriString: string): Promise<FullLocationLink[]> {
+    return getLinksForUriString(this.cli, this.qs, this.dbm, uriString, KeyType.ReferenceQuery, (_src, dest) => dest === uriString);
+  }
+
+  async provideReferences(document: vscode.TextDocument, position: vscode.Position, _context: vscode.ReferenceContext, _token: vscode.CancellationToken): Promise<vscode.Location[]> {
+    const fileLinks = await this.cache.get(document.uri.toString());
+    const locLinks: vscode.Location[] = [];
+    for (const link of fileLinks) {
+      if (link.targetRange!.contains(position)) {
+        locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
+      }
+    }
+    return locLinks;
+  }
+}
+
+interface FileRange {
+  file: vscode.Uri;
+  range: vscode.Range;
+}
+
+async function getLinksFromResults(results: QueryWithResults, cli: CodeQLCliServer, db: DatabaseItem, filter: (srcFile: string, destFile: string) => boolean): Promise<FullLocationLink[]> {
+  const localLinks: FullLocationLink[] = [];
+  const bqrsPath = results.query.resultsPaths.resultsPath;
+  const info = await cli.bqrsInfo(bqrsPath);
+  const selectInfo = getResultSetSchema(SELECT_QUERY_NAME, info);
+  if (selectInfo && selectInfo.columns.length == 3
+    && selectInfo.columns[0].kind == ColumnKindCode.ENTITY
+    && selectInfo.columns[1].kind == ColumnKindCode.ENTITY
+    && selectInfo.columns[2].kind == ColumnKindCode.STRING) {
+    // TODO: Page this
+    const allTuples = await cli.bqrsDecode(bqrsPath, SELECT_QUERY_NAME);
+    for (const tuple of allTuples.tuples) {
+      const src = tuple[0] as EntityValue;
+      const dest = tuple[1] as EntityValue;
+      const srcFile = src.url && fileRangeFromURI(src.url, db);
+      const destFile = dest.url && fileRangeFromURI(dest.url, db);
+      if (srcFile && destFile && filter(srcFile.file.toString(), destFile.file.toString())) {
+        localLinks.push({ targetRange: destFile.range, targetUri: destFile.file, originSelectionRange: srcFile.range, originUri: srcFile.file });
+      }
+    }
+  }
+  return localLinks;
+}
+
+async function getLinksForUriString(
+  cli: CodeQLCliServer,
+  qs: QueryServerClient,
+  dbm: DatabaseManager,
+  uriString: string,
+  keyType: KeyType,
+  filter: (src: string, dest: string) => boolean
+) {
+  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString));
+  const sourceArchiveUri = vscode.Uri.file(uri.sourceArchiveZipPath).with({ scheme: zipArchiveScheme });
+
+  const db = dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+  if (db) {
+    const qlpack = await qlpackOfDatabase(cli, db);
+    if (qlpack === undefined) {
+      throw new Error("Can't infer qlpack from database source archive");
+    }
+    const links: FullLocationLink[] = [];
+    for (const query of await resolveQueries(cli, qlpack, keyType)) {
+      const templates: messages.TemplateDefinitions = {
+        [TEMPLATE_NAME]: {
+          values: {
+            tuples: [[{
+              stringValue: uri.pathWithinSourceArchive
+            }]]
+          }
+        }
+      };
+      const results = await compileAndRunQueryAgainstDatabase(cli, qs, db, false, vscode.Uri.file(query), templates);
+      if (results.result.resultType == messages.QueryResultType.SUCCESS) {
+        links.push(...await getLinksFromResults(results, cli, db, filter));
+      }
+    }
+    return links;
+  } else {
+    return [];
+  }
+}
+
+function fileRangeFromURI(uri: UrlValue, db: DatabaseItem): FileRange | undefined {
+  if (typeof uri === "string") {
+    return undefined;
+  } else if ('startOffset' in uri) {
+    return undefined;
+  } else {
+    const loc = uri as LineColumnLocation;
+    const range = new vscode.Range(Math.max(0, loc.startLine - 1),
+      Math.max(0, loc.startColumn - 1),
+      Math.max(0, loc.endLine - 1),
+      Math.max(0, loc.endColumn));
+    try {
+      const parsed = vscode.Uri.parse(uri.uri, true);
+      if (parsed.scheme === "file") {
+        return { file: db.resolveSourceFile(parsed.fsPath), range };
+      }
+      return undefined;
+    } catch (e) {
+      return undefined;
+    }
+  }
+}

extensions/ql-vscode/src/discovery.ts

@@ -0,0 +1,87 @@
+import { DisposableObject } from 'semmle-vscode-utils';
+
+/**
+ * Base class for "discovery" operations, which scan the file system to find specific kinds of
+ * files. This class automatically prevents more than one discovery operation from running at the
+ * same time.
+ */
+export abstract class Discovery<T> extends DisposableObject {
+  private retry = false;
+  private discoveryInProgress = false;
+
+  constructor() {
+    super();
+  }
+
+  /**
+   * Force the discovery process to run. Normally invoked by the derived class when a relevant file
+   * system change is detected.
+   */
+  public refresh(): void {
+    // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
+    // got a storm of refresh requests due to, say, the copying or deletion of a large directory
+    // tree, we could potentially spawn a separate simultaneous discovery operation for each
+    // individual file change notification.
+    // Our approach is to spawn a discovery operation immediately upon receiving the first refresh
+    // request. If we receive any additional refresh requests before the first one is complete, we
+    // record this fact by setting `this.retry = true`. When the original discovery operation
+    // completes, we discard its results and spawn another one to account for that additional
+    // changes that have happened since.
+    // The means that for the common case of a single file being modified, we'll complete the
+    // discovery and update as soon as possible. If multiple files are being modified, we'll
+    // probably wind up doing discovery at least twice.
+    // We could choose to delay the initial discovery request by a second or two to wait for any
+    // other change notifications that might be coming along. However, this would create more
+    // latency in the common case, in order to save a bit of latency in the uncommon case.
+
+    if (this.discoveryInProgress) {
+      // There's already a discovery operation in progress. Tell it to restart when it's done.
+      this.retry = true;
+    }
+    else {
+      // No discovery in progress, so start one now.
+      this.discoveryInProgress = true;
+      this.launchDiscovery();
+    }
+  }
+
+  /**
+   * Starts the asynchronous discovery operation by invoking the `discover` function. When the
+   * discovery operation completes, the `update` function will be invoked with the results of the
+   * discovery.
+   */
+  private launchDiscovery(): void {
+    const discoveryPromise = this.discover();
+    discoveryPromise.then(results => {
+      if (!this.retry) {
+        // Update any listeners with the results of the discovery.
+        this.discoveryInProgress = false;
+        this.update(results);
+      }
+    });
+    discoveryPromise.finally(() => {
+      if (this.retry) {
+        // Another refresh request came in while we were still running a previous discovery
+        // operation. Since the discovery results we just computed are now stale, we'll launch
+        // another discovery operation instead of updating.
+        // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+        // initial discovery operation failed.
+        this.retry = false;
+        this.launchDiscovery();
+      }
+    });
+  }
+
+  /**
+   * Overridden by the derived class to spawn the actual discovery operation, returning the results.
+   */
+  protected abstract discover(): Promise<T>;
+
+  /**
+   * Overridden by the derived class to atomically update the `Discovery` object with the results of
+   * the discovery operation, and to notify any listeners that the discovery results may have
+   * changed.
+   * @param results The discovery results returned by the `discover` function.
+   */
+  protected abstract update(results: T): void;
+}

extensions/ql-vscode/src/distribution.ts

@@ -2,13 +2,15 @@ import * as fetch from "node-fetch";
 import * as fs from "fs-extra";
 import * as os from "os";
 import * as path from "path";
+import * as semver from "semver";
 import * as unzipper from "unzipper";
 import * as url from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "./config";
-import { InvocationRateLimiter, InvocationRateLimiterResultKind, ProgressUpdate, showAndLogErrorMessage } from "./helpers";
+import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from "./helpers";
 import { logger } from "./logging";
-import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-version";
+import * as helpers from "./helpers";
+import { getCodeQlCliVersion } from "./cli-version";
 
 /**
  * distribution.ts
@@ -19,7 +21,7 @@ import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-versi
 
 /**
  * Default value for the owner name of the extension-managed distribution on GitHub.
- * 
+ *
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
@@ -27,98 +29,140 @@ const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";
 
 /**
  * Default value for the repository name of the extension-managed distribution on GitHub.
- * 
+ *
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
 const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-binaries";
 
 /**
- * Version constraint for the CLI.
- * 
+ * Range of versions of the CLI that are compatible with the extension.
+ *
  * This applies to both extension-managed and CLI distributions.
  */
-export const DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT: VersionConstraint = {
-  description: "2.0.*",
-  isVersionCompatible: (v: Version) => {
-    return v.majorVersion === 2 && v.minorVersion === 0
-  }
-}
+export const DEFAULT_DISTRIBUTION_VERSION_RANGE: semver.Range = new semver.Range("2.x");
 
 export interface DistributionProvider {
-  getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>,
-  onDidChangeDistribution?: Event<void>
+  getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
+  onDidChangeDistribution?: Event<void>;
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
+  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
     this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionConstraint);
+    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
     this._onDidChangeDistribution = config.onDidChangeDistributionConfiguration;
     this._updateCheckRateLimiter = new InvocationRateLimiter(
       extensionContext,
       "extensionSpecificDistributionUpdateCheck",
       () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
     );
-    this._versionConstraint = versionConstraint;
+    this._versionRange = versionRange;
   }
 
   /**
    * Look up a CodeQL launcher binary.
    */
   public async getDistribution(): Promise<FindDistributionResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath === undefined) {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    if (distribution === undefined) {
       return {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(codeQlPath, logger);
-    if (version !== undefined && !this._versionConstraint.isVersionCompatible(version)) {
+    const version = await getCodeQlCliVersion(distribution.codeQlPath, logger);
+    if (version === undefined) {
       return {
-        codeQlPath,
+        distribution,
+        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
+      };
+    }
+
+    /**
+     * Specifies whether prerelease versions of the CodeQL CLI should be accepted.
+     * 
+     * Suppose a user sets the includePrerelease config option, obtains a prerelease, then decides
+     * they no longer want a prerelease, so unsets the includePrerelease config option.
+     * Unsetting the includePrerelease config option should trigger an update check, and this
+     * update check should present them an update that returns them back to a non-prerelease
+     * version.
+     * 
+     * Therefore, we adopt the following:
+     * 
+     * - If the user is managing their own CLI, they can use a prerelease without specifying the
+     * includePrerelease option.
+     * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
+     * includePrerelease config option is set.
+     */
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+
+    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+      return {
+        distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
         version,
       };
     }
-    if (version === undefined) {
-      return {
-        codeQlPath,
-        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
-      }
-    }
     return {
-      codeQlPath,
+      distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
       version
     };
   }
 
+  public async hasDistribution(): Promise<boolean> {
+    const result = await this.getDistribution();
+    return result.kind !== FindDistributionResultKind.NoDistribution;
+  }
+
+  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    return distribution?.codeQlPath;
+  }
+
   /**
    * Returns the path to a possibly-compatible CodeQL launcher binary, or undefined if a binary not be found.
    */
-  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+  async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath !== undefined) {
+    if (this._config.customCodeQlPath) {
       if (!await fs.pathExists(this._config.customCodeQlPath)) {
         showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
           "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
           "that a CodeQL executable exists at the specified path or remove the setting.");
         return undefined;
       }
-      return this._config.customCodeQlPath;
+
+      // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
+      if (
+        deprecatedCodeQlLauncherName() &&
+        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        await this.hasNewLauncherName()
+      ) {
+        warnDeprecatedLauncher();
+      }
+      return {
+        codeQlPath: this._config.customCodeQlPath,
+        kind: DistributionKind.CustomPathConfig
+      };
     }
 
     const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
-      return extensionSpecificCodeQlPath;
+      return {
+        codeQlPath: extensionSpecificCodeQlPath,
+        kind: DistributionKind.ExtensionManaged
+      };
     }
 
     if (process.env.PATH) {
       for (const searchDirectory of process.env.PATH.split(path.delimiter)) {
-        const expectedLauncherPath = path.join(searchDirectory, codeQlLauncherName());
-        if (await fs.pathExists(expectedLauncherPath)) {
-          return expectedLauncherPath;
+        const expectedLauncherPath = await getExecutableFromDirectory(searchDirectory);
+        if (expectedLauncherPath) {
+          return {
+            codeQlPath: expectedLauncherPath,
+            kind: DistributionKind.PathEnvironmentVariable
+          };
         }
       }
       logger.log("INFO: Could not find CodeQL on path.");
@@ -130,14 +174,14 @@ export class DistributionManager implements DistributionProvider {
   /**
    * Check for updates to the extension-managed distribution.  If one has not already been installed,
    * this will return an update available result with the latest available release.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async checkForUpdatesToExtensionManagedDistribution(
     minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
+    const distribution = await this.getDistributionWithoutVersionCheck();
     const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath !== undefined && codeQlPath !== extensionManagedCodeQlPath) {
+    if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
       return createInvalidLocationResult();
     }
@@ -152,11 +196,11 @@ export class DistributionManager implements DistributionProvider {
 
   /**
    * Installs a release of the extension-managed distribution.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: (p: ProgressUpdate) => void): Promise<void> {
+    progressCallback?: helpers.ProgressCallback): Promise<void> {
     return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
   }
 
@@ -164,29 +208,43 @@ export class DistributionManager implements DistributionProvider {
     return this._onDidChangeDistribution;
   }
 
+  /**
+   * @return true if the non-deprecated launcher name exists on the file system
+   * in the same directory as the specified launcher only if using an external
+   * installation. False otherwise.
+   */
+  private async hasNewLauncherName(): Promise<boolean> {
+    if (!this._config.customCodeQlPath) {
+      // not managed externally
+      return false;
+    }
+    const dir = path.dirname(this._config.customCodeQlPath);
+    const newLaunderPath = path.join(dir, codeQlLauncherName());
+    return await fs.pathExists(newLaunderPath);
+  }
+
   private readonly _config: DistributionConfig;
   private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
   private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionConstraint: VersionConstraint;
+  private readonly _versionRange: semver.Range;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
+  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
     this._extensionContext = extensionContext;
     this._config = config;
-    this._versionConstraint = versionConstraint;
+    this._versionRange = versionRange;
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
     if (this.getInstalledRelease() !== undefined) {
       // An extension specific distribution has been installed.
-      const expectedLauncherPath = path.join(this.getDistributionRootPath(), codeQlLauncherName());
-      if (await fs.pathExists(expectedLauncherPath)) {
+      const expectedLauncherPath = await getExecutableFromDirectory(this.getDistributionRootPath(), true);
+      if (expectedLauncherPath) {
         return expectedLauncherPath;
       }
-      logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
-        "Will try PATH.");
+
       try {
         await this.removeDistribution();
       } catch (e) {
@@ -200,7 +258,7 @@ class ExtensionSpecificDistributionManager {
   /**
    * Check for updates to the extension-managed distribution.  If one has not already been installed,
    * this will return an update available result with the latest available release.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async checkForUpdatesToDistribution(): Promise<DistributionUpdateCheckResult> {
@@ -208,7 +266,11 @@ class ExtensionSpecificDistributionManager {
     const extensionSpecificRelease = this.getInstalledRelease();
     const latestRelease = await this.getLatestRelease();
 
-    if (extensionSpecificRelease !== undefined && codeQlPath !== undefined && latestRelease.id === extensionSpecificRelease.id) {
+    if (
+      extensionSpecificRelease !== undefined &&
+      codeQlPath !== undefined &&
+      latestRelease.id === extensionSpecificRelease.id
+    ) {
       return createAlreadyUpToDateResult();
     }
     return createUpdateAvailableResult(latestRelease);
@@ -216,18 +278,18 @@ class ExtensionSpecificDistributionManager {
 
   /**
    * Installs a release of the extension-managed distribution.
-   * 
+   *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async installDistributionRelease(release: Release,
-    progressCallback?: (p: ProgressUpdate) => void): Promise<void> {
+    progressCallback?: helpers.ProgressCallback): Promise<void> {
     await this.downloadDistribution(release, progressCallback);
     // Store the installed release within the global extension state.
     this.storeInstalledRelease(release);
   }
 
   private async downloadDistribution(release: Release,
-    progressCallback?: (p: ProgressUpdate) => void): Promise<void> {
+    progressCallback?: helpers.ProgressCallback): Promise<void> {
     try {
       await this.removeDistribution();
     } catch (e) {
@@ -235,7 +297,18 @@ class ExtensionSpecificDistributionManager {
         `but encountered an error: ${e}.`);
     }
 
-    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(release.assets[0]);
+    // Filter assets to the unique one that we require.
+    const requiredAssetName = this.getRequiredAssetName();
+    const assets = release.assets.filter(asset => asset.name === requiredAssetName);
+    if (assets.length === 0) {
+      throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
+    }
+    if (assets.length > 1) {
+      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+        assets.map(asset => asset.name).join(', '));
+    }
+
+    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(assets[0]);
     const tmpDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-codeql"));
 
     try {
@@ -247,8 +320,8 @@ class ExtensionSpecificDistributionManager {
 
       if (progressCallback && contentLength !== null) {
         const totalNumBytes = parseInt(contentLength, 10);
-        const bytesToDisplayMB = (numBytes: number) => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-        const updateProgress = () => {
+        const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+        const updateProgress = (): void => {
           progressCallback({
             step: numBytesDownloaded,
             maxStep: totalNumBytes,
@@ -282,7 +355,7 @@ class ExtensionSpecificDistributionManager {
 
   /**
    * Remove the extension-managed distribution.
-   * 
+   *
    * This should not be called for a distribution that is currently in use, as remove may fail.
    */
   private async removeDistribution(): Promise<void> {
@@ -292,12 +365,36 @@ class ExtensionSpecificDistributionManager {
     }
   }
 
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  private getRequiredAssetName(): string {
+    if (os.platform() === 'linux') return 'codeql-linux64.zip';
+    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
+    if (os.platform() === 'win32') return 'codeql-win64.zip';
+    return 'codeql.zip';
+  }
+
   private async getLatestRelease(): Promise<Release> {
-    const release = await this.createReleasesApiConsumer().getLatestRelease(this._versionConstraint, this._config.includePrerelease);
-    if (release.assets.length !== 1) {
-      throw new Error("Release had an unexpected number of assets");
-    }
-    return release;
+    const requiredAssetName = this.getRequiredAssetName();
+    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    return this.createReleasesApiConsumer().getLatestRelease(
+      this._versionRange,
+      this._config.includePrerelease,
+      release => {
+        const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
+        if (matchingAssets.length === 0) {
+          // For example, this could be a release with no platform-specific assets.
+          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          return false;
+        }
+        if (matchingAssets.length > 1) {
+          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          return false;
+        }
+        return true;
+      }
+    );
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
@@ -336,7 +433,7 @@ class ExtensionSpecificDistributionManager {
 
   private readonly _config: DistributionConfig;
   private readonly _extensionContext: ExtensionContext;
-  private readonly _versionConstraint: VersionConstraint;
+  private readonly _versionRange: semver.Range;
 
   private static readonly _currentDistributionFolderBaseName = "distribution";
   private static readonly _currentDistributionFolderIndexStateKey = "distributionFolderIndex";
@@ -357,7 +454,7 @@ export class ReleasesApiConsumer {
     this._repoName = repoName;
   }
 
-  public async getLatestRelease(versionConstraint: VersionConstraint, includePrerelease: boolean = false): Promise<Release> {
+  public async getLatestRelease(versionRange: semver.Range, includePrerelease = false, additionalCompatibilityCheck?: (release: GithubRelease) => boolean): Promise<Release> {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases`;
     const allReleases: GithubRelease[] = await (await this.makeApiCall(apiPath)).json();
     const compatibleReleases = allReleases.filter(release => {
@@ -365,20 +462,20 @@ export class ReleasesApiConsumer {
         return false;
       }
 
-      const version = tryParseVersionString(release.tag_name);
-      if (version === undefined || !versionConstraint.isVersionCompatible(version)) {
+      const version = semver.parse(release.tag_name);
+      if (version === null || !semver.satisfies(version, versionRange, { includePrerelease })) {
         return false;
       }
 
-      return true;
+      return !additionalCompatibilityCheck || additionalCompatibilityCheck(release);
     });
-    // tryParseVersionString must succeed due to the previous filtering step
+    // Tag names must all be parsable to semvers due to the previous filtering step.
     const latestRelease = compatibleReleases.sort((a, b) => {
-      const versionComparison = versionCompare(tryParseVersionString(b.tag_name)!, tryParseVersionString(a.tag_name)!);
-      if (versionComparison === 0) {
-        return b.created_at.localeCompare(a.created_at);
+      const versionComparison = semver.compare(semver.parse(b.tag_name)!, semver.parse(a.tag_name)!);
+      if (versionComparison !== 0) {
+        return versionComparison;
       }
-      return versionComparison;
+      return b.created_at.localeCompare(a.created_at, "en-US");
     })[0];
     if (latestRelease === undefined) {
       throw new Error("No compatible CodeQL CLI releases were found. " +
@@ -428,7 +525,7 @@ export class ReleasesApiConsumer {
   private async makeRawRequest(
     requestUrl: string,
     headers: { [key: string]: string },
-    redirectCount: number = 0): Promise<fetch.Response> {
+    redirectCount = 0): Promise<fetch.Response> {
     const response = await fetch.default(requestUrl, {
       headers,
       redirect: "manual"
@@ -447,7 +544,7 @@ export class ReleasesApiConsumer {
         // mechanism is provided.
         delete headers["authorization"];
       }
-      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1)
+      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
     }
 
     return response;
@@ -478,31 +575,12 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-/**
- * Comparison of semantic versions.
- * 
- * Returns a positive number if a is greater than b.
- * Returns 0 if a equals b.
- * Returns a negative number if a is less than b.
- */
-export function versionCompare(a: Version, b: Version): number {
-  if (a.majorVersion !== b.majorVersion) {
-    return a.majorVersion - b.majorVersion;
-  }
-  if (a.minorVersion !== b.minorVersion) {
-    return a.minorVersion - b.minorVersion;
-  }
-  if (a.patchVersion !== b.patchVersion) {
-    return a.patchVersion - b.patchVersion;
-  }
-  if (a.prereleaseVersion !== undefined && b.prereleaseVersion !== undefined) {
-    return a.prereleaseVersion.localeCompare(b.prereleaseVersion);
-  }
-  return 0;
+function codeQlLauncherName(): string {
+  return (os.platform() === "win32") ? "codeql.exe" : "codeql";
 }
 
-function codeQlLauncherName(): string {
-  return (os.platform() === "win32") ? "codeql.cmd" : "codeql";
+function deprecatedCodeQlLauncherName(): string | undefined {
+  return (os.platform() === "win32") ? "codeql.cmd" : undefined;
 }
 
 function isRedirectStatusCode(statusCode: number): boolean {
@@ -513,6 +591,17 @@ function isRedirectStatusCode(statusCode: number): boolean {
  * Types and helper functions relating to those types.
  */
 
+export enum DistributionKind {
+  CustomPathConfig,
+  ExtensionManaged,
+  PathEnvironmentVariable
+}
+
+export interface Distribution {
+  codeQlPath: string;
+  kind: DistributionKind;
+}
+
 export enum FindDistributionResultKind {
   CompatibleDistribution,
   UnknownCompatibilityDistribution,
@@ -520,24 +609,33 @@ export enum FindDistributionResultKind {
   NoDistribution
 }
 
-export type FindDistributionResult = CompatibleDistributionResult | UnknownCompatibilityDistributionResult |
-  IncompatibleDistributionResult | NoDistributionResult;
+export type FindDistributionResult =
+  | CompatibleDistributionResult
+  | UnknownCompatibilityDistributionResult
+  | IncompatibleDistributionResult
+  | NoDistributionResult;
 
-interface CompatibleDistributionResult {
-  codeQlPath: string;
-  kind: FindDistributionResultKind.CompatibleDistribution;
-  version: Version
+/**
+ * A result representing a distribution of the CodeQL CLI that may or may not be compatible with
+ * the extension.
+ */
+interface DistributionResult {
+  distribution: Distribution;
+  kind: FindDistributionResultKind;
 }
 
-interface UnknownCompatibilityDistributionResult {
-  codeQlPath: string;
+interface CompatibleDistributionResult extends DistributionResult {
+  kind: FindDistributionResultKind.CompatibleDistribution;
+  version: semver.SemVer;
+}
+
+interface UnknownCompatibilityDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.UnknownCompatibilityDistribution;
 }
 
-interface IncompatibleDistributionResult {
-  codeQlPath: string;
+interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: Version;
+  version: semver.SemVer;
 }
 
 interface NoDistributionResult {
@@ -551,11 +649,14 @@ export enum DistributionUpdateCheckResultKind {
   UpdateAvailable
 }
 
-type DistributionUpdateCheckResult = AlreadyCheckedRecentlyResult | AlreadyUpToDateResult | InvalidLocationResult |
-  UpdateAvailableResult;
+type DistributionUpdateCheckResult =
+  | AlreadyCheckedRecentlyResult
+  | AlreadyUpToDateResult
+  | InvalidLocationResult
+  | UpdateAvailableResult;
 
 export interface AlreadyCheckedRecentlyResult {
-  kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult
+  kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
 }
 
 export interface AlreadyUpToDateResult {
@@ -599,6 +700,31 @@ function createUpdateAvailableResult(updatedRelease: Release): UpdateAvailableRe
   };
 }
 
+// Exported for testing
+export async function getExecutableFromDirectory(directory: string, warnWhenNotFound = false): Promise<string | undefined> {
+  const expectedLauncherPath = path.join(directory, codeQlLauncherName());
+  const deprecatedLauncherName = deprecatedCodeQlLauncherName();
+  const alternateExpectedLauncherPath = deprecatedLauncherName ? path.join(directory, deprecatedLauncherName) : undefined;
+  if (await fs.pathExists(expectedLauncherPath)) {
+    return expectedLauncherPath;
+  } else if (alternateExpectedLauncherPath && (await fs.pathExists(alternateExpectedLauncherPath))) {
+    warnDeprecatedLauncher();
+    return alternateExpectedLauncherPath;
+  }
+  if (warnWhenNotFound) {
+    logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+      "Will try PATH.");
+  }
+  return undefined;
+}
+
+function warnDeprecatedLauncher() {
+  helpers.showAndLogWarningMessage(
+    `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
+    `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
+  );
+}
+
 /**
  * A release on GitHub.
  */
@@ -649,7 +775,7 @@ export interface GithubRelease {
   assets: GithubReleaseAsset[];
 
   /**
-   * The creation date of the release on GitHub.
+   * The creation date of the release on GitHub, in ISO 8601 format.
    */
   created_at: string;
 
@@ -694,11 +820,6 @@ export interface GithubReleaseAsset {
   size: number;
 }
 
-interface VersionConstraint {
-  description: string;
-  isVersionCompatible(version: Version): boolean;
-}
-
 export class GithubApiError extends Error {
   constructor(public status: number, public body: string) {
     super(`API call failed with status code ${status}, body: ${body}`);

extensions/ql-vscode/src/extension.ts

@@ -1,24 +1,35 @@
-import { commands, Disposable, ExtensionContext, extensions, ProgressLocation, ProgressOptions, window as Window, Uri } from 'vscode';
-import { ErrorCodes, LanguageClient, ResponseError } from 'vscode-languageclient';
+import { commands, Disposable, ExtensionContext, extensions, languages, ProgressLocation, ProgressOptions, Uri, window as Window } from 'vscode';
+import { LanguageClient } from 'vscode-languageclient';
+import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { DistributionConfigListener, QueryServerConfigListener, QueryHistoryConfigListener } from './config';
+import { CodeQLCliServer } from './cli';
+import { DistributionConfigListener, QueryHistoryConfigListener, QueryServerConfigListener } from './config';
+import * as languageSupport from './languageSupport';
 import { DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
+import { TemplateQueryDefinitionProvider, TemplateQueryReferenceProvider } from './definitions';
 import {
-  DistributionUpdateCheckResultKind, DistributionManager, FindDistributionResult, FindDistributionResultKind, GithubApiError,
-  DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT, GithubRateLimitedError
+  DEFAULT_DISTRIBUTION_VERSION_RANGE,
+  DistributionKind,
+  DistributionManager,
+  DistributionUpdateCheckResultKind,
+  FindDistributionResult,
+  FindDistributionResultKind,
+  GithubApiError,
+  GithubRateLimitedError
 } from './distribution';
 import * as helpers from './helpers';
+import { assertNever } from './helpers-pure';
 import { spawnIdeServer } from './ide-server';
 import { InterfaceManager, WebviewReveal } from './interface';
 import { ideServerLogger, logger, queryServerLogger } from './logging';
-import { compileAndRunQueryAgainstDatabase, tmpDirDisposal, UserCancellationException } from './run-queries';
-import { CompletedQuery } from './query-results';
 import { QueryHistoryManager } from './query-history';
+import { CompletedQuery } from './query-results';
 import * as qsClient from './queryserver-client';
-import { CodeQLCliServer } from './cli';
-import { assertNever } from './helpers-pure';
 import { displayQuickQuery } from './quick-query';
+import { compileAndRunQueryAgainstDatabase, tmpDirDisposal, UserCancellationException } from './run-queries';
+import { QLTestAdapterFactory } from './test-adapter';
+import { TestUIService } from './test-ui';
 
 /**
  * extension.ts
@@ -53,14 +64,15 @@ let isInstallingOrUpdatingDistribution = false;
  *
  * @param excludedCommands List of commands for which we should not register error stubs.
  */
-function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => void) {
+function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => void): void {
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
   const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
   const extension = extensions.getExtension(extensionId);
-  if (extension === undefined)
+  if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
+  }
 
   const stubbedCommands: string[]
     = extension.packageJSON.contributes.commands.map((entry: { command: string }) => entry.command);
@@ -73,13 +85,15 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 }
 
 export async function activate(ctx: ExtensionContext): Promise<void> {
-  // Initialise logging, and ensure all loggers are disposed upon exit.
-  ctx.subscriptions.push(logger);
   logger.log('Starting CodeQL extension');
 
+  initializeLogging(ctx);
+  languageSupport.install();
+
   const distributionConfigListener = new DistributionConfigListener();
   ctx.subscriptions.push(distributionConfigListener);
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT);
+  const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
+  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
 
   const shouldUpdateOnNextActivationKey = "shouldUpdateOnNextActivation";
 
@@ -90,6 +104,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
   interface DistributionUpdateConfig {
     isUserInitiated: boolean;
     shouldDisplayMessageWhenNoUpdates: boolean;
+    allowAutoUpdating: boolean;
   }
 
   async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
@@ -97,24 +112,28 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
       helpers.showAndLogInformationMessage : async (message: string) => logger.log(message);
     const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
+
+    // We do want to auto update if there is no distribution at all
+    const allowAutoUpdating = config.allowAutoUpdating || !await distributionManager.hasDistribution();
+
     switch (result.kind) {
       case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
         logger.log("Didn't perform CodeQL CLI update check since a check was already performed within the previous " +
           `${minSecondsSinceLastUpdateCheck} seconds.`);
         break;
       case DistributionUpdateCheckResultKind.AlreadyUpToDate:
-        await noUpdatesLoggingFunc("CodeQL CLI already up to date.");
+        await noUpdatesLoggingFunc('CodeQL CLI already up to date.');
         break;
       case DistributionUpdateCheckResultKind.InvalidLocation:
-        await noUpdatesLoggingFunc("CodeQL CLI is installed externally so could not be updated.");
+        await noUpdatesLoggingFunc('CodeQL CLI is installed externally so could not be updated.');
         break;
       case DistributionUpdateCheckResultKind.UpdateAvailable:
-        if (beganMainExtensionActivation) {
+        if (beganMainExtensionActivation || !allowAutoUpdating) {
           const updateAvailableMessage = `Version "${result.updatedRelease.name}" of the CodeQL CLI is now available. ` +
-            "The update will be installed after Visual Studio Code restarts. Restart now to upgrade?";
+            'Do you wish to upgrade?';
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, true);
-          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, "Restart and Upgrade")) {
-            await commands.executeCommand("workbench.action.reloadWindow");
+          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, 'Restart and Upgrade')) {
+            await commands.executeCommand('workbench.action.reloadWindow');
           }
         } else {
           const progressOptions: ProgressOptions = {
@@ -141,8 +160,12 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     isInstallingOrUpdatingDistribution = true;
     const codeQlInstalled = await distributionManager.getCodeQlPathWithoutVersionCheck() !== undefined;
     const willUpdateCodeQl = ctx.globalState.get(shouldUpdateOnNextActivationKey);
-    const messageText = willUpdateCodeQl ? "Updating CodeQL CLI" :
-      codeQlInstalled ? "Checking for updates to CodeQL CLI" : "Installing CodeQL CLI";
+    const messageText = willUpdateCodeQl
+      ? "Updating CodeQL CLI"
+      : codeQlInstalled
+        ? "Checking for updates to CodeQL CLI"
+        : "Installing CodeQL CLI";
+
     try {
       await installOrUpdateDistributionWithProgressTitle(messageText, config);
     } catch (e) {
@@ -169,11 +192,25 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.rawString})`);
+        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
-      case FindDistributionResultKind.IncompatibleDistribution:
-        helpers.showAndLogWarningMessage("The current version of the CodeQL CLI is incompatible with this extension.");
+      case FindDistributionResultKind.IncompatibleDistribution: {
+        const fixGuidanceMessage = (() => {
+          switch (result.distribution.kind) {
+            case DistributionKind.ExtensionManaged:
+              return "Please update the CodeQL CLI by running the \"CodeQL: Check for CLI Updates\" command.";
+            case DistributionKind.CustomPathConfig:
+              return `Please update the \"CodeQL CLI Executable Path\" setting to point to a CLI in the version range ${codeQlVersionRange}.`;
+            case DistributionKind.PathEnvironmentVariable:
+              return `Please update the CodeQL CLI on your PATH to a version compatible with ${codeQlVersionRange}, or ` +
+                `set the \"CodeQL CLI Executable Path\" setting to the path of a CLI version compatible with ${codeQlVersionRange}.`;
+          }
+        })();
+
+        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
+          "is incompatible with this extension. " + fixGuidanceMessage);
         break;
+      }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
         helpers.showAndLogWarningMessage("Compatibility with the configured CodeQL CLI could not be determined. " +
           "You may experience problems using the extension.");
@@ -198,11 +235,14 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = "Install CodeQL CLI";
-        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, installActionName);
+        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+          items: [installActionName]
+        });
         if (chosenAction === installActionName) {
           installOrUpdateThenTryActivate({
             isUserInitiated: true,
-            shouldDisplayMessageWhenNoUpdates: false
+            shouldDisplayMessageWhenNoUpdates: false,
+            allowAutoUpdating: true
           });
         }
       });
@@ -211,61 +251,71 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
   ctx.subscriptions.push(distributionConfigListener.onDidChangeDistributionConfiguration(() => installOrUpdateThenTryActivate({
     isUserInitiated: true,
-    shouldDisplayMessageWhenNoUpdates: false
+    shouldDisplayMessageWhenNoUpdates: false,
+    allowAutoUpdating: true
   })));
   ctx.subscriptions.push(commands.registerCommand(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
     isUserInitiated: true,
-    shouldDisplayMessageWhenNoUpdates: true
+    shouldDisplayMessageWhenNoUpdates: true,
+    allowAutoUpdating: true
   })));
 
   await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
-    shouldDisplayMessageWhenNoUpdates: false
+    shouldDisplayMessageWhenNoUpdates: false,
+
+    // only auto update on startup if the user has previously requested an update
+    // otherwise, ask user to accept the update
+    allowAutoUpdating: !!ctx.globalState.get(shouldUpdateOnNextActivationKey)
   });
 }
 
-async function activateWithInstalledDistribution(ctx: ExtensionContext, distributionManager: DistributionManager) {
+async function activateWithInstalledDistribution(ctx: ExtensionContext, distributionManager: DistributionManager): Promise<void> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
   errorStubs.forEach(stub => stub.dispose());
 
+  logger.log('Initializing configuration listener...');
   const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(distributionManager);
   ctx.subscriptions.push(qlConfigurationListener);
 
-  ctx.subscriptions.push(queryServerLogger);
-  ctx.subscriptions.push(ideServerLogger);
-
-
+  logger.log('Initializing CodeQL cli server...');
   const cliServer = new CodeQLCliServer(distributionManager, logger);
   ctx.subscriptions.push(cliServer);
 
+  logger.log('Initializing query server client.');
   const qs = new qsClient.QueryServerClient(qlConfigurationListener, cliServer, {
     logger: queryServerLogger,
   }, task => Window.withProgress({ title: 'CodeQL query server', location: ProgressLocation.Window }, task));
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
+  logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qlConfigurationListener, logger);
   ctx.subscriptions.push(dbm);
-  const databaseUI = new DatabaseUI(ctx, cliServer, dbm, qs);
+  logger.log('Initializing database panel.');
+  const databaseUI = new DatabaseUI(ctx, cliServer, dbm, qs, getContextStoragePath(ctx));
   ctx.subscriptions.push(databaseUI);
 
+  logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
   const qhm = new QueryHistoryManager(
     ctx,
     queryHistoryConfigurationListener,
     async item => showResultsForCompletedQuery(item, WebviewReveal.Forced)
   );
+  logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
+  logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
   async function showResultsForCompletedQuery(query: CompletedQuery, forceReveal: WebviewReveal): Promise<void> {
     await intm.showResults(query, forceReveal, false);
   }
 
-  async function compileAndRunQuery(quickEval: boolean, selectedQuery: Uri | undefined) {
+  async function compileAndRunQuery(quickEval: boolean, selectedQuery: Uri | undefined): Promise<void> {
     if (qs !== undefined) {
       try {
         const dbItem = await databaseUI.getDatabaseItem();
@@ -275,25 +325,22 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
         const info = await compileAndRunQueryAgainstDatabase(cliServer, qs, dbItem, quickEval, selectedQuery);
         const item = qhm.addQuery(info);
         await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      }
-      catch (e) {
+      } catch (e) {
         if (e instanceof UserCancellationException) {
-          logger.log(e.message);
-        }
-        else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-          logger.log(e.message);
-        }
-        else if (e instanceof Error)
+          helpers.showAndLogWarningMessage(e.message);
+        } else if (e instanceof Error) {
           helpers.showAndLogErrorMessage(e.message);
-        else
+        } else {
           throw e;
+        }
       }
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
-  let client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
+  logger.log('Initializing CodeQL language server.');
+  const client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
     documentSelector: [
       { language: 'ql', scheme: 'file' },
       { language: 'yaml', scheme: 'file', pattern: '**/qlpack.yml' }
@@ -305,11 +352,59 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     outputChannel: ideServerLogger.outputChannel
   }, true);
 
+  logger.log('Initializing QLTest interface.');
+  const testExplorerExtension = extensions.getExtension<TestHub>(testExplorerExtensionId);
+  if (testExplorerExtension) {
+    const testHub = testExplorerExtension.exports;
+    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
+    ctx.subscriptions.push(testAdapterFactory);
+
+    const testUIService = new TestUIService(testHub);
+    ctx.subscriptions.push(testUIService);
+  }
+
+  logger.log('Registering top-level command palette commands.');
   ctx.subscriptions.push(commands.registerCommand('codeQL.runQuery', async (uri: Uri | undefined) => await compileAndRunQuery(false, uri)));
   ctx.subscriptions.push(commands.registerCommand('codeQL.quickEval', async (uri: Uri | undefined) => await compileAndRunQuery(true, uri)));
   ctx.subscriptions.push(commands.registerCommand('codeQL.quickQuery', async () => displayQuickQuery(ctx, cliServer, databaseUI)));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.restartQueryServer', async () => {
+    await qs.restartQueryServer();
+    helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', { outputLogger: queryServerLogger });
+  }));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseFolder', () => databaseUI.handleChooseDatabaseFolder()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseArchive', () => databaseUI.handleChooseDatabaseArchive()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseLgtm', () => databaseUI.handleChooseDatabaseLgtm()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseInternet', () => databaseUI.handleChooseDatabaseInternet()));
 
+  logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
+
+  // Jump-to-definition and find-references
+  logger.log('Registering jump-to-definition handlers.');
+  languages.registerDefinitionProvider(
+    { scheme: archiveFilesystemProvider.zipArchiveScheme },
+    new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
+  );
+  languages.registerReferenceProvider(
+    { scheme: archiveFilesystemProvider.zipArchiveScheme },
+    new TemplateQueryReferenceProvider(cliServer, qs, dbm)
+  );
+
+  logger.log('Successfully finished extension initialization.');
+}
+
+function getContextStoragePath(ctx: ExtensionContext) {
+  return ctx.storagePath || ctx.globalStoragePath;
+}
+
+function initializeLogging(ctx: ExtensionContext): void {
+  const storagePath = getContextStoragePath(ctx);
+  logger.init(storagePath);
+  queryServerLogger.init(storagePath);
+  ideServerLogger.init(storagePath);
+  ctx.subscriptions.push(logger);
+  ctx.subscriptions.push(queryServerLogger);
+  ctx.subscriptions.push(ideServerLogger);
 }
 
 const checkForUpdatesCommand = 'codeQL.checkForUpdatesToCLI';

extensions/ql-vscode/src/helpers.ts

@@ -1,5 +1,9 @@
+import * as fs from 'fs-extra';
+import * as glob from 'glob-promise';
+import * as yaml from 'js-yaml';
 import * as path from 'path';
 import { CancellationToken, ExtensionContext, ProgressOptions, window as Window, workspace } from 'vscode';
+import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
 import { QueryInfo } from './run-queries';
 
@@ -18,6 +22,8 @@ export interface ProgressUpdate {
   message: string;
 }
 
+export type ProgressCallback = (p: ProgressUpdate) => void;
+
 /**
  * This mediates between the kind of progress callbacks we want to
  * write (where we *set* current progress position and give
@@ -47,37 +53,59 @@ export function withProgress<R>(
  * Show an error message and log it to the console
  *
  * @param message The message to show.
- * @param items A set of items that will be rendered as actions in the message.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogErrorMessage(message: string, ...items: string[]): Thenable<string | undefined> {
-  logger.log(message);
-  return Window.showErrorMessage(message, ...items);
+export async function showAndLogErrorMessage(message: string, {
+  outputLogger = logger,
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
 }
 /**
  * Show a warning message and log it to the console
  *
  * @param message The message to show.
- * @param items A set of items that will be rendered as actions in the message.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogWarningMessage(message: string, ...items: string[]): Thenable<string | undefined> {
-  logger.log(message);
-  return Window.showWarningMessage(message, ...items);
+export async function showAndLogWarningMessage(message: string, {
+  outputLogger = logger,
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showWarningMessage);
 }
 /**
  * Show an information message and log it to the console
  *
  * @param message The message to show.
- * @param items A set of items that will be rendered as actions in the message.
+ * @param options.outputLogger The output logger that will receive the message
+ * @param options.items A set of items that will be rendered as actions in the message.
  *
- * @return A thenable that resolves to the selected item or undefined when being dismissed.
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
-export function showAndLogInformationMessage(message: string, ...items: string[]): Thenable<string | undefined> {
-  logger.log(message);
-  return Window.showInformationMessage(message, ...items);
+export async function showAndLogInformationMessage(message: string, {
+  outputLogger = logger,
+  items = [] as string[]
+} = {}): Promise<string | undefined> {
+  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage);
+}
+
+type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
+
+async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
+  fn: ShowMessageFn): Promise<string | undefined> {
+  const label = 'Show Log';
+  outputLogger.log(message);
+  const result = await fn(message, label, ...items);
+  if (result === label) {
+    outputLogger.show();
+  }
+  return result;
 }
 
 /**
@@ -88,7 +116,7 @@ export function showAndLogInformationMessage(message: string, ...items: string[]
  */
 export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
   const yesItem = { title: 'Yes', isCloseAffordance: false };
-  const noItem = { title: 'No', isCloseAffordance: true }
+  const noItem = { title: 'No', isCloseAffordance: true };
   const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
   return chosenItem === yesItem;
 }
@@ -109,10 +137,10 @@ export async function showInformationMessageWithAction(message: string, actionMe
 /** Gets all active workspace folders that are on the filesystem. */
 export function getOnDiskWorkspaceFolders() {
   const workspaceFolders = workspace.workspaceFolders || [];
-  let diskWorkspaceFolders: string[] = [];
+  const diskWorkspaceFolders: string[] = [];
   for (const workspaceFolder of workspaceFolders) {
     if (workspaceFolder.uri.scheme === "file")
-      diskWorkspaceFolders.push(workspaceFolder.uri.fsPath)
+      diskWorkspaceFolders.push(workspaceFolder.uri.fsPath);
   }
   return diskWorkspaceFolders;
 }
@@ -157,8 +185,12 @@ export class InvocationRateLimiter<T> {
   public async invokeFunctionIfIntervalElapsed(minSecondsSinceLastInvocation: number): Promise<InvocationRateLimiterResult<T>> {
     const updateCheckStartDate = this._createDate();
     const lastInvocationDate = this.getLastInvocationDate();
-    if (minSecondsSinceLastInvocation && lastInvocationDate && lastInvocationDate <= updateCheckStartDate &&
-      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 > updateCheckStartDate.getTime()) {
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 > updateCheckStartDate.getTime()
+    ) {
       return createRateLimitedResult();
     }
     const result = await this._func();
@@ -193,15 +225,15 @@ export enum InvocationRateLimiterResultKind {
  * The function was invoked and returned the value `result`.
  */
 interface InvokedResult<T> {
-  kind: InvocationRateLimiterResultKind.Invoked,
-  result: T
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
 }
 
 /**
  * The function was not invoked as the minimum interval since the last invocation had not elapsed.
  */
 interface RateLimitedResult {
-  kind: InvocationRateLimiterResultKind.RateLimited
+  kind: InvocationRateLimiterResultKind.RateLimited;
 }
 
 type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
@@ -218,3 +250,110 @@ function createRateLimitedResult(): RateLimitedResult {
     kind: InvocationRateLimiterResultKind.RateLimited
   };
 }
+
+
+export type DatasetFolderInfo = {
+  dbscheme: string;
+  qlpack: string;
+}
+
+export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
+  const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
+  const packs: { packDir: string | undefined; packName: string }[] =
+    Object.entries(qlpacks).map(([packName, dirs]) => {
+      if (dirs.length < 1) {
+        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
+        return { packName, packDir: undefined };
+      }
+      if (dirs.length > 1) {
+        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
+      }
+      return {
+        packName,
+        packDir: dirs[0]
+      };
+    });
+  for (const { packDir, packName } of packs) {
+    if (packDir !== undefined) {
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8'));
+      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
+        return packName;
+      }
+    }
+  }
+  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+}
+
+export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise<DatasetFolderInfo> {
+  const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'));
+
+  if (dbschemes.length < 1) {
+    throw new Error(`Can't find dbscheme for current database in ${datasetFolder}`);
+  }
+
+  dbschemes.sort();
+  const dbscheme = dbschemes[0];
+  if (dbschemes.length > 1) {
+    Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
+  }
+
+  const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
+  return { dbscheme, qlpack };
+}
+
+/**
+ * A cached mapping from strings to value of type U.
+ */
+export class CachedOperation<U> {
+  private readonly operation: (t: string) => Promise<U>;
+  private readonly cached: Map<string, U>;
+  private readonly lru: string[];
+  private readonly inProgressCallbacks: Map<string, [(u: U) => void, (reason?: any) => void][]>;
+
+  constructor(operation: (t: string) => Promise<U>, private cacheSize = 100) {
+    this.operation = operation;
+    this.lru = [];
+    this.inProgressCallbacks = new Map<string, [(u: U) => void, (reason?: any) => void][]>();
+    this.cached = new Map<string, U>();
+  }
+
+  async get(t: string): Promise<U> {
+    // Try and retrieve from the cache
+    const fromCache = this.cached.get(t);
+    if (fromCache !== undefined) {
+      // Move to end of lru list
+      this.lru.push(this.lru.splice(this.lru.findIndex(v => v === t), 1)[0]);
+      return fromCache;
+    }
+    // Otherwise check if in progress
+    const inProgressCallback = this.inProgressCallbacks.get(t);
+    if (inProgressCallback !== undefined) {
+      // If so wait for it to resolve
+      return await new Promise((resolve, reject) => {
+        inProgressCallback.push([resolve, reject]);
+      });
+    }
+
+    // Otherwise compute the new value, but leave a callback to allow sharing work
+    const callbacks: [(u: U) => void, (reason?: any) => void][] = [];
+    this.inProgressCallbacks.set(t, callbacks);
+    try {
+      const result = await this.operation(t);
+      callbacks.forEach(f => f[0](result));
+      this.inProgressCallbacks.delete(t);
+      if (this.lru.length > this.cacheSize) {
+        const toRemove = this.lru.shift()!;
+        this.cached.delete(toRemove);
+      }
+      this.lru.push(t);
+      this.cached.set(t, result);
+      return result;
+    } catch (e) {
+      // Rethrow error on all callbacks
+      callbacks.forEach(f => f[1](e));
+      throw e;
+    } finally {
+      this.inProgressCallbacks.delete(t);
+    }
+  }
+}

extensions/ql-vscode/src/ide-server.ts

@@ -17,8 +17,8 @@ export async function spawnIdeServer(config: QueryServerConfig): Promise<StreamI
       ['execute', 'language-server'],
       ['--check-errors', 'ON_CHANGE'],
       ideServerLogger,
-      data => ideServerLogger.logWithoutTrailingNewline(data.toString()),
-      data => ideServerLogger.logWithoutTrailingNewline(data.toString()),
+      data => ideServerLogger.log(data.toString(), { trailingNewline: false }),
+      data => ideServerLogger.log(data.toString(), { trailingNewline: false }),
       progressReporter
     );
     return { writer: child.stdin!, reader: child.stdout! };

extensions/ql-vscode/src/interface-types.ts

@@ -1,5 +1,6 @@
 import * as sarif from 'sarif';
 import { ResolvableLocationValue } from 'semmle-bqrs';
+import { ParsedResultSets } from './adapt';
 
 /**
  * Only ever show this many results per run in interpreted results.
@@ -11,6 +12,11 @@ export const INTERPRETED_RESULTS_PER_RUN_LIMIT = 100;
  */
 export const RAW_RESULTS_LIMIT = 10000;
 
+/**
+ * Show this many rows in a raw result table at a time.
+ */
+export const RAW_RESULTS_PAGE_SIZE = 100;
+
 export interface DatabaseInfo {
   name: string;
   databaseUri: string;
@@ -18,10 +24,10 @@ export interface DatabaseInfo {
 
 /** Arbitrary query metadata */
 export interface QueryMetadata {
-  name?: string,
-  description?: string,
-  id?: string,
-  kind?: string
+  name?: string;
+  description?: string;
+  id?: string;
+  kind?: string;
 }
 
 export interface PreviousExecution {
@@ -70,18 +76,25 @@ export interface SetStateMsg {
   sortedResultsMap: SortedResultsMap;
   interpretation: undefined | Interpretation;
   database: DatabaseInfo;
-  metadata?: QueryMetadata
+  metadata?: QueryMetadata;
   /**
    * Whether to keep displaying the old results while rendering the new results.
    *
    * This is useful to prevent properties like scroll state being lost when rendering the sorted results after sorting a column.
    */
   shouldKeepOldResultsWhileRendering: boolean;
-};
+
+  /**
+   * An experimental way of providing results from the extension.
+   * Should be in the WebviewParsedResultSets branch of the type
+   * unless config.EXPERIMENTAL_BQRS_SETTING is set to true.
+   */
+  parsedResultSets: ParsedResultSets;
+}
 
 /** Advance to the next or previous path no in the path viewer */
 export interface NavigatePathMsg {
-  t: 'navigatePath',
+  t: 'navigatePath';
 
   /** 1 for next, -1 for previous */
   direction: number;
@@ -94,26 +107,33 @@ export type FromResultsViewMsg =
   | ToggleDiagnostics
   | ChangeRawResultsSortMsg
   | ChangeInterpretedResultsSortMsg
-  | ResultViewLoaded;
+  | ResultViewLoaded
+  | ChangePage;
 
 interface ViewSourceFileMsg {
   t: 'viewSourceFile';
   loc: ResolvableLocationValue;
   databaseUri: string;
-};
+}
 
 interface ToggleDiagnostics {
   t: 'toggleDiagnostics';
   databaseUri: string;
-  metadata?: QueryMetadata
+  metadata?: QueryMetadata;
   origResultsPaths: ResultsPaths;
   visible: boolean;
   kind?: string;
-};
+}
 
 interface ResultViewLoaded {
   t: 'resultViewLoaded';
-};
+}
+
+interface ChangePage {
+  t: 'changePage';
+  pageNumber: number; // 0-indexed, displayed to the user as 1-indexed
+  selectedTable: string;
+}
 
 export enum SortDirection {
   asc, desc

extensions/ql-vscode/src/interface-utils.ts

@@ -0,0 +1,22 @@
+import { RawResultSet } from "./adapt";
+import { ResultSetSchema } from "semmle-bqrs";
+import { Interpretation } from "./interface-types";
+
+export const SELECT_TABLE_NAME = '#select';
+export const ALERTS_TABLE_NAME = 'alerts';
+
+export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
+export type PathTableResultSet = { t: 'SarifResultSet'; readonly schema: ResultSetSchema; name: string } & Interpretation;
+
+export type ResultSet =
+  | RawTableResultSet
+  | PathTableResultSet;
+
+export function getDefaultResultSet(resultSets: readonly ResultSet[]): string {
+  return getDefaultResultSetName(resultSets.map(resultSet => resultSet.schema.name));
+}
+
+export function getDefaultResultSetName(resultSetNames: readonly string[]): string {
+  // Choose first available result set from the array
+  return [ALERTS_TABLE_NAME, SELECT_TABLE_NAME, resultSetNames[0]].filter(resultSetName => resultSetNames.includes(resultSetName))[0];
+}

extensions/ql-vscode/src/interface.ts

@@ -10,12 +10,15 @@ import { CodeQLCliServer } from './cli';
 import { DatabaseItem, DatabaseManager } from './databases';
 import { showAndLogErrorMessage } from './helpers';
 import { assertNever } from './helpers-pure';
-import { FromResultsViewMsg, Interpretation, INTERPRETED_RESULTS_PER_RUN_LIMIT, IntoResultsViewMsg, QueryMetadata, ResultsPaths, SortedResultSetInfo, SortedResultsMap, InterpretedResultsSortState, SortDirection } from './interface-types';
+import { FromResultsViewMsg, Interpretation, INTERPRETED_RESULTS_PER_RUN_LIMIT, IntoResultsViewMsg, QueryMetadata, ResultsPaths, SortedResultSetInfo, SortedResultsMap, InterpretedResultsSortState, SortDirection, RAW_RESULTS_PAGE_SIZE } from './interface-types';
 import { Logger } from './logging';
 import * as messages from './messages';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './sarif-utils';
+import { adaptSchema, adaptBqrs, RawResultSet, ParsedResultSets } from './adapt';
+import { EXPERIMENTAL_BQRS_SETTING } from './config';
+import { getDefaultResultSetName } from './interface-utils';
 
 /**
  * interface.ts
@@ -42,7 +45,11 @@ export enum WebviewReveal {
  * Returns HTML to populate the given webview.
  * Uses a content security policy that only loads the given script.
  */
-function getHtmlForWebview(webview: vscode.Webview, scriptUriOnDisk: vscode.Uri, stylesheetUriOnDisk: vscode.Uri) {
+function getHtmlForWebview(
+  webview: vscode.Webview,
+  scriptUriOnDisk: vscode.Uri,
+  stylesheetUriOnDisk: vscode.Uri
+): void {
   // Convert the on-disk URIs into webview URIs.
   const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
   const stylesheetWebviewUri = webview.asWebviewUri(stylesheetUriOnDisk);
@@ -109,25 +116,50 @@ function sortInterpretedResults(results: Sarif.Result[], sortState: InterpretedR
   }
 }
 
+function numPagesOfResultSet(resultSet: RawResultSet): number {
+  return Math.ceil(resultSet.schema.tupleCount / RAW_RESULTS_PAGE_SIZE);
+}
+
 export class InterfaceManager extends DisposableObject {
   private _displayedQuery?: CompletedQuery;
+  private _interpretation?: Interpretation;
   private _panel: vscode.WebviewPanel | undefined;
   private _panelLoaded = false;
   private _panelLoadedCallBacks: (() => void)[] = [];
 
-  private readonly _diagnosticCollection = languages.createDiagnosticCollection(`codeql-query-results`);
-
-  constructor(public ctx: vscode.ExtensionContext, private databaseManager: DatabaseManager,
-    public cliServer: CodeQLCliServer, public logger: Logger) {
+  private readonly _diagnosticCollection = languages.createDiagnosticCollection(
+    `codeql-query-results`
+  );
 
+  constructor(
+    public ctx: vscode.ExtensionContext,
+    private databaseManager: DatabaseManager,
+    public cliServer: CodeQLCliServer,
+    public logger: Logger
+  ) {
     super();
     this.push(this._diagnosticCollection);
-    this.push(vscode.window.onDidChangeTextEditorSelection(this.handleSelectionChange.bind(this)));
-    this.push(vscode.commands.registerCommand('codeQLQueryResults.nextPathStep', this.navigatePathStep.bind(this, 1)));
-    this.push(vscode.commands.registerCommand('codeQLQueryResults.previousPathStep', this.navigatePathStep.bind(this, -1)));
+    this.push(
+      vscode.window.onDidChangeTextEditorSelection(
+        this.handleSelectionChange.bind(this)
+      )
+    );
+    logger.log('Registering path-step navigation commands.');
+    this.push(
+      vscode.commands.registerCommand(
+        "codeQLQueryResults.nextPathStep",
+        this.navigatePathStep.bind(this, 1)
+      )
+    );
+    this.push(
+      vscode.commands.registerCommand(
+        "codeQLQueryResults.previousPathStep",
+        this.navigatePathStep.bind(this, -1)
+      )
+    );
   }
 
-  navigatePathStep(direction: number) {
+  navigatePathStep(direction: number): void {
     this.postMessage({ t: "navigatePath", direction });
   }
 
@@ -136,9 +168,9 @@ export class InterfaceManager extends DisposableObject {
   getPanel(): vscode.WebviewPanel {
     if (this._panel == undefined) {
       const { ctx } = this;
-      const panel = this._panel = Window.createWebviewPanel(
-        'resultsView', // internal name
-        'CodeQL Query Results', // user-visible name
+      const panel = (this._panel = Window.createWebviewPanel(
+        "resultsView", // internal name
+        "CodeQL Query Results", // user-visible name
         { viewColumn: vscode.ViewColumn.Beside, preserveFocus: true },
         {
           enableScripts: true,
@@ -146,61 +178,96 @@ export class InterfaceManager extends DisposableObject {
           retainContextWhenHidden: true,
           localResourceRoots: [
             vscode.Uri.file(tmpDir.name),
-            vscode.Uri.file(path.join(this.ctx.extensionPath, 'out'))
+            vscode.Uri.file(path.join(this.ctx.extensionPath, "out"))
           ]
         }
+      ));
+      this._panel.onDidDispose(
+        () => {
+          this._panel = undefined;
+        },
+        null,
+        ctx.subscriptions
+      );
+      const scriptPathOnDisk = vscode.Uri.file(
+        ctx.asAbsolutePath("out/resultsView.js")
+      );
+      const stylesheetPathOnDisk = vscode.Uri.file(
+        ctx.asAbsolutePath("out/resultsView.css")
+      );
+      getHtmlForWebview(
+        panel.webview,
+        scriptPathOnDisk,
+        stylesheetPathOnDisk
+      );
+      panel.webview.onDidReceiveMessage(
+        async e => this.handleMsgFromView(e),
+        undefined,
+        ctx.subscriptions
       );
-      this._panel.onDidDispose(() => { this._panel = undefined; }, null, ctx.subscriptions);
-      const scriptPathOnDisk = vscode.Uri
-        .file(ctx.asAbsolutePath('out/resultsView.js'));
-      const stylesheetPathOnDisk = vscode.Uri
-        .file(ctx.asAbsolutePath('out/resultsView.css'));
-      getHtmlForWebview(panel.webview, scriptPathOnDisk, stylesheetPathOnDisk);
-      panel.webview.onDidReceiveMessage(async (e) => this.handleMsgFromView(e), undefined, ctx.subscriptions);
     }
     return this._panel;
   }
 
-  private async changeSortState(update: (query: CompletedQuery) => Promise<void>): Promise<void> {
+  private async changeSortState(
+    update: (query: CompletedQuery) => Promise<void>
+  ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage("Failed to sort results since evaluation info was unknown.");
+      showAndLogErrorMessage(
+        "Failed to sort results since evaluation info was unknown."
+      );
       return;
     }
     // Notify the webview that it should expect new results.
-    await this.postMessage({ t: 'resultsUpdating' });
+    await this.postMessage({ t: "resultsUpdating" });
     await update(this._displayedQuery);
-    await this.showResults(this._displayedQuery, WebviewReveal.NotForced, true);
+    await this.showResults(
+      this._displayedQuery,
+      WebviewReveal.NotForced,
+      true
+    );
   }
 
-  private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
+  private async handleMsgFromView(
+    msg: FromResultsViewMsg
+  ): Promise<void> {
     switch (msg.t) {
-      case 'viewSourceFile': {
-        const databaseItem = this.databaseManager.findDatabaseItem(Uri.parse(msg.databaseUri));
+      case "viewSourceFile": {
+        const databaseItem = this.databaseManager.findDatabaseItem(
+          Uri.parse(msg.databaseUri)
+        );
         if (databaseItem !== undefined) {
           try {
             await showLocation(msg.loc, databaseItem);
-          }
-          catch (e) {
+          } catch (e) {
             if (e instanceof Error) {
               if (e.message.match(/File not found/)) {
-                vscode.window.showErrorMessage(`Original file of this result is not in the database's source archive.`);
+                vscode.window.showErrorMessage(
+                  `Original file of this result is not in the database's source archive.`
+                );
+              } else {
+                this.logger.log(
+                  `Unable to handleMsgFromView: ${e.message}`
+                );
               }
-              else {
-                this.logger.log(`Unable to handleMsgFromView: ${e.message}`);
-              }
-            }
-            else {
+            } else {
               this.logger.log(`Unable to handleMsgFromView: ${e}`);
             }
           }
         }
         break;
       }
-      case 'toggleDiagnostics': {
+      case "toggleDiagnostics": {
         if (msg.visible) {
-          const databaseItem = this.databaseManager.findDatabaseItem(Uri.parse(msg.databaseUri));
+          const databaseItem = this.databaseManager.findDatabaseItem(
+            Uri.parse(msg.databaseUri)
+          );
           if (databaseItem !== undefined) {
-            await this.showResultsAsDiagnostics(msg.origResultsPaths, msg.metadata, databaseItem);
+            await this.showResultsAsDiagnostics(
+              msg.origResultsPaths,
+              msg.metadata,
+              databaseItem
+            );
           }
         } else {
           // TODO: Only clear diagnostics on the same database.
@@ -213,11 +280,22 @@ export class InterfaceManager extends DisposableObject {
         this._panelLoadedCallBacks.forEach(cb => cb());
         this._panelLoadedCallBacks = [];
         break;
-      case 'changeSort':
-        await this.changeSortState((query) => query.updateSortState(this.cliServer, msg.resultSetName, msg.sortState));
+      case "changeSort":
+        await this.changeSortState(query =>
+          query.updateSortState(
+            this.cliServer,
+            msg.resultSetName,
+            msg.sortState
+          )
+        );
         break;
-      case 'changeInterpretedSort':
-        await this.changeSortState((query) => query.updateInterpretedSortState(this.cliServer, msg.sortState));
+      case "changeInterpretedSort":
+        await this.changeSortState(query =>
+          query.updateInterpretedSortState(this.cliServer, msg.sortState)
+        );
+        break;
+      case "changePage":
+        await this.showPageOfResults(msg.selectedTable, msg.pageNumber);
         break;
       default:
         assertNever(msg);
@@ -229,51 +307,62 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve, _reject) => {
+    return new Promise(resolve => {
       if (this._panelLoaded) {
         resolve();
       } else {
-        this._panelLoadedCallBacks.push(resolve)
+        this._panelLoadedCallBacks.push(resolve);
       }
-    })
+    });
   }
 
   /**
-   * Show query results in webview panel.
-   * @param results Evaluation info for the executed query.
-   * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
-   * @param forceReveal Force the webview panel to be visible and
-   * Appropriate when the user has just performed an explicit
-   * UI interaction requesting results, e.g. clicking on a query
-   * history entry.
-   */
-  public async showResults(results: CompletedQuery, forceReveal: WebviewReveal, shouldKeepOldResultsWhileRendering: boolean = false): Promise<void> {
+  * Show query results in webview panel.
+  * @param results Evaluation info for the executed query.
+  * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
+  * @param forceReveal Force the webview panel to be visible and
+  * Appropriate when the user has just performed an explicit
+  * UI interaction requesting results, e.g. clicking on a query
+  * history entry.
+  */
+  public async showResults(
+    results: CompletedQuery,
+    forceReveal: WebviewReveal,
+    shouldKeepOldResultsWhileRendering = false
+  ): Promise<void> {
     if (results.result.resultType !== messages.QueryResultType.SUCCESS) {
       return;
     }
 
-    const interpretation = await this.interpretResultsInfo(results.query, results.interpretedResultsSortState);
+    const interpretation = await this.interpretResultsInfo(
+      results.query,
+      results.interpretedResultsSortState
+    );
 
     const sortedResultsMap: SortedResultsMap = {};
-    results.sortedResultsInfo.forEach((v, k) =>
-      sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v));
+    results.sortedResultsInfo.forEach(
+      (v, k) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(
+          v
+        ))
+    );
 
     this._displayedQuery = results;
+    this._interpretation = interpretation;
 
     const panel = this.getPanel();
     await this.waitForPanelLoaded();
     if (forceReveal === WebviewReveal.Forced) {
       panel.reveal(undefined, true);
-    }
-    else if (!panel.visible) {
+    } else if (!panel.visible) {
       // The results panel exists, (`.getPanel()` guarantees it) but
       // is not visible; it's in a not-currently-viewed tab. Show a
       // more asynchronous message to not so abruptly interrupt
       // user's workflow by immediately revealing the panel.
-      const showButton = 'View Results';
+      const showButton = "View Results";
       const queryName = results.queryName;
       const resultPromise = vscode.window.showInformationMessage(
-        `Finished running query ${(queryName.length > 0) ? ` “${queryName}”` : ''}.`,
+        `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ""}.`,
         showButton
       );
       // Address this click asynchronously so we still update the
@@ -285,11 +374,46 @@ export class InterfaceManager extends DisposableObject {
       });
     }
 
+    const getParsedResultSets = async (): Promise<ParsedResultSets> => {
+      if (EXPERIMENTAL_BQRS_SETTING.getValue()) {
+        const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath, RAW_RESULTS_PAGE_SIZE);
+
+        const resultSetNames = schemas["result-sets"].map(resultSet => resultSet.name);
+
+        // This may not wind up being the page we actually show, if there are interpreted results,
+        // but speculatively send it anyway.
+        const selectedTable = getDefaultResultSetName(resultSetNames);
+        const schema = schemas["result-sets"].find(resultSet => resultSet.name == selectedTable)!;
+        if (schema === undefined) {
+          return { t: 'WebviewParsed' };
+        }
+
+        const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name, RAW_RESULTS_PAGE_SIZE, schema.pagination?.offsets[0]);
+        const adaptedSchema = adaptSchema(schema);
+        const resultSet = adaptBqrs(adaptedSchema, chunk);
+
+        return {
+          t: 'ExtensionParsed',
+          pageNumber: 0,
+          numPages: numPagesOfResultSet(resultSet),
+          resultSet,
+          selectedTable: undefined,
+          resultSetNames
+        };
+      }
+      else {
+        return { t: 'WebviewParsed' };
+      }
+    };
+
     await this.postMessage({
-      t: 'setState',
+      t: "setState",
       interpretation,
       origResultsPaths: results.query.resultsPaths,
-      resultsPath: this.convertPathToWebviewUri(results.query.resultsPaths.resultsPath),
+      resultsPath: this.convertPathToWebviewUri(
+        results.query.resultsPaths.resultsPath
+      ),
+      parsedResultSets: await getParsedResultSets(),
       sortedResultsMap,
       database: results.database,
       shouldKeepOldResultsWhileRendering,
@@ -297,8 +421,72 @@ export class InterfaceManager extends DisposableObject {
     });
   }
 
-  private async getTruncatedResults(metadata: QueryMetadata | undefined, resultsPaths: ResultsPaths, sourceInfo: cli.SourceInfo | undefined, sourceLocationPrefix: string, sortState: InterpretedResultsSortState | undefined): Promise<Interpretation> {
-    const sarif = await interpretResults(this.cliServer, metadata, resultsPaths.resultsPath, sourceInfo);
+  /**
+   * Show a page of raw results from the chosen table.
+   */
+  public async showPageOfResults(selectedTable: string, pageNumber: number): Promise<void> {
+    const results = this._displayedQuery;
+    if (results === undefined) {
+      throw new Error('trying to view a page of a query that is not loaded');
+    }
+
+    const sortedResultsMap: SortedResultsMap = {};
+    results.sortedResultsInfo.forEach(
+      (v, k) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(
+          v
+        ))
+    );
+
+    const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath, RAW_RESULTS_PAGE_SIZE);
+
+    const resultSetNames = schemas["result-sets"].map(resultSet => resultSet.name);
+
+    const schema = schemas["result-sets"].find(resultSet => resultSet.name == selectedTable)!;
+    if (schema === undefined)
+      throw new Error(`Query result set '${selectedTable}' not found.`);
+
+    const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name, RAW_RESULTS_PAGE_SIZE, schema.pagination?.offsets[pageNumber]);
+    const adaptedSchema = adaptSchema(schema);
+    const resultSet = adaptBqrs(adaptedSchema, chunk);
+
+    const parsedResultSets: ParsedResultSets = {
+      t: 'ExtensionParsed',
+      pageNumber,
+      resultSet,
+      numPages: numPagesOfResultSet(resultSet),
+      selectedTable: selectedTable,
+      resultSetNames
+    };
+
+    await this.postMessage({
+      t: "setState",
+      interpretation: this._interpretation,
+      origResultsPaths: results.query.resultsPaths,
+      resultsPath: this.convertPathToWebviewUri(
+        results.query.resultsPaths.resultsPath
+      ),
+      parsedResultSets,
+      sortedResultsMap,
+      database: results.database,
+      shouldKeepOldResultsWhileRendering: false,
+      metadata: results.query.metadata
+    });
+  }
+
+  private async getTruncatedResults(
+    metadata: QueryMetadata | undefined,
+    resultsPaths: ResultsPaths,
+    sourceInfo: cli.SourceInfo | undefined,
+    sourceLocationPrefix: string,
+    sortState: InterpretedResultsSortState | undefined
+  ): Promise<Interpretation> {
+    const sarif = await interpretResults(
+      this.cliServer,
+      metadata,
+      resultsPaths,
+      sourceInfo
+    );
     // For performance reasons, limit the number of results we try
     // to serialize and send to the webview. TODO: possibly also
     // limit number of paths per result, number of steps per path,
@@ -311,68 +499,110 @@ export class InterfaceManager extends DisposableObject {
       if (run.results !== undefined) {
         sortInterpretedResults(run.results, sortState);
         if (run.results.length > INTERPRETED_RESULTS_PER_RUN_LIMIT) {
-          numTruncatedResults += run.results.length - INTERPRETED_RESULTS_PER_RUN_LIMIT;
-          run.results = run.results.slice(0, INTERPRETED_RESULTS_PER_RUN_LIMIT);
+          numTruncatedResults +=
+            run.results.length - INTERPRETED_RESULTS_PER_RUN_LIMIT;
+          run.results = run.results.slice(
+            0,
+            INTERPRETED_RESULTS_PER_RUN_LIMIT
+          );
         }
       }
     });
-    return { sarif, sourceLocationPrefix, numTruncatedResults, sortState };
+    return {
+      sarif,
+      sourceLocationPrefix,
+      numTruncatedResults,
+      sortState
+    };
   }
 
-  private async interpretResultsInfo(query: QueryInfo, sortState: InterpretedResultsSortState | undefined): Promise<Interpretation | undefined> {
+  private async interpretResultsInfo(
+    query: QueryInfo,
+    sortState: InterpretedResultsSortState | undefined
+  ): Promise<Interpretation | undefined> {
     let interpretation: Interpretation | undefined = undefined;
-    if (await query.hasInterpretedResults()
-      && query.quickEvalPosition === undefined // never do results interpretation if quickEval
+    if (
+      (await query.canHaveInterpretedResults()) &&
+      query.quickEvalPosition === undefined // never do results interpretation if quickEval
     ) {
       try {
-        const sourceLocationPrefix = await query.dbItem.getSourceLocationPrefix(this.cliServer);
+        const sourceLocationPrefix = await query.dbItem.getSourceLocationPrefix(
+          this.cliServer
+        );
         const sourceArchiveUri = query.dbItem.sourceArchive;
-        const sourceInfo = sourceArchiveUri === undefined ?
-          undefined :
-          { sourceArchive: sourceArchiveUri.fsPath, sourceLocationPrefix };
-        interpretation = await this.getTruncatedResults(query.metadata, query.resultsPaths, sourceInfo, sourceLocationPrefix, sortState);
-      }
-      catch (e) {
+        const sourceInfo =
+          sourceArchiveUri === undefined
+            ? undefined
+            : {
+              sourceArchive: sourceArchiveUri.fsPath,
+              sourceLocationPrefix
+            };
+        interpretation = await this.getTruncatedResults(
+          query.metadata,
+          query.resultsPaths,
+          sourceInfo,
+          sourceLocationPrefix,
+          sortState
+        );
+      } catch (e) {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
-        this.logger.log(`Exception during results interpretation: ${e.message}. Will show raw results instead.`);
+        this.logger.log(
+          `Exception during results interpretation: ${e.message}. Will show raw results instead.`
+        );
       }
     }
     return interpretation;
   }
 
-
-  private async showResultsAsDiagnostics(resultsInfo: ResultsPaths, metadata: QueryMetadata | undefined, database: DatabaseItem) {
-    const sourceLocationPrefix = await database.getSourceLocationPrefix(this.cliServer);
+  private async showResultsAsDiagnostics(
+    resultsInfo: ResultsPaths,
+    metadata: QueryMetadata | undefined,
+    database: DatabaseItem
+  ): Promise<void> {
+    const sourceLocationPrefix = await database.getSourceLocationPrefix(
+      this.cliServer
+    );
     const sourceArchiveUri = database.sourceArchive;
-    const sourceInfo = sourceArchiveUri === undefined ?
-      undefined :
-      { sourceArchive: sourceArchiveUri.fsPath, sourceLocationPrefix };
+    const sourceInfo =
+      sourceArchiveUri === undefined
+        ? undefined
+        : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix
+        };
     const interpretation = await this.getTruncatedResults(
       metadata,
       resultsInfo,
       sourceInfo,
       sourceLocationPrefix,
-      undefined,
+      undefined
     );
 
     try {
-      await this.showProblemResultsAsDiagnostics(interpretation, database);
-    }
-    catch (e) {
+      await this.showProblemResultsAsDiagnostics(
+        interpretation,
+        database
+      );
+    } catch (e) {
       const msg = e instanceof Error ? e.message : e.toString();
-      this.logger.log(`Exception while computing problem results as diagnostics: ${msg}`);
+      this.logger.log(
+        `Exception while computing problem results as diagnostics: ${msg}`
+      );
       this._diagnosticCollection.clear();
     }
-
   }
 
-  private async showProblemResultsAsDiagnostics(interpretation: Interpretation, databaseItem: DatabaseItem): Promise<void> {
+  private async showProblemResultsAsDiagnostics(
+    interpretation: Interpretation,
+    databaseItem: DatabaseItem
+  ): Promise<void> {
     const { sarif, sourceLocationPrefix } = interpretation;
 
-
     if (!sarif.runs || !sarif.runs[0].results) {
-      this.logger.log("Didn't find a run in the sarif results. Error processing sarif?")
+      this.logger.log(
+        "Didn't find a run in the sarif results. Error processing sarif?"
+      );
       return;
     }
 
@@ -381,59 +611,68 @@ export class InterfaceManager extends DisposableObject {
     for (const result of sarif.runs[0].results) {
       const message = result.message.text;
       if (message === undefined) {
-        this.logger.log("Sarif had result without plaintext message")
+        this.logger.log("Sarif had result without plaintext message");
         continue;
       }
       if (!result.locations) {
-        this.logger.log("Sarif had result without location")
+        this.logger.log("Sarif had result without location");
         continue;
       }
 
-      const sarifLoc = parseSarifLocation(result.locations[0], sourceLocationPrefix);
+      const sarifLoc = parseSarifLocation(
+        result.locations[0],
+        sourceLocationPrefix
+      );
       if (sarifLoc.t == "NoLocation") {
         continue;
       }
-      const resultLocation = tryResolveLocation(sarifLoc, databaseItem)
+      const resultLocation = tryResolveLocation(sarifLoc, databaseItem);
       if (!resultLocation) {
-        this.logger.log("Sarif location was not resolvable " + sarifLoc)
+        this.logger.log("Sarif location was not resolvable " + sarifLoc);
         continue;
       }
       const parsedMessage = parseSarifPlainTextMessage(message);
       const relatedInformation: DiagnosticRelatedInformation[] = [];
       const relatedLocationsById: { [k: number]: Sarif.Location } = {};
 
-
-      for (let loc of result.relatedLocations || []) {
+      for (const loc of result.relatedLocations || []) {
         relatedLocationsById[loc.id!] = loc;
       }
-      let resultMessageChunks: string[] = [];
+      const resultMessageChunks: string[] = [];
       for (const section of parsedMessage) {
         if (typeof section === "string") {
           resultMessageChunks.push(section);
         } else {
           resultMessageChunks.push(section.text);
-          const sarifChunkLoc = parseSarifLocation(relatedLocationsById[section.dest], sourceLocationPrefix);
+          const sarifChunkLoc = parseSarifLocation(
+            relatedLocationsById[section.dest],
+            sourceLocationPrefix
+          );
           if (sarifChunkLoc.t == "NoLocation") {
             continue;
           }
-          const referenceLocation = tryResolveLocation(sarifChunkLoc, databaseItem);
-
+          const referenceLocation = tryResolveLocation(
+            sarifChunkLoc,
+            databaseItem
+          );
 
           if (referenceLocation) {
-            const related = new DiagnosticRelatedInformation(referenceLocation,
-              section.text);
+            const related = new DiagnosticRelatedInformation(
+              referenceLocation,
+              section.text
+            );
             relatedInformation.push(related);
           }
         }
       }
-      const diagnostic = new Diagnostic(resultLocation.range, resultMessageChunks.join(""), DiagnosticSeverity.Warning);
+      const diagnostic = new Diagnostic(
+        resultLocation.range,
+        resultMessageChunks.join(""),
+        DiagnosticSeverity.Warning
+      );
       diagnostic.relatedInformation = relatedInformation;
 
-      diagnostics.push([
-        resultLocation.uri,
-        [diagnostic]
-      ]);
-
+      diagnostics.push([resultLocation.uri, [diagnostic]]);
     }
     this._diagnosticCollection.set(diagnostics);
   }
@@ -442,18 +681,22 @@ export class InterfaceManager extends DisposableObject {
     return fileUriToWebviewUri(this.getPanel(), Uri.file(path));
   }
 
-  private convertPathPropertiesToWebviewUris(info: SortedResultSetInfo): SortedResultSetInfo {
+  private convertPathPropertiesToWebviewUris(
+    info: SortedResultSetInfo
+  ): SortedResultSetInfo {
     return {
       resultsPath: this.convertPathToWebviewUri(info.resultsPath),
       sortState: info.sortState
     };
   }
 
-  private handleSelectionChange(event: vscode.TextEditorSelectionChangeEvent) {
+  private handleSelectionChange(
+    event: vscode.TextEditorSelectionChangeEvent
+  ): void {
     if (event.kind === vscode.TextEditorSelectionChangeKind.Command) {
       return; // Ignore selection events we caused ourselves.
     }
-    let editor = vscode.window.activeTextEditor;
+    const editor = vscode.window.activeTextEditor;
     if (editor !== undefined) {
       editor.setDecorations(shownLocationDecoration, []);
       editor.setDecorations(shownLocationLineDecoration, []);
@@ -481,7 +724,7 @@ async function showLocation(loc: ResolvableLocationValue, databaseItem: Database
     const editor = editorsWithDoc.length > 0
       ? editorsWithDoc[0]
       : await Window.showTextDocument(doc, vscode.ViewColumn.One);
-    let range = resolvedLocation.range;
+    const range = resolvedLocation.range;
     // When highlighting the range, vscode's occurrence-match and bracket-match highlighting will
     // trigger based on where we place the cursor/selection, and will compete for the user's attention.
     // For reference:
@@ -492,7 +735,7 @@ async function showLocation(loc: ResolvableLocationValue, databaseItem: Database
     // For single-line ranges, select the whole range, mainly to disable bracket highlighting.
     // For multi-line ranges, place the cursor at the beginning to avoid visual artifacts from selected line-breaks.
     // Multi-line ranges are usually large enough to overshadow the noise from bracket highlighting.
-    let selectionEnd = (range.start.line === range.end.line)
+    const selectionEnd = (range.start.line === range.end.line)
       ? range.end
       : range.start;
     editor.selection = new vscode.Selection(range.start, selectionEnd);

extensions/ql-vscode/src/languageSupport.ts

@@ -0,0 +1,55 @@
+import { IndentAction, languages } from "vscode";
+
+
+/**
+ * OnEnterRules are available in language-configurations, but you cannot specify them in the language-configuration.json.
+ * They can only be specified programmatically.
+ *
+ * Also, we should keep the language-configuration.json as a json file and register it in the package.json because
+ * it is registered first, before the extension is activated, so language features are available quicker.
+ *
+ * See https://github.com/microsoft/vscode/issues/11514
+ * See https://github.com/microsoft/vscode/blob/master/src/vs/editor/test/common/modes/supports/javascriptOnEnterRules.ts
+ */
+export function install() {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const langConfig = require('../language-configuration.json');
+  // setLanguageConfiguration requires a regexp for the wordpattern, not a string
+  langConfig.wordPattern = new RegExp(langConfig.wordPattern);
+  langConfig.onEnterRules = onEnterRules;
+  langConfig.indentationRules = {
+		decreaseIndentPattern: /^((?!.*?\/\*).*\*\/)?\s*[\}\]].*$/,
+		increaseIndentPattern: /^((?!\/\/).)*(\{[^}"'`]*|\([^)"'`]*|\[[^\]"'`]*)$/
+	};
+
+  languages.setLanguageConfiguration('ql', langConfig);
+  languages.setLanguageConfiguration('qll', langConfig);
+  languages.setLanguageConfiguration('dbscheme', langConfig);
+}
+
+const onEnterRules = [
+  {
+    // e.g. /** | */
+    beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+    afterText: /^\s*\*\/$/,
+    action: { indentAction: IndentAction.IndentOutdent, appendText: ' * ' }
+  }, {
+    // e.g. /** ...|
+    beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
+    action: { indentAction: IndentAction.None, appendText: ' * ' }
+  }, {
+    // e.g.  * ...|
+    beforeText: /^(\t|[ ])*[ ]\*([ ]([^\*]|\*(?!\/))*)?$/,
+    oneLineAboveText: /^(\s*(\/\*\*|\*)).*/,
+    action: { indentAction: IndentAction.None, appendText: '* ' }
+  }, {
+    // e.g.  */|
+    beforeText: /^(\t|[ ])*[ ]\*\/\s*$/,
+    action: { indentAction: IndentAction.None, removeText: 1 }
+  },
+  {
+    // e.g.  *-----*/|
+    beforeText: /^(\t|[ ])*[ ]\*[^/]*\*\/\s*$/,
+    action: { indentAction: IndentAction.None, removeText: 1 }
+  }
+];

extensions/ql-vscode/src/logging.ts

@@ -1,33 +1,135 @@
-import { window as Window, OutputChannel, Progress } from 'vscode';
+import { window as Window, OutputChannel, Progress, Disposable } from 'vscode';
 import { DisposableObject } from 'semmle-vscode-utils';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+interface LogOptions {
+  /** If false, don't output a trailing newline for the log entry. Default true. */
+  trailingNewline?: boolean;
+
+  /** If specified, add this log entry to the log file at the specified location. */
+  additionalLogLocation?: string;
+}
 
 export interface Logger {
-  /** Writes the given log message, followed by a newline. */
-  log(message: string): void;
-  /** Writes the given log message, not followed by a newline. */
-  logWithoutTrailingNewline(message: string): void;
+  /** Writes the given log message, optionally followed by a newline. */
+  log(message: string, options?: LogOptions): Promise<void>;
+  /**
+   * Reveal this channel in the UI.
+   *
+   * @param preserveFocus When `true` the channel will not take focus.
+   */
+  show(preserveFocus?: boolean): void;
+
+  /**
+   * Remove the log at the specified location
+   * @param location log to remove
+   */
+  removeAdditionalLogLocation(location: string | undefined): void;
+
+  /**
+   * The base location location where all side log files are stored.
+   */
+  getBaseLocation(): string | undefined;
 }
 
 export type ProgressReporter = Progress<{ message: string }>;
 
 /** A logger that writes messages to an output channel in the Output tab. */
 export class OutputChannelLogger extends DisposableObject implements Logger {
-  outputChannel: OutputChannel;
+  public readonly outputChannel: OutputChannel;
+  private readonly additionalLocations = new Map<string, AdditionalLogLocation>();
+  private additionalLogLocationPath: string | undefined;
 
-  constructor(title: string) {
+  constructor(private title: string) {
     super();
     this.outputChannel = Window.createOutputChannel(title);
     this.push(this.outputChannel);
   }
 
-  log(message: string) {
-    this.outputChannel.appendLine(message);
+  init(storagePath: string): void {
+    this.additionalLogLocationPath = path.join(storagePath, this.title);
+
+    // clear out any old state from previous runs
+    fs.remove(this.additionalLogLocationPath);
   }
 
-  logWithoutTrailingNewline(message: string) {
-    this.outputChannel.append(message);
+  /**
+   * This function is asynchronous and will only resolve once the message is written
+   * to the side log (if required). It is not necessary to await the results of this
+   * function if you don't need to guarantee that the log writing is complete before
+   * continuing.
+   */
+  async log(message: string, options = {} as LogOptions): Promise<void> {
+    if (options.trailingNewline === undefined) {
+      options.trailingNewline = true;
+    }
+
+    if (options.trailingNewline) {
+      this.outputChannel.appendLine(message);
+    } else {
+      this.outputChannel.append(message);
+    }
+
+    if (this.additionalLogLocationPath && options.additionalLogLocation) {
+      const logPath = path.join(this.additionalLogLocationPath, options.additionalLogLocation);
+      let additional = this.additionalLocations.get(logPath);
+      if (!additional) {
+        const msg = `| Log being saved to ${logPath} |`;
+        const separator = new Array(msg.length).fill('-').join('');
+        this.outputChannel.appendLine(separator);
+        this.outputChannel.appendLine(msg);
+        this.outputChannel.appendLine(separator);
+        additional = new AdditionalLogLocation(logPath);
+        this.additionalLocations.set(logPath, additional);
+        this.track(additional);
+      }
+
+      await additional.log(message, options);
+    }
   }
 
+  show(preserveFocus?: boolean): void {
+    this.outputChannel.show(preserveFocus);
+  }
+
+  removeAdditionalLogLocation(location: string | undefined): void {
+    if (this.additionalLogLocationPath && location) {
+      const logPath = location.startsWith(this.additionalLogLocationPath)
+        ? location
+        : path.join(this.additionalLogLocationPath, location);
+      const additional = this.additionalLocations.get(logPath);
+      if (additional) {
+        this.disposeAndStopTracking(additional);
+        this.additionalLocations.delete(logPath);
+      }
+    }
+  }
+
+  getBaseLocation() {
+    return this.additionalLogLocationPath;
+  }
+}
+
+class AdditionalLogLocation extends Disposable {
+  constructor(private location: string) {
+    super(() => { /**/ });
+  }
+
+  async log(message: string, options = {} as LogOptions): Promise<void> {
+    if (options.trailingNewline === undefined) {
+      options.trailingNewline = true;
+    }
+    await fs.ensureFile(this.location);
+
+    await fs.appendFile(this.location, message + (options.trailingNewline ? '\n' : ''), {
+      encoding: 'utf8'
+    });
+  }
+
+  async dispose(): Promise<void> {
+    await fs.remove(this.location);
+  }
 }
 
 /** The global logger for the extension. */
@@ -37,4 +139,9 @@ export const logger = new OutputChannelLogger('CodeQL Extension Log');
 export const queryServerLogger = new OutputChannelLogger('CodeQL Query Server');
 
 /** The logger for messages from the language server. */
-export const ideServerLogger = new OutputChannelLogger('CodeQL Language Server');
+export const ideServerLogger = new OutputChannelLogger(
+  'CodeQL Language Server'
+);
+
+/** The logger for messages from tests. */
+export const testLogger = new OutputChannelLogger('CodeQL Tests');

extensions/ql-vscode/src/messages.ts

@@ -1,6 +1,17 @@
 /**
  * Types for messages exchanged during jsonrpc communication with the
  * the CodeQL query server.
+ *
+ * This file exists in the queryserver and in the vscode extension, and
+ * should be kept in sync between them.
+ *
+ * A note about the namespaces below, which look like they are
+ * essentially enums, namely Severity, ResultColumnKind, and
+ * QueryResultType. By design, for the sake of extensibility, clients
+ * receiving messages of this protocol are supposed to accept any
+ * number for any of these types. We commit to the given meaning of
+ * the numbers listed in constants in the namespaces, and we commit to
+ * the fact that any unknown QueryResultType value counts as an error.
  */
 
 import * as rpc from 'vscode-jsonrpc';
@@ -205,19 +216,19 @@ export interface QlFileSet {
   /**
    * The files imported by the given file
    */
-  imports: { [key: string]: string[]; };
+  imports: { [key: string]: string[] };
   /**
    * An id of each file
    */
-  nodeNumbering: { [key: string]: number; };
+  nodeNumbering: { [key: string]: number };
   /**
    * The code for each file
    */
-  qlCode: { [key: string]: string; };
+  qlCode: { [key: string]: string };
   /**
    * The resolution of an import in each directory.
    */
-  resolvedDirImports: { [key: string]: { [key: string]: string; }; };
+  resolvedDirImports: { [key: string]: { [key: string]: string } };
 }
 
 /**
@@ -294,11 +305,15 @@ export interface CompilationMessage {
   /**
    * The severity of the message
    */
-  severity: number;
+  severity: Severity;
 }
+
+export type Severity = number;
 /**
- * Severity of different messages
+ * Severity of different messages. This namespace is intentionally not
+ * an enum, see "for the sake of extensibility" comment above.
  */
+// eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace Severity {
   /**
    * The message is a compilation error.
@@ -333,7 +348,7 @@ export interface ResultColumn {
    * The kind of the column. See `ResultColumnKind`
    * for the current possible meanings
    */
-  kind: number;
+  kind: ResultColumnKind;
   /**
    * The name of the column.
    * This may be compiler generated for complex select expressions.
@@ -341,9 +356,12 @@ export interface ResultColumn {
   name: string;
 }
 
+export type ResultColumnKind = number;
 /**
- * The kind of a result column.
+ * The kind of a result column. This namespace is intentionally not an enum, see "for the sake of
+ * extensibility" comment above.
  */
+// eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace ResultColumnKind {
   /**
    * A column of type `float`
@@ -619,6 +637,8 @@ export interface EvaluateQueriesParams {
   useSequenceHint: boolean;
 }
 
+export type TemplateDefinitions = { [key: string]: TemplateSource }
+
 /**
  * A single query that should be run
  */
@@ -642,7 +662,7 @@ export interface QueryToRun {
   /**
    * Values to set for each template
    */
-  templateValues?: { [key: string]: TemplateSource; };
+  templateValues?: TemplateDefinitions;
   /**
    * Whether templates without values in the templateValues
    * map should be set to the empty set or give an error.
@@ -730,7 +750,7 @@ export interface ResultSet {
 /**
  * The type returned when the evaluation is complete
  */
-export interface EvaluationComplete { }
+export type EvaluationComplete = {};
 
 /**
  * The result of a single query
@@ -748,7 +768,7 @@ export interface EvaluationResult {
    * The type of the result. See QueryResultType for
    * possible meanings. Any other result should be interpreted as an error.
    */
-  resultType: number;
+  resultType: QueryResultType;
   /**
    * The wall clock time it took to evaluate the query.
    * The time is from when we initially tried to evaluate the query
@@ -760,11 +780,19 @@ export interface EvaluationResult {
    * An error message if an error happened
    */
   message?: string;
+
+  /**
+   * Full path to file with all log messages emitted while this query was active, if one exists
+   */
+  logFileLocation?: string;
 }
 
+export type QueryResultType = number;
 /**
- * The result of running a query,
+ * The result of running a query. This namespace is intentionally not
+ * an enum, see "for the sake of extensibility" comment above.
  */
+// eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace QueryResultType {
   /**
    * The query ran successfully
@@ -818,7 +846,7 @@ export interface RunUpgradeResult {
    * The type of the result. See QueryResultType for
    * possible meanings. Any other result should be interpreted as an error.
    */
-  resultType: number;
+  resultType: QueryResultType;
   /**
    * The error message if an error occurred
    */
@@ -837,11 +865,11 @@ export interface WithProgressId<T> {
   /**
    * The main body
    */
-  body: T,
+  body: T;
   /**
    * The id used to report progress updates
    */
-  progressId: number
+  progressId: number;
 }
 
 export interface ProgressMessage {
@@ -910,7 +938,7 @@ export const runUpgrade = new rpc.RequestType<WithProgressId<RunUpgradeParams>,
  * Request returned to the client to notify completion of a query.
  * The full runQueries job is completed when all queries are acknowledged.
  */
-export const completeQuery = new rpc.RequestType<EvaluationResult, Object, void, void>('evaluation/queryCompleted');
+export const completeQuery = new rpc.RequestType<EvaluationResult, Record<string, any>, void, void>('evaluation/queryCompleted');
 
 /**
  * A notification that the progress has been changed.

extensions/ql-vscode/src/qlpack-discovery.ts

@@ -0,0 +1,59 @@
+import { EventEmitter, Event, Uri, WorkspaceFolder, RelativePattern } from 'vscode';
+import { MultiFileSystemWatcher } from 'semmle-vscode-utils';
+import { CodeQLCliServer, QlpacksInfo } from './cli';
+import { Discovery } from './discovery';
+
+export interface QLPack {
+  name: string;
+  uri: Uri;
+}
+
+/**
+ * Service to discover all available QL packs in a workspace folder.
+ */
+export class QLPackDiscovery extends Discovery<QlpacksInfo> {
+  private readonly _onDidChangeQLPacks = this.push(new EventEmitter<void>());
+  private readonly watcher = this.push(new MultiFileSystemWatcher());
+  private _qlPacks: readonly QLPack[] = [];
+
+  constructor(private readonly workspaceFolder: WorkspaceFolder,
+    private readonly cliServer: CodeQLCliServer) {
+
+    super();
+
+    // Watch for any changes to `qlpack.yml` files in this workspace folder.
+    // TODO: The CLI server should tell us what paths to watch for.
+    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/qlpack.yml'));
+    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/.codeqlmanifest.json'));
+    this.push(this.watcher.onDidChange(this.handleQLPackFileChanged, this));
+
+    this.refresh();
+  }
+
+  public get onDidChangeQLPacks(): Event<void> { return this._onDidChangeQLPacks.event; }
+
+  public get qlPacks(): readonly QLPack[] { return this._qlPacks; }
+
+  private handleQLPackFileChanged(_uri: Uri): void {
+    this.refresh();
+  }
+
+  protected discover(): Promise<QlpacksInfo> {
+    // Only look for QL packs in this workspace folder.
+    return this.cliServer.resolveQlpacks([this.workspaceFolder.uri.fsPath], []);
+  }
+
+  protected update(results: QlpacksInfo): void {
+    const qlPacks: QLPack[] = [];
+    for (const id in results) {
+      qlPacks.push(...results[id].map(fsPath => {
+        return {
+          name: id,
+          uri: Uri.file(fsPath)
+        };
+      }));
+    }
+    this._qlPacks = qlPacks;
+    this._onDidChangeQLPacks.fire();
+  }
+}

extensions/ql-vscode/src/qltest-discovery.ts

@@ -0,0 +1,223 @@
+import * as path from 'path';
+import { QLPackDiscovery } from './qlpack-discovery';
+import { Discovery } from './discovery';
+import { EventEmitter, Event, Uri, RelativePattern } from 'vscode';
+import { MultiFileSystemWatcher } from 'semmle-vscode-utils';
+import { CodeQLCliServer } from './cli';
+
+/**
+ * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`.
+ */
+export abstract class QLTestNode {
+  constructor(private _path: string, private _name: string) {
+  }
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public abstract get children(): readonly QLTestNode[];
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more QL tests or other test directories.
+ */
+export class QLTestDirectory extends QLTestNode {
+  private _children: QLTestNode[] = [];
+
+  constructor(_path: string, _name: string) {
+    super(_path, _name);
+  }
+
+  public get children(): readonly QLTestNode[] {
+    return this._children;
+  }
+
+  public addChild(child: QLTestNode): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): QLTestDirectory {
+    const dirName = path.dirname(relativePath);
+    if (dirName === '.') {
+      return this.createChildDirectory(relativePath);
+    }
+    else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(path.basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    this._children.sort((a, b) => a.name.localeCompare(b.name));
+    for (const child of this._children) {
+      child.finish();
+    }
+  }
+
+  private createChildDirectory(name: string): QLTestDirectory {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return existingChild as QLTestDirectory;
+    }
+    else {
+      const newChild = new QLTestDirectory(path.join(this.path, name), name);
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single QL test. This will be either a `.ql` file or a `.qlref` file.
+ */
+export class QLTestFile extends QLTestNode {
+  constructor(_path: string, _name: string) {
+    super(_path, _name);
+  }
+
+  public get children(): readonly QLTestNode[] {
+    return [];
+  }
+
+  public finish(): void {
+    /**/
+  }
+}
+
+/**
+ * The results of discovering QL tests.
+ */
+interface QLTestDiscoveryResults {
+  /**
+   * The root test directory for each QL pack that contains tests.
+   */
+  testDirectories: QLTestDirectory[];
+  /**
+   * The list of file system paths to watch. If any of these paths changes, the discovery results
+   * may be out of date.
+   */
+  watchPaths: string[];
+}
+
+/**
+ * Discovers all QL tests contained in the QL packs in a given workspace folder.
+ */
+export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
+  private readonly _onDidChangeTests = this.push(new EventEmitter<void>());
+  private readonly watcher: MultiFileSystemWatcher = this.push(new MultiFileSystemWatcher());
+  private _testDirectories: QLTestDirectory[] = [];
+
+  constructor(private readonly qlPackDiscovery: QLPackDiscovery,
+    private readonly cliServer: CodeQLCliServer) {
+
+    super();
+
+    this.push(this.qlPackDiscovery.onDidChangeQLPacks(this.handleDidChangeQLPacks, this));
+    this.push(this.watcher.onDidChange(this.handleDidChange, this));
+
+    this.refresh();
+  }
+
+  /**
+   * Event to be fired when the set of discovered tests may have changed.
+   */
+  public get onDidChangeTests(): Event<void> { return this._onDidChangeTests.event; }
+
+  /**
+   * The root test directory for each QL pack that contains tests.
+   */
+  public get testDirectories(): QLTestDirectory[] { return this._testDirectories; }
+
+  private handleDidChangeQLPacks(): void {
+    this.refresh();
+  }
+
+  private handleDidChange(uri: Uri): void {
+    if (!QLTestDiscovery.ignoreTestPath(uri.fsPath)) {
+      this.refresh();
+    }
+  }
+
+  protected async discover(): Promise<QLTestDiscoveryResults> {
+    const testDirectories: QLTestDirectory[] = [];
+    const watchPaths: string[] = [];
+    const qlPacks = this.qlPackDiscovery.qlPacks;
+    for (const qlPack of qlPacks) {
+      //HACK: Assume that only QL packs whose name ends with '-tests' contain tests.
+      if (qlPack.name.endsWith('-tests')) {
+        watchPaths.push(qlPack.uri.fsPath);
+        const testPackage = await this.discoverTests(qlPack.uri.fsPath, qlPack.name);
+        if (testPackage !== undefined) {
+          testDirectories.push(testPackage);
+        }
+      }
+    }
+
+    return {
+      testDirectories: testDirectories,
+      watchPaths: watchPaths
+    };
+  }
+
+  protected update(results: QLTestDiscoveryResults): void {
+    this._testDirectories = results.testDirectories;
+
+    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
+    this.watcher.clear();
+    results.watchPaths.forEach(watchPath => {
+      this.watcher.addWatch(new RelativePattern(watchPath, '**/*.{ql,qlref}'));
+    });
+    this._onDidChangeTests.fire();
+  }
+
+  /**
+   * Discover all QL tests in the specified directory and its subdirectories.
+   * @param fullPath The full path of the test directory.
+   * @param name The display name to use for the returned `TestDirectory` object.
+   * @returns A `QLTestDirectory` object describing the contents of the directory, or `undefined` if
+   *   no tests were found.
+   */
+  private async discoverTests(fullPath: string, name: string): Promise<QLTestDirectory | undefined> {
+    const resolvedTests = (await this.cliServer.resolveTests(fullPath))
+      .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
+
+    if (resolvedTests.length === 0) {
+      return undefined;
+    }
+    else {
+      const rootDirectory = new QLTestDirectory(fullPath, name);
+      for (const testPath of resolvedTests) {
+        const relativePath = path.normalize(path.relative(fullPath, testPath));
+        const dirName = path.dirname(relativePath);
+        const parentDirectory = rootDirectory.createDirectory(dirName);
+        parentDirectory.addChild(new QLTestFile(testPath, path.basename(testPath)));
+      }
+
+      rootDirectory.finish();
+
+      return rootDirectory;
+    }
+  }
+
+  /**
+   * Determine if the specified QL test should be ignored based on its filename.
+   * @param testPath Path to the test file.
+   */
+  private static ignoreTestPath(testPath: string): boolean {
+    switch (path.extname(testPath).toLowerCase()) {
+      case '.ql':
+      case '.qlref':
+        return path.basename(testPath).startsWith('__');
+
+      default:
+        return false;
+    }
+  }
+}

extensions/ql-vscode/src/query-history.ts

@@ -4,6 +4,9 @@ import { ExtensionContext, window as Window } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
+import * as helpers from './helpers';
+import { logger } from './logging';
+import { URLSearchParams } from 'url';
 
 /**
  * query-history.ts
@@ -15,14 +18,37 @@ import { QueryWithResults } from './run-queries';
  */
 
 export type QueryHistoryItemOptions = {
-  label?: string, // user-settable label
-  queryText?: string, // stored query for quick query
+  label?: string; // user-settable label
+  queryText?: string; // text of the selected file
+  isQuickQuery?: boolean;
 }
 
+const SHOW_QUERY_TEXT_MSG = `\
+////////////////////////////////////////////////////////////////////////////////////
+// This is the text of the entire query file when it was executed for this query  //
+// run. The text or dependent libraries may have changed since then.              //
+//                                                                                //
+// This buffer is readonly. To re-execute this query, you must open the original  //
+// query file.                                                                    //
+////////////////////////////////////////////////////////////////////////////////////
+
+`;
+
+const SHOW_QUERY_TEXT_QUICK_EVAL_MSG = `\
+////////////////////////////////////////////////////////////////////////////////////
+// This is the Quick Eval selection of the query file when it was executed for    //
+// this query run. The text or dependent libraries may have changed since then.   //
+//                                                                                //
+// This buffer is readonly. To re-execute this query, you must open the original  //
+// query file.                                                                    //
+////////////////////////////////////////////////////////////////////////////////////
+
+`;
+
 /**
  * Path to icon to display next to a failed query history item.
  */
-const FAILED_QUERY_HISTORY_ITEM_ICON: string = 'media/red-x.svg';
+const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
 
 /**
  * Tree data provider for the query history view.
@@ -48,7 +74,7 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
   constructor(private ctx: ExtensionContext) {
   }
 
-  getTreeItem(element: CompletedQuery): vscode.TreeItem {
+  async getTreeItem(element: CompletedQuery): Promise<vscode.TreeItem> {
     const it = new vscode.TreeItem(element.toString());
 
     it.command = {
@@ -57,6 +83,11 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
       arguments: [element],
     };
 
+    // Mark this query history item according to whether it has a
+    // SARIF file so that we can make context menu items conditionally
+    // available.
+    it.contextValue = await element.query.hasInterpretedResults() ? 'interpretedResultsItem' : 'rawResultsItem';
+
     if (!element.didRunSuccessfully) {
       it.iconPath = path.join(this.ctx.extensionPath, FAILED_QUERY_HISTORY_ITEM_ICON);
     }
@@ -122,7 +153,7 @@ export class QueryHistoryManager {
   ctx: ExtensionContext;
   treeView: vscode.TreeView<CompletedQuery>;
   selectedCallback: ((item: CompletedQuery) => void) | undefined;
-  lastItemClick: { time: Date, item: CompletedQuery } | undefined;
+  lastItemClick: { time: Date; item: CompletedQuery } | undefined;
 
   async invokeCallbackOn(queryHistoryItem: CompletedQuery) {
     if (this.selectedCallback !== undefined) {
@@ -135,7 +166,7 @@ export class QueryHistoryManager {
     const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.query.program.queryPath));
     const editor = await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
     const queryText = queryHistoryItem.options.queryText;
-    if (queryText !== undefined) {
+    if (queryText !== undefined && queryHistoryItem.options.isQuickQuery) {
       await editor.edit(edit => edit.replace(textDocument.validateRange(
         new vscode.Range(0, 0, textDocument.lineCount, 0)), queryText)
       );
@@ -144,6 +175,7 @@ export class QueryHistoryManager {
 
   async handleRemoveHistoryItem(queryHistoryItem: CompletedQuery) {
     this.treeDataProvider.remove(queryHistoryItem);
+    queryHistoryItem.dispose();
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
       this.treeView.reveal(current);
@@ -187,6 +219,85 @@ export class QueryHistoryManager {
     }
   }
 
+  async handleShowQueryLog(queryHistoryItem: CompletedQuery) {
+    if (queryHistoryItem.logFileLocation) {
+      const uri = vscode.Uri.file(queryHistoryItem.logFileLocation);
+      try {
+        await vscode.window.showTextDocument(uri);
+      } catch (e) {
+        if (e.message.includes('Files above 50MB cannot be synchronized with extensions')) {
+          const res = await helpers.showBinaryChoiceDialog(
+            `VS Code does not allow extensions to open files >50MB. This file
+exceeds that limit. Do you want to open it outside of VS Code?
+
+You can also try manually opening it inside VS Code by selecting
+the file in the file explorer and dragging it into the workspace.`
+          );
+          if (res) {
+            try {
+              await vscode.commands.executeCommand('revealFileInOS', uri);
+            } catch (e) {
+              helpers.showAndLogErrorMessage(e.message);
+            }
+          }
+        } else {
+          helpers.showAndLogErrorMessage(`Could not open log file ${queryHistoryItem.logFileLocation}`);
+          logger.log(e.message);
+          logger.log(e.stack);
+        }
+
+      }
+    } else {
+      helpers.showAndLogWarningMessage('No log file available');
+    }
+  }
+
+  async handleShowQueryText(queryHistoryItem: CompletedQuery) {
+    try {
+      const queryName = queryHistoryItem.queryName.endsWith('.ql') ? queryHistoryItem.queryName : queryHistoryItem.queryName + '.ql';
+      const params = new URLSearchParams({
+        isQuickEval: String(!!queryHistoryItem.query.quickEvalPosition),
+        queryText: await this.getQueryText(queryHistoryItem)
+      });
+      const uri = vscode.Uri.parse(`codeql:${queryHistoryItem.query.queryID}-${queryName}?${params.toString()}`);
+      const doc = await vscode.workspace.openTextDocument(uri);
+      await vscode.window.showTextDocument(doc, { preview: false });
+    } catch (e) {
+      helpers.showAndLogErrorMessage(e.message);
+    }
+  }
+
+  async handleViewSarif(queryHistoryItem: CompletedQuery) {
+    try {
+      const hasInterpretedResults = await queryHistoryItem.query.canHaveInterpretedResults();
+      if (hasInterpretedResults) {
+        const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.query.resultsPaths.interpretedResultsPath));
+        await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+      }
+      else {
+        const label = queryHistoryItem.getLabel();
+        helpers.showAndLogInformationMessage(`Query ${label} has no interpreted results.`);
+      }
+    } catch (e) {
+      helpers.showAndLogErrorMessage(e.message);
+    }
+  }
+
+  async getQueryText(queryHistoryItem: CompletedQuery): Promise<string> {
+    if (queryHistoryItem.options.queryText) {
+      return queryHistoryItem.options.queryText;
+    } else if (queryHistoryItem.query.quickEvalPosition) {
+      // capture all selected lines
+      const startLine = queryHistoryItem.query.quickEvalPosition.line;
+      const endLine = queryHistoryItem.query.quickEvalPosition.endLine;
+      const textDocument =
+        await vscode.workspace.openTextDocument(queryHistoryItem.query.quickEvalPosition.fileName);
+      return textDocument.getText(new vscode.Range(startLine - 1, 0, endLine, 0));
+    } else {
+      return '';
+    }
+  }
+
   constructor(
     ctx: ExtensionContext,
     private queryHistoryConfigListener: QueryHistoryConfig,
@@ -205,15 +316,30 @@ export class QueryHistoryManager {
         this.updateTreeViewSelectionIfVisible();
       }
     });
+    logger.log('Registering query history panel commands.');
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.openQuery', this.handleOpenQuery));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.removeHistoryItem', this.handleRemoveHistoryItem.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.setLabel', this.handleSetLabel.bind(this)));
+    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryLog', this.handleShowQueryLog.bind(this)));
+    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryText', this.handleShowQueryText.bind(this)));
+    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.viewSarif', this.handleViewSarif.bind(this)));
     ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.itemClicked', async (item) => {
       return this.handleItemClicked(item);
     }));
     queryHistoryConfigListener.onDidChangeQueryHistoryConfiguration(() => {
       this.treeDataProvider.refresh();
     });
+
+    // displays query text in a read-only document
+    vscode.workspace.registerTextDocumentContentProvider('codeql', {
+      provideTextDocumentContent(uri: vscode.Uri): vscode.ProviderResult<string> {
+        const params = new URLSearchParams(uri.query);
+
+        return (
+          JSON.parse(params.get('isQuickEval') || '') ? SHOW_QUERY_TEXT_QUICK_EVAL_MSG : SHOW_QUERY_TEXT_MSG
+        ) + params.get('queryText');
+      }
+    });
   }
 
   addQuery(info: QueryWithResults): CompletedQuery {

extensions/ql-vscode/src/query-results.ts

@@ -5,7 +5,7 @@ import * as cli from './cli';
 import * as sarif from 'sarif';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState } from "./interface-types";
+import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState, ResultsPaths } from "./interface-types";
 import { QueryHistoryConfig } from "./config";
 import { QueryHistoryItemOptions } from "./query-history";
 
@@ -14,7 +14,9 @@ export class CompletedQuery implements QueryWithResults {
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
   readonly database: DatabaseInfo;
+  readonly logFileLocation?: string;
   options: QueryHistoryItemOptions;
+  dispose: () => void;
 
   /**
    * Map from result set name to SortedResultSetInfo.
@@ -31,15 +33,18 @@ export class CompletedQuery implements QueryWithResults {
   interpretedResultsSortState: InterpretedResultsSortState | undefined;
 
   constructor(
-    evalaution: QueryWithResults,
+    evaluation: QueryWithResults,
     public config: QueryHistoryConfig,
   ) {
-    this.query = evalaution.query;
-    this.result = evalaution.result;
-    this.database = evalaution.database;
+    this.query = evaluation.query;
+    this.result = evaluation.result;
+    this.database = evaluation.database;
+    this.logFileLocation = evaluation.logFileLocation;
+    this.options = evaluation.options;
+    this.dispose = evaluation.dispose;
+
     this.time = new Date().toLocaleString();
     this.sortedResultsInfo = new Map();
-    this.options = evalaution.options;
   }
 
   get databaseName(): string {
@@ -49,13 +54,6 @@ export class CompletedQuery implements QueryWithResults {
     return helpers.getQueryName(this.query);
   }
 
-  /**
-   * Holds if this query should produce interpreted results.
-   */
-  canInterpretedResults(): Promise<boolean> {
-    return this.query.dbItem.hasMetadataFile();
-  }
-
   get statusString(): string {
     switch (this.result.resultType) {
       case messages.QueryResultType.CANCELLATION:
@@ -68,7 +66,7 @@ export class CompletedQuery implements QueryWithResults {
         return `timed out after ${this.result.evaluationTime / 1000} seconds`;
       case messages.QueryResultType.OTHER_ERROR:
       default:
-        return `failed`;
+        return this.result.message ? `failed: ${this.result.message}` : 'failed';
     }
   }
 
@@ -125,9 +123,8 @@ export class CompletedQuery implements QueryWithResults {
 /**
  * Call cli command to interpret results.
  */
-export async function interpretResults(server: cli.CodeQLCliServer, metadata: QueryMetadata | undefined, resultsPath: string, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
-  const interpretedResultsPath = resultsPath + ".interpreted.sarif"
-
+export async function interpretResults(server: cli.CodeQLCliServer, metadata: QueryMetadata | undefined, resultsPaths: ResultsPaths, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
+  const { resultsPath, interpretedResultsPath } = resultsPaths;
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,14 +1,18 @@
 import * as cp from 'child_process';
-import { DisposableObject } from 'semmle-vscode-utils';
+import * as path from 'path';
+// Import from the specific module within `semmle-vscode-utils`, rather than via `index.ts`, because
+// we avoid taking an accidental runtime dependency on `vscode` this way.
+import { DisposableObject } from 'semmle-vscode-utils/out/disposable-object';
 import { Disposable } from 'vscode';
 import { CancellationToken, createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
 import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
 import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './messages';
+import * as messages from './messages';
 
 type ServerOpts = {
-  logger: Logger
+  logger: Logger;
 }
 
 /** A running query server process and its associated message connection. */
@@ -23,7 +27,7 @@ class ServerProcess implements Disposable {
     this.logger = logger;
   }
 
-  dispose() {
+  dispose(): void {
     this.logger.log('Stopping query server...');
     this.connection.dispose();
     this.child.stdin!.end();
@@ -51,12 +55,16 @@ export class QueryServerClient extends DisposableObject {
   nextCallback: number;
   nextProgress: number;
   withProgressReporting: WithProgressReporting;
+  public activeQueryName: string | undefined;
 
   constructor(readonly config: QueryServerConfig, readonly cliServer: cli.CodeQLCliServer, readonly opts: ServerOpts, withProgressReporting: WithProgressReporting) {
     super();
     // When the query server configuration changes, restart the query server.
     if (config.onDidChangeQueryServerConfiguration !== undefined) {
-      this.push(config.onDidChangeQueryServerConfiguration(async () => await this.restartQueryServer(), this));
+      this.push(config.onDidChangeQueryServerConfiguration(async () => {
+        this.logger.log('Restarting query server due to configuration changes...');
+        await this.restartQueryServer();
+      }, this));
     }
     this.withProgressReporting = withProgressReporting;
     this.nextCallback = 0;
@@ -65,32 +73,37 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
-  get logger() { return this.opts.logger; }
+  get logger(): Logger {
+    return this.opts.logger;
+  }
 
   /** Stops the query server by disposing of the current server process. */
-  private stopQueryServer() {
+  private stopQueryServer(): void {
     if (this.serverProcess !== undefined) {
       this.disposeAndStopTracking(this.serverProcess);
     } else {
-      this.logger.log('No server process to be stopped.')
+      this.logger.log('No server process to be stopped.');
     }
   }
 
   /** Restarts the query server by disposing of the current server process and then starting a new one. */
-  private async restartQueryServer() {
-    this.logger.log('Restarting query server due to configuration changes...');
+  async restartQueryServer(): Promise<void> {
     this.stopQueryServer();
     await this.startQueryServer();
   }
 
+  showLog(): void {
+    this.logger.show();
+  }
+
   /** Starts a new query server process, sending progress messages to the status bar. */
-  async startQueryServer() {
+  async startQueryServer(): Promise<void> {
     // Use an arrow function to preserve the value of `this`.
     return this.withProgressReporting((progress, _) => this.startQueryServerImpl(progress));
   }
 
   /** Starts a new query server process, sending progress messages to the given reporter. */
-  private async startQueryServerImpl(progressReporter: ProgressReporter) {
+  private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
     if (this.config.debug) {
@@ -102,7 +115,10 @@ export class QueryServerClient extends DisposableObject {
       ['execute', 'query-server'],
       args,
       this.logger,
-      data => this.logger.logWithoutTrailingNewline(data.toString()),
+      data => this.logger.log(data.toString(), {
+        trailingNewline: false,
+        additionalLogLocation: this.activeQueryName
+      }),
       undefined, // no listener for stdout
       progressReporter
     );
@@ -113,16 +129,20 @@ export class QueryServerClient extends DisposableObject {
         this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
       }
       else {
+        const baseLocation = this.logger.getBaseLocation();
+        if (baseLocation && this.activeQueryName) {
+          res.logFileLocation = path.join(baseLocation, this.activeQueryName);
+        }
         this.evaluationResultCallbacks[res.runId](res);
       }
       return {};
-    })
+    });
     connection.onNotification(progress, res => {
-      let callback = this.progressCallbacks[res.id];
+      const callback = this.progressCallbacks[res.id];
       if (callback) {
         callback(res);
       }
-    })
+    });
     this.serverProcess = new ServerProcess(child, connection, this.opts.logger);
     // Ensure the server process is disposed together with this client.
     this.track(this.serverProcess);
@@ -140,7 +160,7 @@ export class QueryServerClient extends DisposableObject {
     return id;
   }
 
-  unRegisterCallback(id: number) {
+  unRegisterCallback(id: number): void {
     delete this.evaluationResultCallbacks[id];
   }
 
@@ -149,8 +169,10 @@ export class QueryServerClient extends DisposableObject {
   }
 
   async sendRequest<P, R, E, RO>(type: RequestType<WithProgressId<P>, R, E, RO>, parameter: P, token?: CancellationToken, progress?: (res: ProgressMessage) => void): Promise<R> {
-    let id = this.nextProgress++;
+    const id = this.nextProgress++;
     this.progressCallbacks[id] = progress;
+
+    this.updateActiveQuery(type.method, parameter);
     try {
       if (this.serverProcess === undefined) {
         throw new Error('No query server process found.');
@@ -160,4 +182,19 @@ export class QueryServerClient extends DisposableObject {
       delete this.progressCallbacks[id];
     }
   }
+
+  /**
+   * Updates the active query every time there is a new request to compile.
+   * The active query is used to specify the side log.
+   *
+   * This isn't ideal because in situations where there are queries running
+   * in parallel, each query's log messages are interleaved. Fixing this
+   * properly will require a change in the query server.
+   */
+  private updateActiveQuery(method: string, parameter: any): void {
+    if (method === messages.compileQuery.method) {
+      const queryPath = parameter?.queryToCheck?.queryPath || 'unknown';
+      this.activeQueryName = `query-${path.basename(queryPath)}-${this.nextProgress}.log`;
+    }
+  }
 }

extensions/ql-vscode/src/quick-query.ts

@@ -1,5 +1,4 @@
 import * as fs from 'fs-extra';
-import * as glob from 'glob-promise';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
 import { ExtensionContext, window as Window, workspace, Uri } from 'vscode';
@@ -12,38 +11,12 @@ import { UserCancellationException } from './run-queries';
 
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
+const QUICK_QUERY_WORKSPACE_FOLDER_NAME = 'Quick Queries';
 
 export function isQuickQueryPath(queryPath: string): boolean {
   return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
 }
 
-async function getQlPackFor(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
-  const qlpacks = await cliServer.resolveQlpacks(helpers.getOnDiskWorkspaceFolders());
-  const packs: { packDir: string | undefined, packName: string }[] =
-    Object.entries(qlpacks).map(([packName, dirs]) => {
-      if (dirs.length < 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
-        return { packName, packDir: undefined };
-      }
-      if (dirs.length > 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
-      }
-      return {
-        packName,
-        packDir: dirs[0]
-      }
-    });
-  for (const { packDir, packName } of packs) {
-    if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8'));
-      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
-        return packName;
-      }
-    }
-  }
-  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
-}
-
 /**
  * `getBaseText` heuristically returns an appropriate import statement
  * prelude based on the filename of the dbscheme file given. TODO: add
@@ -59,7 +32,7 @@ function getBaseText(dbschemeBase: string) {
   return 'select ""';
 }
 
-async function getQuickQueriesDir(ctx: ExtensionContext): Promise<string> {
+function getQuickQueriesDir(ctx: ExtensionContext): string {
   const storagePath = ctx.storagePath;
   if (storagePath === undefined) {
     throw new Error('Workspace storage path is undefined');
@@ -69,11 +42,25 @@ async function getQuickQueriesDir(ctx: ExtensionContext): Promise<string> {
   return queriesPath;
 }
 
+
+
+
 /**
  * Show a buffer the user can enter a simple query into.
  */
 export async function displayQuickQuery(ctx: ExtensionContext, cliServer: CodeQLCliServer, databaseUI: DatabaseUI) {
+
+  function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
+    workspace.updateWorkspaceFolders(
+      index,
+      len,
+      { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
+    );
+  }
+
   try {
+    const workspaceFolders = workspace.workspaceFolders || [];
+    const queriesDir = await getQuickQueriesDir(ctx);
 
     // If there is already a quick query open, don't clobber it, just
     // show it.
@@ -83,19 +70,29 @@ export async function displayQuickQuery(ctx: ExtensionContext, cliServer: CodeQL
       return;
     }
 
-    const queriesDir = await getQuickQueriesDir(ctx);
-
-    // We need this folder in workspace folders so the language server
-    // knows how to find its qlpack.yml
-    if (workspace.workspaceFolders === undefined
-      || !workspace.workspaceFolders.some(folder => folder.uri.fsPath === queriesDir)) {
-      workspace.updateWorkspaceFolders(
-        (workspace.workspaceFolders || []).length,
-        0,
-        { uri: Uri.file(queriesDir), name: "Quick Queries" }
-      );
+    // We need to have a multi-root workspace to make quick query work
+    // at all. Changing the workspace from single-root to multi-root
+    // causes a restart of the whole extension host environment, so we
+    // basically can't do anything that survives that restart.
+    //
+    // So if we are currently in a single-root workspace (of which the
+    // only reliable signal appears to be `workspace.workspaceFile`
+    // being undefined) just let the user know that they're in for a
+    // restart.
+    if (workspace.workspaceFile === undefined) {
+      const makeMultiRoot = await helpers.showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
+      if (makeMultiRoot) {
+        updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
+      }
+      return;
     }
 
+    const index = workspaceFolders.findIndex(folder => folder.name === QUICK_QUERY_WORKSPACE_FOLDER_NAME);
+    if (index === -1)
+      updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
+    else
+      updateQuickQueryDir(queriesDir, index, 1);
+
     // We're going to infer which qlpack to use from the current database
     const dbItem = await databaseUI.getDatabaseItem();
     if (dbItem === undefined) {
@@ -103,19 +100,7 @@ export async function displayQuickQuery(ctx: ExtensionContext, cliServer: CodeQL
     }
 
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
-    const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'))
-
-    if (dbschemes.length < 1) {
-      throw new Error(`Can't find dbscheme for current database in ${datasetFolder}`);
-    }
-
-    dbschemes.sort();
-    const dbscheme = dbschemes[0];
-    if (dbschemes.length > 1) {
-      Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
-    }
-
-    const qlpack = await getQlPackFor(cliServer, dbscheme);
+    const { qlpack, dbscheme } = await helpers.resolveDatasetFolder(cliServer, datasetFolder);
     const quickQueryQlpackYaml: any = {
       name: "quick-query",
       version: "1.0.0",

extensions/ql-vscode/src/result-keys.ts

@@ -38,12 +38,12 @@ export function getResult(sarif: sarif.Log, key: Result): sarif.Result | undefin
  * Looks up a specific path in a result set.
  */
 export function getPath(sarif: sarif.Log, key: Path): sarif.ThreadFlow | undefined {
-  let result = getResult(sarif, key);
+  const result = getResult(sarif, key);
   if (result === undefined) return undefined;
   let index = -1;
   if (result.codeFlows === undefined) return undefined;
-  for (let codeFlows of result.codeFlows) {
-    for (let threadFlow of codeFlows.threadFlows) {
+  for (const codeFlows of result.codeFlows) {
+    for (const threadFlow of codeFlows.threadFlows) {
       ++index;
       if (index == key.pathIndex)
         return threadFlow;
@@ -56,7 +56,7 @@ export function getPath(sarif: sarif.Log, key: Path): sarif.ThreadFlow | undefin
  * Looks up a specific path node in a result set.
  */
 export function getPathNode(sarif: sarif.Log, key: PathNode): sarif.Location | undefined {
-  let path = getPath(sarif, key);
+  const path = getPath(sarif, key);
   if (path === undefined) return undefined;
   return path.locations[key.pathNodeIndex];
 }
@@ -85,7 +85,7 @@ export function equalsNotUndefined(key1: PathNode | undefined, key2: PathNode |
  */
 export function getAllPaths(result: sarif.Result): sarif.ThreadFlow[] {
   if (result.codeFlows === undefined) return [];
-  let paths = [];
+  const paths = [];
   for (const codeFlow of result.codeFlows) {
     for (const threadFlow of codeFlow.threadFlows) {
       paths.push(threadFlow);

extensions/ql-vscode/src/run-queries.ts

@@ -4,7 +4,10 @@ import * as path from 'path';
 import * as tmp from 'tmp';
 import { promisify } from 'util';
 import * as vscode from 'vscode';
+import { ErrorCodes, ResponseError } from 'vscode-languageclient';
+
 import * as cli from './cli';
+import * as config from './config';
 import { DatabaseItem, getUpgradesDirectories } from './databases';
 import * as helpers from './helpers';
 import { DatabaseInfo, QueryMetadata, ResultsPaths } from './interface-types';
@@ -54,6 +57,7 @@ export class QueryInfo {
     public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
     public readonly quickEvalPosition?: messages.Position,
     public readonly metadata?: QueryMetadata,
+    public readonly templates?: messages.TemplateDefinitions,
   ) {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
@@ -72,45 +76,55 @@ export class QueryInfo {
   ): Promise<messages.EvaluationResult> {
     let result: messages.EvaluationResult | null = null;
 
-    const callbackId = qs.registerCallback(res => { result = res });
+    const callbackId = qs.registerCallback(res => { result = res; });
 
     const queryToRun: messages.QueryToRun = {
       resultsPath: this.resultsPaths.resultsPath,
       qlo: vscode.Uri.file(this.compiledQueryPath).toString(),
       allowUnknownTemplates: true,
+      templateValues: this.templates,
       id: callbackId,
       timeoutSecs: qs.config.timeoutSecs,
-    }
+    };
     const dataset: messages.Dataset = {
       dbDir: this.dataset.fsPath,
       workingSet: 'default'
-    }
+    };
     const params: messages.EvaluateQueriesParams = {
       db: dataset,
       evaluateId: callbackId,
       queries: [queryToRun],
       stopOnError: false,
       useSequenceHint: false
-    }
+    };
     try {
       await helpers.withProgress({
         location: vscode.ProgressLocation.Notification,
         title: "Running Query",
         cancellable: true,
       }, (progress, token) => {
-        return qs.sendRequest(messages.runQueries, params, token, progress)
+        return qs.sendRequest(messages.runQueries, params, token, progress);
       });
     } finally {
       qs.unRegisterCallback(callbackId);
     }
-    return result || { evaluationTime: 0, message: "No result from server", queryId: -1, runId: callbackId, resultType: messages.QueryResultType.OTHER_ERROR };
+    return result || {
+      evaluationTime: 0,
+      message: "No result from server",
+      queryId: -1,
+      runId: callbackId,
+      resultType: messages.QueryResultType.OTHER_ERROR
+    };
   }
 
   async compile(
     qs: qsClient.QueryServerClient,
   ): Promise<messages.CompilationMessage[]> {
-    let compiled: messages.CheckQueryResult;
+    let compiled: messages.CheckQueryResult | undefined;
     try {
+      const target = this.quickEvalPosition ? {
+        quickEval: { quickEvalPos: this.quickEvalPosition }
+      } : { query: {} };
       const params: messages.CompileQueryParams = {
         compilationOptions: {
           computeNoLocationUrls: true,
@@ -126,10 +140,9 @@ export class QueryInfo {
         },
         queryToCheck: this.program,
         resultPath: this.compiledQueryPath,
-        target: !!this.quickEvalPosition ? { quickEval: { quickEvalPos: this.quickEvalPosition } } : { query: {} }
+        target,
       };
 
-
       compiled = await helpers.withProgress({
         location: vscode.ProgressLocation.Notification,
         title: "Compiling Query",
@@ -140,20 +153,26 @@ export class QueryInfo {
     } finally {
       qs.logger.log(" - - - COMPILATION DONE - - - ");
     }
-
-    return (compiled.messages || []).filter(msg => msg.severity == 0);
+    return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
   }
 
   /**
-   * Holds if this query should produce interpreted results.
+   * Holds if this query can in principle produce interpreted results.
    */
-  async hasInterpretedResults(): Promise<boolean> {
+  async canHaveInterpretedResults(): Promise<boolean> {
     const hasMetadataFile = await this.dbItem.hasMetadataFile();
     if (!hasMetadataFile) {
       logger.log("Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.");
     }
     return hasMetadataFile;
   }
+
+  /**
+   * Holds if this query actually has produced interpreted results.
+   */
+  async hasInterpretedResults(): Promise<boolean> {
+    return fs.pathExists(this.resultsPaths.interpretedResultsPath);
+  }
 }
 
 export interface QueryWithResults {
@@ -161,10 +180,13 @@ export interface QueryWithResults {
   readonly result: messages.EvaluationResult;
   readonly database: DatabaseInfo;
   readonly options: QueryHistoryItemOptions;
+  readonly logFileLocation?: string;
+  readonly dispose: () => void;
 }
 
-export async function clearCacheInDatabase(qs: qsClient.QueryServerClient, dbItem: DatabaseItem):
-  Promise<messages.ClearCacheResult> {
+export async function clearCacheInDatabase(
+  qs: qsClient.QueryServerClient, dbItem: DatabaseItem
+): Promise<messages.ClearCacheResult> {
   if (dbItem.contents === undefined) {
     throw new Error('Can\'t clear the cache in an invalid database.');
   }
@@ -224,8 +246,10 @@ async function getSelectedPosition(editor: vscode.TextEditor): Promise<messages.
   // Convert from 0-based to 1-based line and column numbers.
   return {
     fileName: await convertToQlPath(editor.document.fileName),
-    line: pos.line + 1, column: pos.character + 1,
-    endLine: posEnd.line + 1, endColumn: posEnd.character + 1
+    line: pos.line + 1,
+    column: pos.character + 1,
+    endLine: posEnd.line + 1,
+    endColumn: posEnd.character + 1
   };
 }
 
@@ -244,9 +268,9 @@ async function checkDbschemeCompatibility(
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
     const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
-    async function hash(filename: string): Promise<string> {
+    const hash = async function(filename: string): Promise<string> {
       return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
-    }
+    };
 
     // At this point, we have learned about three dbschemes:
 
@@ -279,19 +303,39 @@ async function checkDbschemeCompatibility(
   }
 }
 
-/** Prompts the user to save `document` if it has unsaved changes. */
-async function promptUserToSaveChanges(document: vscode.TextDocument) {
+/**
+ * Prompts the user to save `document` if it has unsaved changes.
+ * Returns true if we should save changes.
+ */
+async function promptUserToSaveChanges(document: vscode.TextDocument): Promise<boolean> {
   if (document.isDirty) {
-    // TODO: add 'always save' button which records preference in configuration
-    if (await helpers.showBinaryChoiceDialog('Query file has unsaved changes. Save now?')) {
-      await document.save();
+    if (config.AUTOSAVE_SETTING.getValue()) {
+      return true;
+    }
+    else {
+      const yesItem = { title: 'Yes', isCloseAffordance: false };
+      const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
+      const noItem = { title: 'No', isCloseAffordance: true };
+      const message = 'Query file has unsaved changes. Save now?';
+      const chosenItem = await vscode.window.showInformationMessage(message, { modal: true }, yesItem, alwaysItem, noItem);
+
+      if (chosenItem === alwaysItem) {
+        await config.AUTOSAVE_SETTING.updateValue(true, vscode.ConfigurationTarget.Workspace);
+        return true;
+      }
+
+      if (chosenItem === yesItem) {
+        return true;
+      }
     }
   }
+  return false;
 }
 
 type SelectedQuery = {
-  queryPath: string,
-  quickEvalPosition?: messages.Position
+  queryPath: string;
+  quickEvalPosition?: messages.Position;
+  quickEvalText?: string;
 };
 
 /**
@@ -304,7 +348,7 @@ type SelectedQuery = {
  * @param selectedResourceUri The selected resource when the command was run.
  * @param quickEval Whether the command being run is `Quick Evaluation`.
 */
-async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefined, quickEval: boolean): Promise<SelectedQuery> {
+export async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefined, quickEval: boolean): Promise<SelectedQuery> {
   const editor = vscode.window.activeTextEditor;
 
   // Choose which QL file to use.
@@ -325,16 +369,30 @@ async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefine
   if (queryUri.scheme !== 'file') {
     throw new Error('Can only run queries that are on disk.');
   }
-  const queryPath = queryUri.fsPath;
+  const queryPath = queryUri.fsPath || '';
+
+  if (quickEval) {
+    if (!(queryPath.endsWith('.ql') || queryPath.endsWith('.qll'))) {
+      throw new Error('The selected resource is not a CodeQL file; It should have the extension ".ql" or ".qll".');
+    }
+  }
+  else {
+    if (!(queryPath.endsWith('.ql'))) {
+      throw new Error('The selected resource is not a CodeQL query file; It should have the extension ".ql".');
+    }
+  }
 
   // Whether we chose the file from the active editor or from a context menu,
   // if the same file is open with unsaved changes in the active editor,
   // then prompt the user to save it first.
   if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
-    await promptUserToSaveChanges(editor.document);
+    if (await promptUserToSaveChanges(editor.document)) {
+      editor.document.save();
+    }
   }
 
   let quickEvalPosition: messages.Position | undefined = undefined;
+  let quickEvalText: string | undefined = undefined;
   if (quickEval) {
     if (editor == undefined) {
       throw new Error('Can\'t run quick evaluation without an active editor.');
@@ -345,9 +403,10 @@ async function determineSelectedQuery(selectedResourceUri: vscode.Uri | undefine
       throw new Error('The selected resource for quick evaluation should match the active editor.');
     }
     quickEvalPosition = await getSelectedPosition(editor);
+    quickEvalText = editor.document.getText(editor.selection);
   }
 
-  return { queryPath, quickEvalPosition };
+  return { queryPath, quickEvalPosition, quickEvalText };
 }
 
 export async function compileAndRunQueryAgainstDatabase(
@@ -355,7 +414,8 @@ export async function compileAndRunQueryAgainstDatabase(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
   quickEval: boolean,
-  selectedQueryUri: vscode.Uri | undefined
+  selectedQueryUri: vscode.Uri | undefined,
+  templates?: messages.TemplateDefinitions,
 ): Promise<QueryWithResults> {
 
   if (!db.contents || !db.contents.dbSchemeUri) {
@@ -363,12 +423,14 @@ export async function compileAndRunQueryAgainstDatabase(
   }
 
   // Determine which query to run, based on the selection and the active editor.
-  const { queryPath, quickEvalPosition } = await determineSelectedQuery(selectedQueryUri, quickEval);
+  const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, quickEval);
 
   // If this is quick query, store the query text
   const historyItemOptions: QueryHistoryItemOptions = {};
-  if (isQuickQueryPath(queryPath)) {
-    historyItemOptions.queryText = await fs.readFile(queryPath, 'utf8');
+  historyItemOptions.queryText = await fs.readFile(queryPath, 'utf8');
+  historyItemOptions.isQuickQuery === isQuickQueryPath(queryPath);
+  if (quickEval) {
+    historyItemOptions.queryText = quickEvalText;
   }
 
   // Get the workspace folder paths.
@@ -408,13 +470,27 @@ export async function compileAndRunQueryAgainstDatabase(
     logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
   }
 
-  const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata);
+  const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
   await checkDbschemeCompatibility(cliServer, qs, query);
 
-  const errors = await query.compile(qs);
+  let errors;
+  try {
+    errors = await query.compile(qs);
+  } catch (e) {
+    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+    } else {
+      throw e;
+    }
+  }
 
   if (errors.length == 0) {
     const result = await query.run(qs);
+    if (result.resultType !== messages.QueryResultType.SUCCESS) {
+      const message = result.message || 'Failed to run query';
+      logger.log(message);
+      helpers.showAndLogErrorMessage(message);
+    }
     return {
       query,
       result,
@@ -422,7 +498,11 @@ export async function compileAndRunQueryAgainstDatabase(
         name: db.name,
         databaseUri: db.databaseUri.toString(true)
       },
-      options: historyItemOptions
+      options: historyItemOptions,
+      logFileLocation: result.logFileLocation,
+      dispose: () => {
+        qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+      }
     };
   } else {
     // Error dialogs are limited in size and scrollability,
@@ -431,7 +511,7 @@ export async function compileAndRunQueryAgainstDatabase(
     // However we don't show quick eval errors there so we need to display them anyway.
     qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
 
-    let formattedMessages: string[] = [];
+    const formattedMessages: string[] = [];
 
     for (const error of errors) {
       const message = error.message || "[no error message available]";
@@ -448,20 +528,32 @@ export async function compileAndRunQueryAgainstDatabase(
         " and choose CodeQL Query Server from the dropdown.");
     }
 
-    return {
-      query,
-      result: {
-        evaluationTime: 0,
-        resultType: messages.QueryResultType.OTHER_ERROR,
-        queryId: -1,
-        runId: -1,
-        message: "Query had compilation errors"
-      },
-      database: {
-        name: db.name,
-        databaseUri: db.databaseUri.toString(true)
-      },
-      options: historyItemOptions,
-    };
+    return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
   }
 }
+
+function createSyntheticResult(
+  query: QueryInfo,
+  db: DatabaseItem,
+  historyItemOptions: QueryHistoryItemOptions,
+  message: string,
+  resultType: number
+): QueryWithResults {
+
+  return {
+    query,
+    result: {
+      evaluationTime: 0,
+      resultType: resultType,
+      queryId: -1,
+      runId: -1,
+      message
+    },
+    database: {
+      name: db.name,
+      databaseUri: db.databaseUri.toString(true)
+    },
+    options: historyItemOptions,
+    dispose: () => { /**/ },
+  };
+}

extensions/ql-vscode/src/sarif-utils.ts

@@ -1,10 +1,10 @@
-import * as Sarif from "sarif"
-import * as path from "path"
+import * as Sarif from "sarif";
+import * as path from "path";
 import { LocationStyle, ResolvableLocationValue } from "semmle-bqrs";
 
 export interface SarifLink {
-  dest: number
-  text: string
+  dest: number;
+  text: string;
 }
 
 
@@ -15,7 +15,7 @@ type ParsedSarifLocation =
   // doesn't really need to see. We ensure that `userVisibleFile` will not contain
   // that, and is appropriate for display in the UI.
   & { userVisibleFile: string }
-  | { t: 'NoLocation', hint: string };
+  | { t: 'NoLocation'; hint: string };
 
 export type SarifMessageComponent = string | SarifLink
 
@@ -27,7 +27,7 @@ export function unescapeSarifText(message: string): string {
 }
 
 export function parseSarifPlainTextMessage(message: string): SarifMessageComponent[] {
-  let results: SarifMessageComponent[] = [];
+  const results: SarifMessageComponent[] = [];
 
   // We want something like "[linkText](4)", except that "[" and "]" may be escaped. The lookbehind asserts
   // that the initial [ is not escaped. Then we parse a link text with "[" and "]" escaped. Then we parse the numerical target.

extensions/ql-vscode/src/test-adapter.ts

@@ -0,0 +1,229 @@
+import * as path from 'path';
+import * as vscode from 'vscode';
+import {
+  TestAdapter,
+  TestLoadStartedEvent,
+  TestLoadFinishedEvent,
+  TestRunStartedEvent,
+  TestRunFinishedEvent,
+  TestSuiteEvent,
+  TestEvent,
+  TestSuiteInfo,
+  TestInfo,
+  TestHub
+} from 'vscode-test-adapter-api';
+import { TestAdapterRegistrar } from 'vscode-test-adapter-util';
+import { QLTestFile, QLTestNode, QLTestDirectory, QLTestDiscovery } from './qltest-discovery';
+import { Event, EventEmitter, CancellationTokenSource, CancellationToken } from 'vscode';
+import { DisposableObject } from 'semmle-vscode-utils';
+import { QLPackDiscovery } from './qlpack-discovery';
+import { CodeQLCliServer } from './cli';
+import { getOnDiskWorkspaceFolders } from './helpers';
+import { testLogger } from './logging';
+
+/**
+ * Get the full path of the `.expected` file for the specified QL test.
+ * @param testPath The full path to the test file.
+ */
+export function getExpectedFile(testPath: string): string {
+  return getTestOutputFile(testPath, '.expected');
+}
+
+/**
+ * Get the full path of the `.actual` file for the specified QL test.
+ * @param testPath The full path to the test file.
+ */
+export function getActualFile(testPath: string): string {
+  return getTestOutputFile(testPath, '.actual');
+}
+
+/**
+ * Get the directory containing the specified QL test.
+ * @param testPath The full path to the test file.
+ */
+export function getTestDirectory(testPath: string): string {
+  return path.dirname(testPath);
+}
+
+/**
+ * Gets the the full path to a particular output file of the specified QL test.
+ * @param testPath The full path to the QL test.
+ * @param extension The file extension of the output file.
+ */
+function getTestOutputFile(testPath: string, extension: string): string {
+  return changeExtension(testPath, extension);
+}
+
+/**
+ * A factory service that creates `QLTestAdapter` objects for workspace folders on demand.
+ */
+export class QLTestAdapterFactory extends DisposableObject {
+  constructor(testHub: TestHub, cliServer: CodeQLCliServer) {
+    super();
+
+    // this will register a QLTestAdapter for each WorkspaceFolder
+    this.push(new TestAdapterRegistrar(
+      testHub,
+      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer)
+    ));
+  }
+}
+
+/**
+ * Change the file extension of the specified path.
+ * @param p The original file path.
+ * @param ext The new extension, including the `.`.
+ */
+function changeExtension(p: string, ext: string): string {
+  return p.substr(0, p.length - path.extname(p).length) + ext;
+}
+
+/**
+ * Test adapter for QL tests.
+ */
+export class QLTestAdapter extends DisposableObject implements TestAdapter {
+  private readonly qlPackDiscovery: QLPackDiscovery;
+  private readonly qlTestDiscovery: QLTestDiscovery;
+  private readonly _tests = this.push(
+    new EventEmitter<TestLoadStartedEvent | TestLoadFinishedEvent>());
+  private readonly _testStates = this.push(
+    new EventEmitter<TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent | TestEvent>());
+  private readonly _autorun = this.push(new EventEmitter<void>());
+  private runningTask?: vscode.CancellationTokenSource = undefined;
+
+  constructor(
+    public readonly workspaceFolder: vscode.WorkspaceFolder,
+    private readonly cliServer: CodeQLCliServer
+  ) {
+    super();
+
+    this.qlPackDiscovery = this.push(new QLPackDiscovery(workspaceFolder, cliServer));
+    this.qlTestDiscovery = this.push(new QLTestDiscovery(this.qlPackDiscovery, cliServer));
+
+    this.push(this.qlTestDiscovery.onDidChangeTests(this.discoverTests, this));
+  }
+
+  public get tests(): Event<TestLoadStartedEvent | TestLoadFinishedEvent> {
+    return this._tests.event;
+  }
+
+  public get testStates(): Event<TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent | TestEvent> {
+    return this._testStates.event;
+  }
+
+  public get autorun(): Event<void> | undefined {
+    return this._autorun.event;
+  }
+
+  private static createTestOrSuiteInfos(testNodes: readonly QLTestNode[]): (TestSuiteInfo | TestInfo)[] {
+    return testNodes.map((childNode) => {
+      return QLTestAdapter.createTestOrSuiteInfo(childNode);
+    });
+  }
+
+  private static createTestOrSuiteInfo(testNode: QLTestNode): TestSuiteInfo | TestInfo {
+    if (testNode instanceof QLTestFile) {
+      return QLTestAdapter.createTestInfo(testNode);
+    } else if (testNode instanceof QLTestDirectory) {
+      return QLTestAdapter.createTestSuiteInfo(testNode, testNode.name);
+    } else {
+      throw new Error('Unexpected test type.');
+    }
+  }
+
+  private static createTestInfo(testFile: QLTestFile): TestInfo {
+    return {
+      type: 'test',
+      id: testFile.path,
+      label: testFile.name,
+      tooltip: testFile.path,
+      file: testFile.path
+    };
+  }
+
+  private static createTestSuiteInfo(testDirectory: QLTestDirectory, label: string): TestSuiteInfo {
+    return {
+      type: 'suite',
+      id: testDirectory.path,
+      label: label,
+      children: QLTestAdapter.createTestOrSuiteInfos(testDirectory.children),
+      tooltip: testDirectory.path
+    };
+  }
+
+  public async load(): Promise<void> {
+    this.discoverTests();
+  }
+
+  private discoverTests(): void {
+    this._tests.fire({ type: 'started' } as TestLoadStartedEvent);
+
+    const testDirectories = this.qlTestDiscovery.testDirectories;
+    const children = testDirectories.map(
+      testDirectory => QLTestAdapter.createTestSuiteInfo(testDirectory, testDirectory.name)
+    );
+    const testSuite: TestSuiteInfo = {
+      type: 'suite',
+      label: 'CodeQL',
+      id: '.',
+      children
+    };
+
+    this._tests.fire({
+      type: 'finished',
+      suite: children.length > 0 ? testSuite : undefined
+    } as TestLoadFinishedEvent);
+  }
+
+  public async run(tests: string[]): Promise<void> {
+    if (this.runningTask !== undefined) {
+      throw new Error('Tests already running.');
+    }
+
+    testLogger.outputChannel.clear();
+    testLogger.outputChannel.show(true);
+
+    this.runningTask = this.track(new CancellationTokenSource());
+
+    this._testStates.fire({ type: 'started', tests: tests } as TestRunStartedEvent);
+
+    try {
+      await this.runTests(tests, this.runningTask.token);
+    }
+    catch (e) {
+      /**/
+    }
+    this._testStates.fire({ type: 'finished' } as TestRunFinishedEvent);
+    this.clearTask();
+  }
+
+  private clearTask(): void {
+    if (this.runningTask !== undefined) {
+      const runningTask = this.runningTask;
+      this.runningTask = undefined;
+      this.disposeAndStopTracking(runningTask);
+    }
+  }
+
+  public cancel(): void {
+    if (this.runningTask !== undefined) {
+      testLogger.log('Cancelling test run...');
+      this.runningTask.cancel();
+      this.clearTask();
+    }
+  }
+
+  private async runTests(tests: string[], cancellationToken: CancellationToken): Promise<void> {
+    const workspacePaths = await getOnDiskWorkspaceFolders();
+    for await (const event of await this.cliServer.runTests(tests, workspacePaths, {
+      cancellationToken: cancellationToken,
+      logger: testLogger
+    })) {
+      this._testStates.fire({
+        type: 'test',
+        state: event.pass ? 'passed' : 'failed',
+        test: event.test
+      });
+    }
+  }
+}

extensions/ql-vscode/src/test-tree-node.ts

@@ -0,0 +1,9 @@
+import { TestSuiteInfo, TestInfo } from 'vscode-test-adapter-api';
+
+/**
+ * Tree view node for a test, suite, or collection. This object is passed as the argument to the
+ * command handler of a context menu item for a tree view item. 
+ */
+export interface TestTreeNode {
+  readonly info: TestSuiteInfo | TestInfo;
+}

extensions/ql-vscode/src/test-ui.ts

@@ -0,0 +1,90 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { Uri, TextDocumentShowOptions, commands, window } from 'vscode';
+import { TestTreeNode } from './test-tree-node';
+import { DisposableObject, UIService } from 'semmle-vscode-utils';
+import { TestHub, TestController, TestAdapter, TestRunStartedEvent, TestRunFinishedEvent, TestEvent, TestSuiteEvent } from 'vscode-test-adapter-api';
+import { QLTestAdapter, getExpectedFile, getActualFile } from './test-adapter';
+import { logger } from './logging';
+
+type VSCodeTestEvent = TestRunStartedEvent | TestRunFinishedEvent | TestSuiteEvent | TestEvent;
+
+/**
+ * Test event listener. Currently unused, but left in to keep the plumbing hooked up for future use.
+ */
+class QLTestListener extends DisposableObject {
+  constructor(adapter: TestAdapter) {
+    super();
+
+    this.push(adapter.testStates(this.onTestStatesEvent, this));
+  }
+
+  private onTestStatesEvent(_e: VSCodeTestEvent): void {
+    /**/
+  }
+}
+
+/**
+ * Service that implements all UI and commands for QL tests.
+ */
+export class TestUIService extends UIService implements TestController {
+  private readonly listeners: Map<TestAdapter, QLTestListener> = new Map();
+
+  constructor(private readonly testHub: TestHub) {
+    super();
+
+    logger.log('Registering CodeQL test panel commands.');
+    this.registerCommand('codeQLTests.showOutputDifferences', this.showOutputDifferences);
+    this.registerCommand('codeQLTests.acceptOutput', this.acceptOutput);
+
+    testHub.registerTestController(this);
+  }
+
+  public dispose(): void {
+    this.testHub.unregisterTestController(this);
+
+    super.dispose();
+  }
+
+  public registerTestAdapter(adapter: TestAdapter): void {
+    this.listeners.set(adapter, new QLTestListener(adapter));
+  }
+
+  public unregisterTestAdapter(adapter: TestAdapter): void {
+    if (adapter instanceof QLTestAdapter) {
+      this.listeners.delete(adapter);
+    }
+  }
+
+  private async acceptOutput(node: TestTreeNode): Promise<void> {
+    const testId = node.info.id;
+    const stat = await fs.lstat(testId);
+    if (stat.isFile()) {
+      const expectedPath = getExpectedFile(testId);
+      const actualPath = getActualFile(testId);
+      await fs.copy(actualPath, expectedPath, { overwrite: true });
+    }
+  }
+
+  private async showOutputDifferences(node: TestTreeNode): Promise<void> {
+    const testId = node.info.id;
+    const stat = await fs.lstat(testId);
+    if (stat.isFile()) {
+      const expectedPath = getExpectedFile(testId);
+      const expectedUri = Uri.file(expectedPath);
+      const actualPath = getActualFile(testId);
+      const options: TextDocumentShowOptions = {
+        preserveFocus: true,
+        preview: true
+      };
+      if (await fs.pathExists(actualPath)) {
+        const actualUri = Uri.file(actualPath);
+        await commands.executeCommand<void>('vscode.diff', expectedUri, actualUri,
+          `Expected vs. Actual for ${path.basename(testId)}`, options);
+      }
+      else {
+        await window.showTextDocument(expectedUri, options);
+      }
+    }
+  }
+}

extensions/ql-vscode/src/upgrades.ts

@@ -19,8 +19,9 @@ const MAX_UPGRADE_MESSAGE_LINES = 10;
  * Reports errors to both the user and the console.
  * @returns the `UpgradeParams` needed to start the upgrade, if the upgrade is possible and was confirmed by the user, or `undefined` otherwise.
  */
-async function checkAndConfirmDatabaseUpgrade(qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]):
-  Promise<messages.UpgradeParams | undefined> {
+async function checkAndConfirmDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]
+): Promise<messages.UpgradeParams | undefined> {
   if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
     helpers.showAndLogErrorMessage("Database is invalid, and cannot be upgraded.");
     return;
@@ -79,8 +80,8 @@ async function checkAndConfirmDatabaseUpgrade(qs: qsClient.QueryServerClient, db
 
   const showLogItem: vscode.MessageItem = { title: 'No, Show Changes', isCloseAffordance: true };
   const yesItem = { title: 'Yes', isCloseAffordance: false };
-  const noItem = { title: 'No', isCloseAffordance: true }
-  let dialogOptions: vscode.MessageItem[] = [yesItem, noItem];
+  const noItem = { title: 'No', isCloseAffordance: true };
+  const dialogOptions: vscode.MessageItem[] = [yesItem, noItem];
 
   let messageLines = descriptionMessage.split('\n');
   if (messageLines.length > MAX_UPGRADE_MESSAGE_LINES) {
@@ -110,8 +111,9 @@ async function checkAndConfirmDatabaseUpgrade(qs: qsClient.QueryServerClient, db
  * First performs a dry-run and prompts the user to confirm the upgrade.
  * Reports errors during compilation and evaluation of upgrades to the user.
  */
-export async function upgradeDatabase(qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]):
-  Promise<messages.RunUpgradeResult | undefined> {
+export async function upgradeDatabase(
+  qs: qsClient.QueryServerClient, db: DatabaseItem, targetDbScheme: vscode.Uri, upgradesDirectories: vscode.Uri[]
+): Promise<messages.RunUpgradeResult | undefined> {
   const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories);
 
   if (upgradeParams === undefined) {
@@ -127,7 +129,7 @@ export async function upgradeDatabase(qs: qsClient.QueryServerClient, db: Databa
     return;
   }
   finally {
-    qs.logger.log('Done compiling database upgrade.')
+    qs.logger.log('Done compiling database upgrade.');
   }
 
   if (compileUpgradeResult.compiledUpgrades === undefined) {
@@ -146,12 +148,13 @@ export async function upgradeDatabase(qs: qsClient.QueryServerClient, db: Databa
     return;
   }
   finally {
-    qs.logger.log('Done running database upgrade.')
+    qs.logger.log('Done running database upgrade.');
   }
 }
 
-async function checkDatabaseUpgrade(qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams):
-  Promise<messages.CheckUpgradeResult> {
+async function checkDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams
+): Promise<messages.CheckUpgradeResult> {
   return helpers.withProgress({
     location: vscode.ProgressLocation.Notification,
     title: "Checking for database upgrades",
@@ -159,12 +162,13 @@ async function checkDatabaseUpgrade(qs: qsClient.QueryServerClient, upgradeParam
   }, (progress, token) => qs.sendRequest(messages.checkUpgrade, upgradeParams, token, progress));
 }
 
-async function compileDatabaseUpgrade(qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams):
-  Promise<messages.CompileUpgradeResult> {
+async function compileDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, upgradeParams: messages.UpgradeParams
+): Promise<messages.CompileUpgradeResult> {
   const params: messages.CompileUpgradeParams = {
     upgrade: upgradeParams,
     upgradeTempDir: upgradesTmpDir.name
-  }
+  };
 
   return helpers.withProgress({
     location: vscode.ProgressLocation.Notification,
@@ -173,8 +177,9 @@ async function compileDatabaseUpgrade(qs: qsClient.QueryServerClient, upgradePar
   }, (progress, token) => qs.sendRequest(messages.compileUpgrade, params, token, progress));
 }
 
-async function runDatabaseUpgrade(qs: qsClient.QueryServerClient, db: DatabaseItem, upgrades: messages.CompiledUpgrades):
-  Promise<messages.RunUpgradeResult> {
+async function runDatabaseUpgrade(
+  qs: qsClient.QueryServerClient, db: DatabaseItem, upgrades: messages.CompiledUpgrades
+): Promise<messages.RunUpgradeResult> {
 
   if (db.contents === undefined || db.contents.datasetUri === undefined) {
     throw new Error('Can\'t upgrade an invalid database.');

extensions/ql-vscode/src/view/.eslintrc.js

@@ -0,0 +1,13 @@
+module.exports = {
+  env: {
+    browser: true
+  },
+  extends: [
+    "plugin:react/recommended"
+  ],
+  settings: {
+    react: {
+      version: 'detect'
+    }
+  }
+}

extensions/ql-vscode/src/view/alert-table.tsx

@@ -5,9 +5,10 @@ import * as Keys from '../result-keys';
 import { LocationStyle } from 'semmle-bqrs';
 import * as octicons from './octicons';
 import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection } from './result-table-utils';
-import { PathTableResultSet, onNavigation, NavigationEvent, vscode } from './results';
+import { onNavigation, NavigationEvent, vscode } from './results';
 import { parseSarifPlainTextMessage, parseSarifLocation } from '../sarif-utils';
 import { InterpretedResultsSortColumn, SortDirection, InterpretedResultsSortState } from '../interface-types';
+import { PathTableResultSet } from '../interface-utils';
 
 export type PathTableProps = ResultTableProps & { resultSet: PathTableResultSet };
 export interface PathTableState {
@@ -69,6 +70,14 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
     });
   }
 
+  renderNoResults(): JSX.Element {
+    if (this.props.nonemptyRawResults) {
+      return <span>No Alerts. See <a href='#' onClick={this.props.showRawResults}>raw results</a>.</span>;
+    } else {
+      return <span>No Alerts</span>;
+    }
+  }
+
   render(): JSX.Element {
     const { databaseUri, resultSet } = this.props;
 
@@ -84,7 +93,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
 
     function renderRelatedLocations(msg: string, relatedLocations: Sarif.Location[]): JSX.Element[] {
       const relatedLocationsById: { [k: string]: Sarif.Location } = {};
-      for (let loc of relatedLocations) {
+      for (const loc of relatedLocations) {
         relatedLocationsById[loc.id!] = loc;
       }
 
@@ -116,7 +125,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
           ...previousState,
           selectedPathNode: pathNodeKey
         }));
-      }
+      };
     };
 
     function renderSarifLocationWithText(text: string | undefined, loc: Sarif.Location, pathNodeKey: Keys.PathNode | undefined): JSX.Element | undefined {
@@ -156,18 +165,19 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
       return (e) => this.toggle(e, indices);
     };
 
-    const noResults = <span>No Results</span>; // TODO: Maybe make this look nicer
+    if (resultSet.sarif.runs.length === 0 ||
+      resultSet.sarif.runs[0].results === undefined ||
+      resultSet.sarif.runs[0].results.length === 0) {
+      return this.renderNoResults();
+    }
 
     let expansionIndex = 0;
 
-    if (resultSet.sarif.runs.length === 0) return noResults;
-    if (resultSet.sarif.runs[0].results === undefined) return noResults;
-
     resultSet.sarif.runs[0].results.forEach((result, resultIndex) => {
       const text = result.message.text || '[no text]';
       const msg: JSX.Element[] =
         result.relatedLocations === undefined ?
-          [<span>{text}</span>] :
+          [<span key="0">{text}</span>] :
           renderRelatedLocations(text, result.relatedLocations);
 
       const currentResultExpanded = this.state.expanded[expansionIndex];
@@ -239,7 +249,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
               const additionalMsg = step.location !== undefined ?
                 renderSarifLocation(step.location, pathNodeKey) :
                 '';
-              let isSelected = Keys.equalsNotUndefined(this.state.selectedPathNode, pathNodeKey);
+              const isSelected = Keys.equalsNotUndefined(this.state.selectedPathNode, pathNodeKey);
               const stepIndex = pathNodeIndex + 1; // Convert to 1-based
               const zebraIndex = resultIndex + stepIndex;
               rows.push(
@@ -271,23 +281,23 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
 
   private handleNavigationEvent(event: NavigationEvent) {
     this.setState(prevState => {
-      let { selectedPathNode } = prevState;
+      const { selectedPathNode } = prevState;
       if (selectedPathNode === undefined) return prevState;
 
-      let path = Keys.getPath(this.props.resultSet.sarif, selectedPathNode);
+      const path = Keys.getPath(this.props.resultSet.sarif, selectedPathNode);
       if (path === undefined) return prevState;
 
-      let nextIndex = selectedPathNode.pathNodeIndex + event.direction;
+      const nextIndex = selectedPathNode.pathNodeIndex + event.direction;
       if (nextIndex < 0 || nextIndex >= path.locations.length) return prevState;
 
-      let sarifLoc = path.locations[nextIndex].location;
+      const sarifLoc = path.locations[nextIndex].location;
       if (sarifLoc === undefined) return prevState;
 
-      let loc = parseSarifLocation(sarifLoc, this.props.resultSet.sourceLocationPrefix);
+      const loc = parseSarifLocation(sarifLoc, this.props.resultSet.sourceLocationPrefix);
       if (loc.t === 'NoLocation') return prevState;
 
       jumpToLocation(loc, this.props.databaseUri);
-      let newSelection = { ...selectedPathNode, pathNodeIndex: nextIndex };
+      const newSelection = { ...selectedPathNode, pathNodeIndex: nextIndex };
       return { ...prevState, selectedPathNode: newSelection };
     });
   }

extensions/ql-vscode/src/view/event-handler-list.ts

@@ -4,22 +4,22 @@ export type EventHandler<T> = (event: T) => void;
  * A set of listeners for events of type `T`.
  */
 export class EventHandlers<T> {
-    private handlers: EventHandler<T>[] = [];
+  private handlers: EventHandler<T>[] = [];
 
-    public addListener(handler: EventHandler<T>) {
-        this.handlers.push(handler);
-    }
+  public addListener(handler: EventHandler<T>) {
+    this.handlers.push(handler);
+  }
 
-    public removeListener(handler: EventHandler<T>) {
-        let index = this.handlers.indexOf(handler);
-        if (index !== -1) {
-            this.handlers.splice(index, 1);
-        }
+  public removeListener(handler: EventHandler<T>) {
+    const index = this.handlers.indexOf(handler);
+    if (index !== -1) {
+      this.handlers.splice(index, 1);
     }
+  }
 
-    public fire(event: T) {
-        for (let handler of this.handlers) {
-            handler(event);
-        }
+  public fire(event: T) {
+    for (const handler of this.handlers) {
+      handler(event);
     }
+  }
 }

extensions/ql-vscode/src/view/raw-results-table.tsx

@@ -1,11 +1,14 @@
 import * as React from "react";
 import { renderLocation, ResultTableProps, zebraStripe, className, nextSortDirection } from "./result-table-utils";
-import { RawTableResultSet, ResultValue, vscode } from "./results";
+import { vscode } from "./results";
+import { ResultValue } from "../adapt";
 import { SortDirection, RAW_RESULTS_LIMIT, RawResultsSortState } from "../interface-types";
+import { RawTableResultSet } from "../interface-utils";
 
 export type RawTableProps = ResultTableProps & {
-  resultSet: RawTableResultSet,
+  resultSet: RawTableResultSet;
   sortState?: RawResultsSortState;
+  offset: number;
 };
 
 export class RawTable extends React.Component<RawTableProps, {}> {
@@ -27,7 +30,7 @@ export class RawTable extends React.Component<RawTableProps, {}> {
       <tr key={rowIndex} {...zebraStripe(rowIndex)}>
         {
           [
-            <td key={-1}>{rowIndex + 1}</td>,
+            <td key={-1}>{rowIndex + 1 + this.props.offset}</td>,
             ...row.map((value, columnIndex) =>
               <td key={columnIndex}>
                 {
@@ -67,7 +70,7 @@ export class RawTable extends React.Component<RawTableProps, {}> {
     </table>;
   }
 
-  private toggleSortStateForColumn(index: number) {
+  private toggleSortStateForColumn(index: number): void {
     const sortState = this.props.sortState;
     const prevDirection = sortState && sortState.columnIndex === index ? sortState.sortDirection : undefined;
     const nextDirection = nextSortDirection(prevDirection);
@@ -88,7 +91,7 @@ export class RawTable extends React.Component<RawTableProps, {}> {
  */
 function renderTupleValue(v: ResultValue, databaseUri: string): JSX.Element {
   if (typeof v === 'string') {
-    return <span>{v}</span>
+    return <span>{v}</span>;
   }
   else if ('uri' in v) {
     return <a href={v.uri}>{v.uri}</a>;

extensions/ql-vscode/src/view/result-table-utils.tsx

@@ -1,15 +1,29 @@
 import * as React from 'react';
 import { LocationValue, ResolvableLocationValue, tryGetResolvableLocation } from 'semmle-bqrs';
 import { RawResultsSortState, QueryMetadata, SortDirection } from '../interface-types';
-import { ResultSet, vscode } from './results';
+import { vscode } from './results';
 import { assertNever } from '../helpers-pure';
+import { ResultSet } from '../interface-utils';
 
 export interface ResultTableProps {
   resultSet: ResultSet;
   databaseUri: string;
-  metadata?: QueryMetadata
+  metadata?: QueryMetadata;
   resultsPath: string | undefined;
   sortState?: RawResultsSortState;
+  offset: number;
+
+  /**
+   * Holds if there are any raw results. When that is the case, we
+   * want to direct users to pay attention to raw results if
+   * interpreted results are empty.
+   */
+  nonemptyRawResults: boolean;
+
+  /**
+   * Callback to show raw results.
+   */
+  showRawResults: () => void;
 }
 
 export const className = 'vscode-codeql__result-table';
@@ -34,7 +48,7 @@ export function jumpToLocationHandler(
   };
 }
 
-export function jumpToLocation(loc: ResolvableLocationValue, databaseUri: string) {
+export function jumpToLocation(loc: ResolvableLocationValue, databaseUri: string): void {
   vscode.postMessage({
     t: 'viewSourceFile',
     loc,
@@ -66,7 +80,7 @@ export function renderLocation(loc: LocationValue | undefined, label: string | u
       return <span title={title}>{displayLabel}</span>;
     }
   }
-  return <span />
+  return <span />;
 }
 
 /**
@@ -83,7 +97,7 @@ export function zebraStripe(index: number, ...otherClasses: string[]): { classNa
 export function selectableZebraStripe(isSelected: boolean, index: number, ...otherClasses: string[]): { className: string } {
   return isSelected
     ? { className: [selectedRowClassName, ...otherClasses].join(' ') }
-    : zebraStripe(index, ...otherClasses)
+    : zebraStripe(index, ...otherClasses);
 }
 
 /**

extensions/ql-vscode/src/view/result-tables.tsx

@@ -1,18 +1,21 @@
 import * as React from 'react';
-import { DatabaseInfo, Interpretation, RawResultsSortState, QueryMetadata, ResultsPaths, InterpretedResultsSortState } from '../interface-types';
+import { DatabaseInfo, Interpretation, RawResultsSortState, QueryMetadata, ResultsPaths, InterpretedResultsSortState, RAW_RESULTS_PAGE_SIZE } from '../interface-types';
 import { PathTable } from './alert-table';
 import { RawTable } from './raw-results-table';
 import { ResultTableProps, tableSelectionHeaderClassName, toggleDiagnosticsClassName, alertExtrasClassName } from './result-table-utils';
-import { ResultSet, vscode } from './results';
+import { vscode } from './results';
+import { ParsedResultSets, ExtensionParsedResultSets } from '../adapt';
+import { ResultSet, ALERTS_TABLE_NAME, SELECT_TABLE_NAME, getDefaultResultSet } from '../interface-utils';
 
 /**
  * Properties for the `ResultTables` component.
  */
 export interface ResultTablesProps {
+  parsedResultSets: ParsedResultSets;
   rawResultSets: readonly ResultSet[];
   interpretation: Interpretation | undefined;
   database: DatabaseInfo;
-  metadata?: QueryMetadata
+  metadata?: QueryMetadata;
   resultsPath: string;
   origResultsPaths: ResultsPaths;
   sortStates: Map<string, RawResultsSortState>;
@@ -25,10 +28,9 @@ export interface ResultTablesProps {
  */
 interface ResultTablesState {
   selectedTable: string; // name of selected result set
+  selectedPage: string; // stringified selected page
 }
 
-const ALERTS_TABLE_NAME = 'alerts';
-const SELECT_TABLE_NAME = '#select';
 const UPDATING_RESULTS_TEXT_CLASS_NAME = "vscode-codeql__result-tables-updating-text";
 
 function getResultCount(resultSet: ResultSet): number {
@@ -75,23 +77,66 @@ export class ResultTables
     return resultSets;
   }
 
+  private getResultSetNames(resultSets: ResultSet[]): string[] {
+    if (this.props.parsedResultSets.t === 'ExtensionParsed') {
+      return this.props.parsedResultSets.resultSetNames.concat([ALERTS_TABLE_NAME]);
+    }
+    else {
+      return resultSets.map(resultSet => resultSet.schema.name);
+    }
+  }
+
+  /**
+   * Holds if we have a result set obtained from the extension that came
+   * from the ExtensionParsed branch of ParsedResultSets. This is evidence
+   * that the user has the experimental flag turned on that allows extension-side
+   * bqrs parsing.
+   */
+  paginationAllowed(): boolean {
+    return this.props.parsedResultSets.t === 'ExtensionParsed';
+  }
+
+  /**
+   * Holds if we actually should show pagination interface right now. This is
+   * still false for the time being when we're viewing alerts.
+   */
+  paginationEnabled(): boolean {
+    return this.paginationAllowed() &&
+      this.props.parsedResultSets.selectedTable !== ALERTS_TABLE_NAME &&
+      this.state.selectedTable !== ALERTS_TABLE_NAME;
+  }
+
   constructor(props: ResultTablesProps) {
     super(props);
 
-    this.state = {
-      // Get the result set that should be displayed by default
-      selectedTable: ResultTables.getDefaultResultSet(this.getResultSets())
-    };
-  }
+    const selectedTable = props.parsedResultSets.selectedTable || getDefaultResultSet(this.getResultSets());
 
-  private static getDefaultResultSet(resultSets: readonly ResultSet[]): string {
-    const resultSetNames = resultSets.map(resultSet => resultSet.schema.name)
-    // Choose first available result set from the array
-    return [ALERTS_TABLE_NAME, SELECT_TABLE_NAME, resultSets[0].schema.name].filter(resultSetName => resultSetNames.includes(resultSetName))[0];
+    let selectedPage: string;
+    switch (props.parsedResultSets.t) {
+      case 'ExtensionParsed':
+        selectedPage = (props.parsedResultSets.pageNumber + 1) + '';
+        break;
+      case 'WebviewParsed':
+        selectedPage = '';
+        break;
+    }
+
+    this.state = { selectedTable, selectedPage };
   }
 
   private onTableSelectionChange = (event: React.ChangeEvent<HTMLSelectElement>): void => {
-    this.setState({ selectedTable: event.target.value });
+    const selectedTable = event.target.value;
+    const fetchPageFromExtension = this.paginationAllowed() && selectedTable !== ALERTS_TABLE_NAME;
+
+    if (fetchPageFromExtension) {
+      vscode.postMessage({
+        t: 'changePage',
+        pageNumber: 0,
+        selectedTable
+      });
+    }
+    else
+      this.setState({ selectedTable });
   }
 
   private alertTableExtras(): JSX.Element | undefined {
@@ -115,26 +160,84 @@ export class ResultTables
 
     return <div className={alertExtrasClassName}>
       {displayProblemsAsAlertsToggle}
-    </div>
+    </div>;
+  }
+
+  getOffset(): number {
+    const { parsedResultSets } = this.props;
+    switch (parsedResultSets.t) {
+      case 'ExtensionParsed':
+        return parsedResultSets.pageNumber * RAW_RESULTS_PAGE_SIZE;
+      case 'WebviewParsed':
+        return 0;
+    }
+  }
+
+  renderPageButtons(resultSets: ExtensionParsedResultSets): JSX.Element {
+    const selectedTable = this.state.selectedTable;
+    const onChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+      this.setState({ selectedPage: e.target.value });
+    };
+    const choosePage = (input: string) => {
+      const pageNumber = parseInt(input);
+      if (pageNumber !== undefined && !isNaN(pageNumber)) {
+        const actualPageNumber = Math.max(0, Math.min(pageNumber - 1, resultSets.numPages - 1));
+        vscode.postMessage({
+          t: 'changePage',
+          pageNumber: actualPageNumber,
+          selectedTable,
+        });
+      }
+    };
+
+    const prevPage = (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => {
+      vscode.postMessage({
+        t: 'changePage',
+        pageNumber: Math.max(resultSets.pageNumber - 1, 0),
+        selectedTable,
+      });
+    };
+    const nextPage = (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => {
+      vscode.postMessage({
+        t: 'changePage',
+        pageNumber: Math.min(resultSets.pageNumber + 1, resultSets.numPages - 1),
+        selectedTable,
+      });
+    };
+    return <span>
+      <button onClick={prevPage} >&lt;</button>
+      <input value={this.state.selectedPage} onChange={onChange}
+        onBlur={e => choosePage(e.target.value)}
+        onKeyDown={e => { if (e.keyCode === 13) choosePage((e.target as HTMLInputElement).value); }}
+      />
+      <button value=">" onClick={nextPage} >&gt;</button>
+    </span>;
+  }
+
+  renderButtons(): JSX.Element {
+    if (this.props.parsedResultSets.t === 'ExtensionParsed' && this.paginationEnabled())
+      return this.renderPageButtons(this.props.parsedResultSets);
+    else
+      return <span />;
   }
 
   render(): React.ReactNode {
     const { selectedTable } = this.state;
     const resultSets = this.getResultSets();
+    const resultSetNames = this.getResultSetNames(resultSets);
 
     const resultSet = resultSets.find(resultSet => resultSet.schema.name == selectedTable);
+    const nonemptyRawResults = resultSets.some(resultSet => resultSet.t == 'RawResultSet' && resultSet.rows.length > 0);
     const numberOfResults = resultSet && renderResultCountString(resultSet);
 
+    const resultSetOptions =
+      resultSetNames.map(name => <option key={name} value={name}>{name}</option>);
+
     return <div>
+      {this.renderButtons()}
       <div className={tableSelectionHeaderClassName}>
         <select value={selectedTable} onChange={this.onTableSelectionChange}>
-          {
-            resultSets.map(resultSet =>
-              <option key={resultSet.schema.name} value={resultSet.schema.name}>
-                {resultSet.schema.name}
-              </option>
-            )
-          }
+          {resultSetOptions}
         </select>
         {numberOfResults}
         {selectedTable === ALERTS_TABLE_NAME ? this.alertTableExtras() : undefined}
@@ -149,7 +252,10 @@ export class ResultTables
         <ResultTable key={resultSet.schema.name} resultSet={resultSet}
           databaseUri={this.props.database.databaseUri}
           resultsPath={this.props.resultsPath}
-          sortState={this.props.sortStates.get(resultSet.schema.name)} />
+          sortState={this.props.sortStates.get(resultSet.schema.name)}
+          nonemptyRawResults={nonemptyRawResults}
+          showRawResults={() => { this.setState({ selectedTable: SELECT_TABLE_NAME }); }}
+          offset={this.getOffset()} />
       }
     </div>;
   }

extensions/ql-vscode/src/view/results.tsx

@@ -1,11 +1,13 @@
 import * as React from 'react';
 import * as Rdom from 'react-dom';
 import * as bqrs from 'semmle-bqrs';
-import { ElementBase, LocationValue, PrimitiveColumnValue, PrimitiveTypeKind, ResultSetSchema, tryGetResolvableLocation } from 'semmle-bqrs';
+import { ElementBase, PrimitiveColumnValue, PrimitiveTypeKind, tryGetResolvableLocation } from 'semmle-bqrs';
 import { assertNever } from '../helpers-pure';
 import { DatabaseInfo, FromResultsViewMsg, Interpretation, IntoResultsViewMsg, SortedResultSetInfo, RawResultsSortState, NavigatePathMsg, QueryMetadata, ResultsPaths } from '../interface-types';
-import { ResultTables } from './result-tables';
 import { EventHandlers as EventHandlerList } from './event-handler-list';
+import { ResultTables } from './result-tables';
+import { ResultValue, ResultRow, ParsedResultSets } from '../adapt';
+import { ResultSet } from '../interface-utils';
 
 /**
  * results.tsx
@@ -23,31 +25,6 @@ interface VsCodeApi {
 declare const acquireVsCodeApi: () => VsCodeApi;
 export const vscode = acquireVsCodeApi();
 
-export interface ResultElement {
-  label: string,
-  location?: LocationValue
-}
-
-export interface ResultUri {
-  uri: string;
-}
-
-export type ResultValue = ResultElement | ResultUri | string;
-
-export type ResultRow = ResultValue[];
-
-export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
-export type PathTableResultSet = { t: 'SarifResultSet', readonly schema: ResultSetSchema, name: string } & Interpretation;
-
-export type ResultSet =
-  | RawTableResultSet
-  | PathTableResultSet;
-
-export interface RawResultSet {
-  readonly schema: ResultSetSchema;
-  readonly rows: readonly ResultRow[];
-}
-
 async function* getChunkIterator(response: Response): AsyncIterableIterator<Uint8Array> {
   if (!response.ok) {
     throw new Error(`Failed to load results: (${response.status}) ${response.statusText}`);
@@ -58,13 +35,11 @@ async function* getChunkIterator(response: Response): AsyncIterableIterator<Uint
     if (done) {
       return;
     }
-    yield value;
+    yield value!;
   }
 }
 
-function translatePrimitiveValue(value: PrimitiveColumnValue, type: PrimitiveTypeKind):
-  ResultValue {
-
+function translatePrimitiveValue(value: PrimitiveColumnValue, type: PrimitiveTypeKind): ResultValue {
   switch (type) {
     case 'i':
     case 'f':
@@ -126,6 +101,7 @@ async function parseResultSets(response: Response): Promise<readonly ResultSet[]
 }
 
 interface ResultsInfo {
+  parsedResultSets: ParsedResultSets;
   resultsPath: string;
   origResultsPaths: ResultsPaths;
   database: DatabaseInfo;
@@ -135,7 +111,7 @@ interface ResultsInfo {
    * See {@link SetStateMsg.shouldKeepOldResultsWhileRendering}.
    */
   shouldKeepOldResultsWhileRendering: boolean;
-  metadata?: QueryMetadata
+  metadata?: QueryMetadata;
 }
 
 interface Results {
@@ -187,6 +163,7 @@ class App extends React.Component<{}, ResultsViewState> {
       case 'setState':
         this.updateStateWithNewResultsInfo({
           resultsPath: msg.resultsPath,
+          parsedResultSets: msg.parsedResultSets,
           origResultsPaths: msg.origResultsPaths,
           sortedResultsMap: new Map(Object.entries(msg.sortedResultsMap)),
           database: msg.database,
@@ -212,7 +189,7 @@ class App extends React.Component<{}, ResultsViewState> {
 
   private updateStateWithNewResultsInfo(resultsInfo: ResultsInfo): void {
     this.setState(prevState => {
-      const stateWithDisplayedResults = (displayedResults: ResultsState) => ({
+      const stateWithDisplayedResults = (displayedResults: ResultsState): ResultsViewState => ({
         displayedResults,
         isExpectingResultsUpdate: prevState.isExpectingResultsUpdate,
         nextResultsInfo: resultsInfo
@@ -238,6 +215,16 @@ class App extends React.Component<{}, ResultsViewState> {
     });
   }
 
+  private async getResultSets(resultsInfo: ResultsInfo): Promise<readonly ResultSet[]> {
+    const parsedResultSets = resultsInfo.parsedResultSets;
+    switch (parsedResultSets.t) {
+      case 'WebviewParsed': return await this.fetchResultSets(resultsInfo);
+      case 'ExtensionParsed': {
+        return [{ t: 'RawResultSet', ...parsedResultSets.resultSet }];
+      }
+    }
+  }
+
   private async loadResults(): Promise<void> {
     const resultsInfo = this.state.nextResultsInfo;
     if (resultsInfo === null) {
@@ -245,10 +232,11 @@ class App extends React.Component<{}, ResultsViewState> {
     }
 
     let results: Results | null = null;
-    let statusText: string = '';
+    let statusText = '';
     try {
+      const resultSets = await this.getResultSets(resultsInfo);
       results = {
-        resultSets: await this.getResultSets(resultsInfo),
+        resultSets,
         database: resultsInfo.database,
         sortStates: this.getSortStates(resultsInfo)
       };
@@ -277,11 +265,15 @@ class App extends React.Component<{}, ResultsViewState> {
         },
         nextResultsInfo: null,
         isExpectingResultsUpdate: false
-      }
+      };
     });
   }
 
-  private async getResultSets(resultsInfo: ResultsInfo): Promise<readonly ResultSet[]> {
+  /**
+   * This is deprecated, because it calls `fetch`. We are moving
+   * towards doing all bqrs parsing in the extension.
+   */
+  private async fetchResultSets(resultsInfo: ResultsInfo): Promise<readonly ResultSet[]> {
     const unsortedResponse = await fetch(resultsInfo.resultsPath);
     const unsortedResultSets = await parseResultSets(unsortedResponse);
     return Promise.all(unsortedResultSets.map(async unsortedResultSet => {
@@ -304,10 +296,13 @@ class App extends React.Component<{}, ResultsViewState> {
       [key, sortedResultSetInfo.sortState]));
   }
 
-  render() {
+  render(): JSX.Element {
     const displayedResults = this.state.displayedResults;
     if (displayedResults.results !== null && displayedResults.resultsInfo !== null) {
-      return <ResultTables rawResultSets={displayedResults.results.resultSets}
+      const parsedResultSets = displayedResults.resultsInfo.parsedResultSets;
+      return <ResultTables
+        parsedResultSets={parsedResultSets}
+        rawResultSets={displayedResults.results.resultSets}
         interpretation={displayedResults.resultsInfo ? displayedResults.resultsInfo.interpretation : undefined}
         database={displayedResults.results.database}
         origResultsPaths={displayedResults.resultsInfo.origResultsPaths}
@@ -322,12 +317,12 @@ class App extends React.Component<{}, ResultsViewState> {
     }
   }
 
-  componentDidMount() {
+  componentDidMount(): void {
     this.vscodeMessageHandler = evt => this.handleMessage(evt.data as IntoResultsViewMsg);
     window.addEventListener('message', this.vscodeMessageHandler);
   }
 
-  componentWillUnmount() {
+  componentWillUnmount(): void {
     if (this.vscodeMessageHandler) {
       window.removeEventListener('message', this.vscodeMessageHandler);
     }
@@ -341,4 +336,4 @@ Rdom.render(
   document.getElementById('root')
 );
 
-vscode.postMessage({ t: "resultViewLoaded" })
+vscode.postMessage({ t: "resultViewLoaded" });

extensions/ql-vscode/src/view/resultsView.css

@@ -75,7 +75,7 @@
 }
 
 .vscode-codeql__result-table-location-link {
-  font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo, Courier, monospace;
+  text-decoration: none;
 }
 
 select {

extensions/ql-vscode/src/vscode-tests/.eslintrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  env: {
+    mocha: true
+  }
+}

extensions/ql-vscode/src/vscode-tests/index-template.ts

@@ -58,4 +58,4 @@ export function runTestsInDirectory(testsRoot: string): Promise<void> {
       }
     });
   });
-}
+}

extensions/ql-vscode/src/vscode-tests/minimal-workspace/activation.test.ts

@@ -1,6 +1,12 @@
 import * as assert from 'assert';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+import 'mocha';
 import * as path from 'path';
 import * as vscode from 'vscode';
+import * as determiningSelectedQueryTest from './determining-selected-query-test';
+
+chai.use(chaiAsPromised);
 
 describe('launching with a minimal workspace', async () => {
   const ext = vscode.extensions.getExtension('GitHub.vscode-codeql');
@@ -23,4 +29,6 @@ describe('launching with a minimal workspace', async () => {
       assert(ext!.isActive);
     }, 1000);
   });
-});
+});
+
+determiningSelectedQueryTest.run();

extensions/ql-vscode/src/vscode-tests/minimal-workspace/determining-selected-query-test.ts

@@ -0,0 +1,44 @@
+import { expect } from 'chai';
+import * as path from 'path';
+import * as vscode from 'vscode';
+import { Uri } from 'vscode';
+import { determineSelectedQuery } from '../../run-queries';
+
+async function showQlDocument(name: string): Promise<vscode.TextDocument> {
+  const folderPath = vscode.workspace.workspaceFolders![0].uri.fsPath;
+  const documentPath = path.resolve(folderPath, name);
+  const document = await vscode.workspace.openTextDocument(documentPath);
+  await vscode.window.showTextDocument(document!);
+  return document;
+}
+
+export function run() {
+  describe('Determining selected query', async () => {
+    it('should allow ql files to be queried', async () => {
+      const q = await determineSelectedQuery(Uri.parse('file:///tmp/queryname.ql'), false);
+      expect(q.queryPath).to.equal(path.join('/', 'tmp', 'queryname.ql'));
+      expect(q.quickEvalPosition).to.equal(undefined);
+    });
+
+    it('should allow ql files to be quick-evaled', async () => {
+      const doc = await showQlDocument('query.ql');
+      const q = await determineSelectedQuery(doc.uri, true);
+      expect(q.queryPath).to.satisfy((p: string) => p.endsWith(path.join('ql-vscode', 'test', 'data', 'query.ql')));
+    });
+
+    it('should allow qll files to be quick-evaled', async () => {
+      const doc = await showQlDocument('library.qll');
+      const q = await determineSelectedQuery(doc.uri, true);
+      expect(q.queryPath).to.satisfy((p: string) => p.endsWith(path.join('ql-vscode', 'test', 'data', 'library.qll')));
+    });
+
+    it('should reject non-ql files when running a query', async () => {
+      await expect(determineSelectedQuery(Uri.parse('file:///tmp/queryname.txt'), false)).to.be.rejectedWith(Error, 'The selected resource is not a CodeQL query file');
+      await expect(determineSelectedQuery(Uri.parse('file:///tmp/queryname.qll'), false)).to.be.rejectedWith(Error, 'The selected resource is not a CodeQL query file');
+    });
+
+    it('should reject non-ql[l] files when running a quick eval', async () => {
+      await expect(determineSelectedQuery(Uri.parse('file:///tmp/queryname.txt'), true)).to.be.rejectedWith(Error, 'The selected resource is not a CodeQL file');
+    });
+  });
+}

extensions/ql-vscode/src/vscode-tests/minimal-workspace/index.ts

@@ -1,4 +1,4 @@
 import { runTestsInDirectory } from '../index-template';
 export function run(): Promise<void> {
   return runTestsInDirectory(__dirname);
-}
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/activation.test.ts

@@ -10,4 +10,4 @@ describe('launching with no specified workspace', () => {
   it('should not activate the extension at first', () => {
     assert(ext!.isActive === false);
   });
-});
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/archive-filesystem-provider.test.ts

@@ -1,8 +1,10 @@
-import { expect } from "chai";
-import * as path from "path";
-import { ArchiveFileSystemProvider, decodeSourceArchiveUri, encodeSourceArchiveUri, ZipFileReference } from "../../archive-filesystem-provider";
+import { expect } from 'chai';
+import * as path from 'path';
 
-describe("archive filesystem provider", () => {
+import { encodeSourceArchiveUri, ArchiveFileSystemProvider, decodeSourceArchiveUri, ZipFileReference } from '../../archive-filesystem-provider';
+import { FileType, FileSystemError } from 'vscode';
+
+describe('archive-filesystem-provider', () => {
   it("reads empty file correctly", async () => {
     const archiveProvider = new ArchiveFileSystemProvider();
     const uri = encodeSourceArchiveUri({
@@ -12,10 +14,102 @@ describe("archive filesystem provider", () => {
     const data = await archiveProvider.readFile(uri);
     expect(data.length).to.equal(0);
   });
+
+  it("read non-empty file correctly", async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/textFile.txt"
+    });
+    const data = await archiveProvider.readFile(uri);
+    expect(Buffer.from(data).toString('utf8')).to.be.equal('I am a text\n');
+  });
+
+  it("read a directory", async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1"
+    });
+    const files = await archiveProvider.readDirectory(uri);
+    expect(files).to.be.deep.equal([
+      ['folder2', FileType.Directory],
+      ['textFile.txt', FileType.File],
+      ['textFile2.txt', FileType.File],
+    ]);
+  });
+
+  it('should handle a missing directory', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/not-here"
+    });
+    try {
+      await archiveProvider.readDirectory(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it('should handle a missing file', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/not-here"
+    });
+    try {
+      await archiveProvider.readFile(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it('should handle reading a file as a directory', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/textFile.txt"
+    });
+    try {
+      await archiveProvider.readDirectory(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it('should handle reading a directory as a file', async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/folder2"
+    });
+    try {
+      await archiveProvider.readFile(uri);
+      throw new Error('Failed');
+    } catch (e) {
+      expect(e).to.be.instanceOf(FileSystemError);
+    }
+  });
+
+  it("read a nested directory", async () => {
+    const archiveProvider = new ArchiveFileSystemProvider();
+    const uri = encodeSourceArchiveUri({
+      sourceArchiveZipPath: path.resolve(__dirname, "data/archive-filesystem-provider-test/zip_with_folder.zip"),
+      pathWithinSourceArchive: "folder1/folder2"
+    });
+    const files = await archiveProvider.readDirectory(uri);
+    expect(files).to.be.deep.equal([
+      ['textFile3.txt', FileType.File],
+    ]);
+  });
 });
 
 describe('source archive uri encoding', function() {
-  const testCases: { name: string, input: ZipFileReference }[] = [
+  const testCases: { name: string; input: ZipFileReference }[] = [
     {
       name: 'mixed case and unicode',
       input: { sourceArchiveZipPath: "/I-\u2665-codeql.zip", pathWithinSourceArchive: "/foo/bar" }

extensions/ql-vscode/src/vscode-tests/no-workspace/cli-version.test.ts

@@ -1,50 +0,0 @@
-import { expect } from "chai";
-import "mocha";
-import { tryParseVersionString } from "../../cli-version";
-
-describe("Version parsing", () => {
-  it("should accept version without prerelease and build metadata", () => {
-    const v = tryParseVersionString("3.2.4")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.be.undefined;
-    expect(v.buildMetadata).to.be.undefined;
-  });
-
-  it("should accept v at the beginning of the version", () => {
-    const v = tryParseVersionString("v3.2.4")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.be.undefined;
-    expect(v.buildMetadata).to.be.undefined;
-  });
-
-  it("should accept version with prerelease", () => {
-    const v = tryParseVersionString("v3.2.4-alpha.0")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.equal("alpha.0");
-    expect(v.buildMetadata).to.be.undefined;
-  });
-
-  it("should accept version with prerelease and build metadata", () => {
-    const v = tryParseVersionString("v3.2.4-alpha.0+abcdef0")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.equal("alpha.0");
-    expect(v.buildMetadata).to.equal("abcdef0");
-  });
-
-  it("should accept version with build metadata", () => {
-    const v = tryParseVersionString("v3.2.4+abcdef0")!;
-    expect(v.majorVersion).to.equal(3);
-    expect(v.minorVersion).to.equal(2);
-    expect(v.patchVersion).to.equal(4);
-    expect(v.prereleaseVersion).to.be.undefined;
-    expect(v.buildMetadata).to.equal("abcdef0");
-  });
-});

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -0,0 +1,140 @@
+import "vscode-test";
+import "mocha";
+import * as chaiAsPromised from "chai-as-promised";
+import * as sinon from "sinon";
+// import * as sinonChai from 'sinon-chai';
+import * as path from "path";
+import * as fs from "fs-extra";
+import * as tmp from "tmp";
+import * as chai from "chai";
+import { window } from "vscode";
+
+import {
+  convertToDatabaseUrl,
+  looksLikeLgtmUrl,
+  findDirWithFile,
+} from "../../databaseFetcher";
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
+describe("databaseFetcher", () => {
+  describe("convertToDatabaseUrl", () => {
+    let quickPickSpy: sinon.SinonStub;
+    beforeEach(() => {
+      quickPickSpy = sinon.stub(window, "showQuickPick");
+    });
+
+    afterEach(() => {
+      (window.showQuickPick as sinon.SinonStub).restore();
+    });
+
+    it("should convert a project url to a database url", async () => {
+      quickPickSpy.returns("javascript" as any);
+      const lgtmUrl = "https://lgtm.com/projects/g/github/codeql";
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+
+      expect(dbUrl).to.equal(
+        "https://lgtm.com/api/v1.0/snapshots/1506465042581/javascript"
+      );
+      expect(quickPickSpy.firstCall.args[0]).to.contain("javascript");
+      expect(quickPickSpy.firstCall.args[0]).to.contain("python");
+    });
+
+    it("should convert a project url to a database url with extra path segments", async () => {
+      quickPickSpy.returns("python" as any);
+      const lgtmUrl =
+        "https://lgtm.com/projects/g/github/codeql/subpage/subpage2?query=xxx";
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+
+      expect(dbUrl).to.equal(
+        "https://lgtm.com/api/v1.0/snapshots/1506465042581/python"
+      );
+    });
+
+    it("should fail on a nonexistant prohect", async () => {
+      quickPickSpy.returns("javascript" as any);
+      const lgtmUrl = "https://lgtm.com/projects/g/github/hucairz";
+      expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
+    });
+  });
+
+  describe("looksLikeLgtmUrl", () => {
+    it("should handle invalid urls", () => {
+      expect(looksLikeLgtmUrl("")).to.be.false;
+      expect(looksLikeLgtmUrl("http://lgtm.com/projects/g/github/codeql")).to.be
+        .false;
+      expect(looksLikeLgtmUrl("https://ww.lgtm.com/projects/g/github/codeql"))
+        .to.be.false;
+      expect(looksLikeLgtmUrl("https://ww.lgtm.com/projects/g/github")).to.be
+        .false;
+    });
+
+    it("should handle valid urls", () => {
+      expect(looksLikeLgtmUrl("https://lgtm.com/projects/g/github/codeql")).to
+        .be.true;
+      expect(looksLikeLgtmUrl("https://www.lgtm.com/projects/g/github/codeql"))
+        .to.be.true;
+      expect(
+        looksLikeLgtmUrl("https://lgtm.com/projects/g/github/codeql/sub/pages")
+      ).to.be.true;
+      expect(
+        looksLikeLgtmUrl(
+          "https://lgtm.com/projects/g/github/codeql/sub/pages?query=string"
+        )
+      ).to.be.true;
+    });
+  });
+
+  describe("findDirWithFile", () => {
+    let dir: tmp.DirResult;
+    beforeEach(() => {
+      dir = tmp.dirSync({ unsafeCleanup: true });
+      createFile("a");
+      createFile("b");
+      createFile("c");
+
+      createDir("dir1");
+      createFile("dir1", "d");
+      createFile("dir1", "e");
+      createFile("dir1", "f");
+
+      createDir("dir2");
+      createFile("dir2", "g");
+      createFile("dir2", "h");
+      createFile("dir2", "i");
+
+      createDir("dir2", "dir3");
+      createFile("dir2", "dir3", "j");
+      createFile("dir2", "dir3", "k");
+      createFile("dir2", "dir3", "l");
+    });
+
+    it("should find files", async () => {
+      expect(await findDirWithFile(dir.name, "k")).to.equal(
+        path.join(dir.name, "dir2", "dir3")
+      );
+      expect(await findDirWithFile(dir.name, "h")).to.equal(
+        path.join(dir.name, "dir2")
+      );
+      expect(await findDirWithFile(dir.name, "z", "a")).to.equal(dir.name);
+      // there's some slight indeterminism when more than one name exists
+      // but in general, this will find files in the current directory before
+      // finding files in sub-dirs
+      expect(await findDirWithFile(dir.name, "k", "a")).to.equal(dir.name);
+    });
+
+
+    it("should not find files", async () => {
+      expect(await findDirWithFile(dir.name, "x", "y", "z")).to.be.undefined;
+    });
+
+
+    function createFile(...segments: string[]) {
+      fs.createFileSync(path.join(dir.name, ...segments));
+    }
+
+    function createDir(...segments: string[]) {
+      fs.mkdirSync(path.join(dir.name, ...segments));
+    }
+  });
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts

@@ -0,0 +1,47 @@
+import 'vscode-test';
+import 'mocha';
+import * as tmp from 'tmp';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { expect } from 'chai';
+import { Uri } from 'vscode';
+
+import { DatabaseUI } from '../../databases-ui';
+
+describe('databases-ui', () => {
+  describe('fixDbUri', () => {
+    const fixDbUri = (DatabaseUI.prototype as any).fixDbUri;
+    it('should choose current directory direcory normally', async () => {
+      const dir = tmp.dirSync().name;
+      const uri = await fixDbUri(Uri.file(dir));
+      expect(uri.toString()).to.eq(Uri.file(dir).toString());
+    });
+
+    it('should choose parent direcory when file is selected', async () => {
+      const file = tmp.fileSync().name;
+      const uri = await fixDbUri(Uri.file(file));
+      expect(uri.toString()).to.eq(Uri.file(path.dirname(file)).toString());
+    });
+
+    it('should choose parent direcory when db-* is selected', async () => {
+      const dir = tmp.dirSync().name;
+      const dbDir = path.join(dir, 'db-hucairz');
+      await fs.mkdirs(dbDir);
+
+      const uri = await fixDbUri(Uri.file(dbDir));
+      expect(uri.toString()).to.eq(Uri.file(dir).toString());
+    });
+
+    it('should choose parent\'s parent direcory when file selected is in db-*', async () => {
+      const dir = tmp.dirSync().name;
+      const dbDir = path.join(dir, 'db-hucairz');
+      const file = path.join(dbDir, 'nested');
+      await fs.mkdirs(dbDir);
+      await fs.createFile(file);
+
+      const uri = await fixDbUri(Uri.file(file));
+      expect(uri.toString()).to.eq(Uri.file(dir).toString());
+    });
+  });
+
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -1,52 +1,74 @@
-import { expect } from "chai";
+import * as chai from "chai";
+import * as path from "path";
 import * as fetch from "node-fetch";
+import "chai/register-should";
+import * as semver from "semver";
+import * as sinonChai from "sinon-chai";
+import * as sinon from "sinon";
+import * as pq from "proxyquire";
 import "mocha";
-import { Version } from "../../cli-version";
-import { GithubRelease, GithubReleaseAsset, ReleasesApiConsumer, versionCompare } from "../../distribution"
+
+import { GithubRelease, GithubReleaseAsset, ReleasesApiConsumer } from "../../distribution";
+
+const proxyquire = pq.noPreserveCache();
+chai.use(sinonChai);
+const expect = chai.expect;
 
 describe("Releases API consumer", () => {
   const owner = "someowner";
   const repo = "somerepo";
-  const sampleReleaseResponse: GithubRelease[] = [
-    {
-      "assets": [],
-      "created_at": "2019-09-01T00:00:00Z",
-      "id": 1,
-      "name": "",
-      "prerelease": false,
-      "tag_name": "v2.1.0"
-    },
-    {
-      "assets": [],
-      "created_at": "2019-08-10T00:00:00Z",
-      "id": 2,
-      "name": "",
-      "prerelease": false,
-      "tag_name": "v3.1.1"
-    },
-    {
-      "assets": [],
-      "created_at": "2019-09-05T00:00:00Z",
-      "id": 3,
-      "name": "",
-      "prerelease": false,
-      "tag_name": "v2.0.0"
-    },
-    {
-      "assets": [],
-      "created_at": "2019-08-11T00:00:00Z",
-      "id": 4,
-      "name": "",
-      "prerelease": true,
-      "tag_name": "v3.1.2-pre"
-    },
-  ];
-  const unconstrainedVersionConstraint = {
-    description: "*",
-    isVersionCompatible: () => true
-  };
+  const unconstrainedVersionRange = new semver.Range("*");
+
+  describe("picking the latest release", () => {
+    const sampleReleaseResponse: GithubRelease[] = [
+      {
+        "assets": [],
+        "created_at": "2019-09-01T00:00:00Z",
+        "id": 1,
+        "name": "",
+        "prerelease": false,
+        "tag_name": "v2.1.0"
+      },
+      {
+        "assets": [],
+        "created_at": "2019-08-10T00:00:00Z",
+        "id": 2,
+        "name": "",
+        "prerelease": false,
+        "tag_name": "v3.1.1"
+      },
+      {
+        "assets": [{
+          id: 1,
+          name: "exampleAsset.txt",
+          size: 1
+        }],
+        "created_at": "2019-09-05T00:00:00Z",
+        "id": 3,
+        "name": "",
+        "prerelease": false,
+        "tag_name": "v2.0.0"
+      },
+      {
+        "assets": [],
+        "created_at": "2019-08-11T00:00:00Z",
+        "id": 4,
+        "name": "",
+        "prerelease": true,
+        "tag_name": "v3.1.2-pre-1.1"
+      },
+      // Release ID 5 is older than release ID 4 but its version has a higher precedence, so release
+      // ID 5 should be picked over release ID 4.
+      {
+        "assets": [],
+        "created_at": "2019-08-09T00:00:00Z",
+        "id": 5,
+        "name": "",
+        "prerelease": true,
+        "tag_name": "v3.1.2-pre-2.0"
+      },
+    ];
 
-  it("picking latest release: is based on version", async () => {
     class MockReleasesApiConsumer extends ReleasesApiConsumer {
       protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
         if (apiPath === `/repos/${owner}/${repo}/releases`) {
@@ -56,45 +78,55 @@ describe("Releases API consumer", () => {
       }
     }
 
-    const consumer = new MockReleasesApiConsumer(owner, repo);
+    it("picked release has version with the highest precedence", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
 
-    const latestRelease = await consumer.getLatestRelease(unconstrainedVersionConstraint);
-    expect(latestRelease.id).to.equal(2);
-  });
-
-  it("picking latest release: obeys version constraints", async () => {
-    class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
-        if (apiPath === `/repos/${owner}/${repo}/releases`) {
-          return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
-        }
-        return Promise.reject(new Error(`Unknown API path: ${apiPath}`));
-      }
-    }
-
-    const consumer = new MockReleasesApiConsumer(owner, repo);
-
-    const latestRelease = await consumer.getLatestRelease({
-      description: "2.*.*",
-      isVersionCompatible: version => version.majorVersion === 2
+      const latestRelease = await consumer.getLatestRelease(unconstrainedVersionRange);
+      expect(latestRelease.id).to.equal(2);
     });
-    expect(latestRelease.id).to.equal(1);
-  });
 
-  it("picking latest release: includes prereleases when option set", async () => {
-    class MockReleasesApiConsumer extends ReleasesApiConsumer {
-      protected async makeApiCall(apiPath: string): Promise<fetch.Response> {
-        if (apiPath === `/repos/${owner}/${repo}/releases`) {
-          return Promise.resolve(new fetch.Response(JSON.stringify(sampleReleaseResponse)));
-        }
-        return Promise.reject(new Error(`Unknown API path: ${apiPath}`));
-      }
-    }
+    it("version of picked release is within the version range", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
 
-    const consumer = new MockReleasesApiConsumer(owner, repo);
+      const latestRelease = await consumer.getLatestRelease(new semver.Range("2.*.*"));
+      expect(latestRelease.id).to.equal(1);
+    });
 
-    const latestRelease = await consumer.getLatestRelease(unconstrainedVersionConstraint, true);
-    expect(latestRelease.id).to.equal(4);
+    it("fails if none of the releases are within the version range", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      await chai.expect(
+        consumer.getLatestRelease(new semver.Range("5.*.*"))
+      ).to.be.rejectedWith(Error);
+    });
+
+    it("picked release passes additional compatibility test if an additional compatibility test is specified", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      const latestRelease = await consumer.getLatestRelease(
+        new semver.Range("2.*.*"),
+        true,
+        release => release.assets.some(asset => asset.name === "exampleAsset.txt")
+      );
+      expect(latestRelease.id).to.equal(3);
+    });
+
+    it("fails if none of the releases pass the additional compatibility test", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      await chai.expect(consumer.getLatestRelease(
+        new semver.Range("2.*.*"),
+        true,
+        release => release.assets.some(asset => asset.name === "otherExampleAsset.txt")
+      )).to.be.rejectedWith(Error);
+    });
+
+    it("picked release is the most recent prerelease when includePrereleases is set", async () => {
+      const consumer = new MockReleasesApiConsumer(owner, repo);
+
+      const latestRelease = await consumer.getLatestRelease(unconstrainedVersionRange, true);
+      expect(latestRelease.id).to.equal(5);
+    });
   });
 
   it("gets correct assets for a release", async () => {
@@ -131,7 +163,7 @@ describe("Releases API consumer", () => {
 
     const consumer = new MockReleasesApiConsumer(owner, repo);
 
-    const assets = (await consumer.getLatestRelease(unconstrainedVersionConstraint)).assets;
+    const assets = (await consumer.getLatestRelease(unconstrainedVersionRange)).assets;
 
     expect(assets.length).to.equal(expectedAssets.length);
     expectedAssets.map((expectedAsset, index) => {
@@ -142,37 +174,145 @@ describe("Releases API consumer", () => {
   });
 });
 
-describe("Release version ordering", () => {
-  function createVersion(majorVersion: number, minorVersion: number, patchVersion: number, prereleaseVersion?: string, buildMetadata?: string): Version {
-    return {
-      buildMetadata,
-      majorVersion,
-      minorVersion,
-      patchVersion,
-      prereleaseVersion,
-      rawString: `${majorVersion}.${minorVersion}.${patchVersion}` +
-        prereleaseVersion ? `-${prereleaseVersion}` : "" +
-          buildMetadata ? `+${buildMetadata}` : ""
-    };
+describe('Launcher path', () => {
+  const pathToCmd = `abc${path.sep}codeql.cmd`;
+  const pathToExe = `abc${path.sep}codeql.exe`;
+
+  let sandbox: sinon.SinonSandbox;
+  let warnSpy: sinon.SinonSpy;
+  let errorSpy: sinon.SinonSpy;
+  let logSpy: sinon.SinonSpy;
+  let fsSpy: sinon.SinonSpy;
+  let platformSpy: sinon.SinonSpy;
+
+  let getExecutableFromDirectory: Function;
+
+  let launcherThatExists = '';
+
+  beforeEach(() => {
+    getExecutableFromDirectory = createModule().getExecutableFromDirectory;
+  });
+
+  beforeEach(() => {
+    sandbox.restore();
+  });
+
+  it('should not warn with proper launcher name', async () => {
+    launcherThatExists = 'codeql.exe';
+    const result = await getExecutableFromDirectory('abc');
+    expect(fsSpy).to.have.been.calledWith(pathToExe);
+
+    // correct launcher has been found, so alternate one not looked for
+    expect(fsSpy).not.to.have.been.calledWith(pathToCmd);
+
+    // no warning message
+    expect(warnSpy).not.to.have.been.calledWith(sinon.match.string);
+    // No log message
+    expect(logSpy).not.to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(pathToExe);
+  });
+
+  it('should warn when using a hard-coded deprecated launcher name', async () => {
+    launcherThatExists = 'codeql.cmd';
+    path.sep;
+    const result = await getExecutableFromDirectory('abc');
+    expect(fsSpy).to.have.been.calledWith(pathToExe);
+    expect(fsSpy).to.have.been.calledWith(pathToCmd);
+
+    // Should have opened a warning message
+    expect(warnSpy).to.have.been.calledWith(sinon.match.string);
+    // No log message
+    expect(logSpy).not.to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(pathToCmd);
+  });
+
+  it('should avoid warn when no launcher is found', async () => {
+    launcherThatExists = 'xxx';
+    const result = await getExecutableFromDirectory('abc', false);
+    expect(fsSpy).to.have.been.calledWith(pathToExe);
+    expect(fsSpy).to.have.been.calledWith(pathToCmd);
+
+    // no warning message
+    expect(warnSpy).not.to.have.been.calledWith(sinon.match.string);
+    // log message sent out
+    expect(logSpy).not.to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(undefined);
+  });
+
+  it('should warn when no launcher is found', async () => {
+    launcherThatExists = 'xxx';
+    const result = await getExecutableFromDirectory('abc', true);
+    expect(fsSpy).to.have.been.calledWith(pathToExe);
+    expect(fsSpy).to.have.been.calledWith(pathToCmd);
+
+    // no warning message
+    expect(warnSpy).not.to.have.been.calledWith(sinon.match.string);
+    // log message sent out
+    expect(logSpy).to.have.been.calledWith(sinon.match.string);
+    expect(result).to.equal(undefined);
+  });
+
+  it('should not warn when deprecated launcher is used, but no new launcher is available', async () => {
+    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    launcherThatExists = 'codeql.cmd';
+
+    const result = await manager.getCodeQlPathWithoutVersionCheck();
+    expect(result).to.equal(pathToCmd);
+
+    // no warning or error message
+    expect(warnSpy).to.have.callCount(0);
+    expect(errorSpy).to.have.callCount(0);
+  });
+
+  it('should warn when deprecated launcher is used, and new launcher is available', async () => {
+    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    launcherThatExists = ''; // pretend both launchers exist
+
+    const result = await manager.getCodeQlPathWithoutVersionCheck();
+    expect(result).to.equal(pathToCmd);
+
+    // has warning message
+    expect(warnSpy).to.have.callCount(1);
+    expect(errorSpy).to.have.callCount(0);
+  });
+
+  it('should warn when launcher path is incorrect', async () => {
+    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    launcherThatExists = 'xxx'; // pretend neither launcher exists
+
+    const result = await manager.getCodeQlPathWithoutVersionCheck();
+    expect(result).to.equal(undefined);
+
+    // no error message
+    expect(warnSpy).to.have.callCount(0);
+    expect(errorSpy).to.have.callCount(1);
+  });
+
+  function createModule() {
+    sandbox = sinon.createSandbox();
+    warnSpy = sandbox.spy();
+    errorSpy = sandbox.spy();
+    logSpy = sandbox.spy();
+    // pretend that only the .cmd file exists
+    fsSpy = sandbox.stub().callsFake(arg => arg.endsWith(launcherThatExists) ? true : false);
+    platformSpy = sandbox.stub().returns('win32');
+
+    return proxyquire('../../distribution', {
+      './helpers': {
+        showAndLogWarningMessage: warnSpy,
+        showAndLogErrorMessage: errorSpy
+      },
+      './logging': {
+        'logger': {
+          log: logSpy
+        }
+      },
+      'fs-extra': {
+        pathExists: fsSpy
+      },
+      os: {
+        platform: platformSpy
+      }
+    });
   }
-
-  it("major versions compare correctly", () => {
-    expect(versionCompare(createVersion(3, 0, 0), createVersion(2, 9, 9)) > 0).to.be.true;
-  });
-
-  it("minor versions compare correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 0), createVersion(2, 0, 9)) > 0).to.be.true;
-  });
-
-  it("patch versions compare correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 2), createVersion(2, 1, 1)) > 0).to.be.true;
-  });
-
-  it("prerelease versions compare correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 0, "alpha.2"), createVersion(2, 1, 0, "alpha.1")) > 0).to.true;
-  });
-
-  it("build metadata compares correctly", () => {
-    expect(versionCompare(createVersion(2, 1, 0, "alpha.1", "abcdef0"), createVersion(2, 1, 0, "alpha.1", "bcdef01"))).to.equal(0);
-  });
 });

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -87,16 +87,16 @@ describe("Invocation rate limiter", () => {
 });
 
 class MockExtensionContext implements ExtensionContext {
-  subscriptions: { dispose(): unknown; }[] = [];
+  subscriptions: { dispose(): unknown }[] = [];
   workspaceState: Memento = new MockMemento();
   globalState: Memento = new MockMemento();
-  extensionPath: string = "";
+  extensionPath = "";
   asAbsolutePath(_relativePath: string): string {
     throw new Error("Method not implemented.");
   }
-  storagePath: string = "";
-  globalStoragePath: string = "";
-  logPath: string = "";
+  storagePath = "";
+  globalStoragePath = "";
+  logPath = "";
 }
 
 class MockMemento implements Memento {

extensions/ql-vscode/src/vscode-tests/no-workspace/index.ts

@@ -1,4 +1,4 @@
 import { runTestsInDirectory } from '../index-template';
 export function run(): Promise<void> {
   return runTestsInDirectory(__dirname);
-}
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/sarifMessageParsingTest.test.ts

@@ -6,7 +6,7 @@ import { parseSarifPlainTextMessage } from '../../sarif-utils';
 
 describe('parsing sarif', () => {
   it('should be able to parse a simple message from the spec', async function() {
-    const message = "Tainted data was used. The data came from [here](3)."
+    const message = "Tainted data was used. The data came from [here](3).";
     const results = parseSarifPlainTextMessage(message);
     expect(results).to.deep.equal([
       "Tainted data was used. The data came from ",
@@ -15,7 +15,7 @@ describe('parsing sarif', () => {
   });
 
   it('should be able to parse a complex message from the spec', async function() {
-    const message = "Prohibited term used in [para\\[0\\]\\\\spans\\[2\\]](1)."
+    const message = "Prohibited term used in [para\\[0\\]\\\\spans\\[2\\]](1).";
     const results = parseSarifPlainTextMessage(message);
     expect(results).to.deep.equal([
       "Prohibited term used in ",
@@ -23,14 +23,14 @@ describe('parsing sarif', () => {
     ]);
   });
   it('should be able to parse a broken complex message from the spec', async function() {
-    const message = "Prohibited term used in [para\\[0\\]\\\\spans\\[2\\](1)."
+    const message = "Prohibited term used in [para\\[0\\]\\\\spans\\[2\\](1).";
     const results = parseSarifPlainTextMessage(message);
     expect(results).to.deep.equal([
       "Prohibited term used in [para[0]\\spans[2](1)."
     ]);
   });
   it('should be able to parse a message with extra escaping the spec', async function() {
-    const message = "Tainted data was used. The data came from \\[here](3)."
+    const message = "Tainted data was used. The data came from \\[here](3).";
     const results = parseSarifPlainTextMessage(message);
     expect(results).to.deep.equal([
       "Tainted data was used. The data came from [here](3)."

extensions/ql-vscode/src/vscode-tests/no-workspace/webview-uri.test.ts

@@ -32,7 +32,7 @@ describe('webview uri conversion', function() {
     return {
       fileUriOnDisk,
       panel
-    }
+    };
   }
 
   it('should correctly round trip from filesystem to webview and back', function() {

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -1,6 +1,46 @@
 import * as path from 'path';
+import * as os from 'os';
 import { runTests } from 'vscode-test';
 
+// A subset of the fields in TestOptions from vscode-test, which we
+// would simply use instead, but for the fact that it doesn't export
+// it.
+type Suite = {
+  extensionDevelopmentPath: string;
+  extensionTestsPath: string;
+  launchArgs: string[];
+};
+
+/**
+ * Run an integration test suite `suite`, retrying if it segfaults, at
+ * most `tries` times.
+ */
+async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise<void> {
+  for (let t = 0; t < tries; t++) {
+    try {
+      // Download and unzip VS Code if necessary, and run the integration test suite.
+      await runTests(suite);
+      return;
+    } catch (err) {
+      if (err === 'SIGSEGV') {
+        console.error('Test runner segfaulted.');
+        if (t < tries - 1)
+          console.error('Retrying...');
+      }
+      else if (os.platform() === 'win32') {
+        console.error(`Test runner caught exception (${err})`);
+        if (t < tries - 1)
+          console.error('Retrying...');
+      }
+      else {
+        throw err;
+      }
+    }
+  }
+  console.error(`Tried running suite ${tries} time(s), still failed, giving up.`);
+  process.exit(1);
+}
+
 /**
  * Integration test runner. Launches the VSCode Extension Development Host with this extension installed.
  * See https://github.com/microsoft/vscode-test/blob/master/sample/test/runTest.ts
@@ -32,11 +72,10 @@ async function main() {
     ];
 
     for (const integrationTestSuite of integrationTestSuites) {
-      // Download and unzip VS Code if necessary, and run the integration test suite.
-      await runTests(integrationTestSuite);
+      await runTestsWithRetryOnSegfault(integrationTestSuite, 3);
     }
   } catch (err) {
-    console.error('Failed to run tests');
+    console.error(`Unexpected exception while running tests: ${err}`);
     process.exit(1);
   }
 }