v1.4.7

* Add a polyfill for the Node.js path module

Webpack >v5 doesn't include polyfills for core modules from Node.js by
default. Since we use `path` in the results table UI, we need to include
our own polyfill. This commit adds `path-browserify` to the
distributed extension.

As future work, we could move SARIF location rendering into the core
extension so we don't need to use `path.basename` in the UI. This would
allow us to remove the polyfill.

* Add changelog note

.github/pull_request_template.md

@@ -9,4 +9,4 @@ Replace this with a description of the changes your pull request makes.
 
 - [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
 - [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] `@github/docs-content-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.
+- [ ] `@github/docs-content-codeql` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -2,14 +2,14 @@ name: "Code Scanning - CodeQL"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
     - cron: '0 0 * * 0'
 
 jobs:
   codeql:
-    strategy:
-      fail-fast: false
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository

.github/workflows/main.yml

@@ -1,5 +1,6 @@
 name: Build Extension
 on:
+  workflow_dispatch:
   pull_request:
   push:
     branches:
@@ -30,6 +31,8 @@ jobs:
 
       - name: Build
         working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
           npm run build
         shell: bash
@@ -71,6 +74,8 @@ jobs:
 
       - name: Build
         working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
           npm run build
         shell: bash
@@ -121,7 +126,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.2.6', 'v2.3.3', 'v2.4.0']
+        version: ['v2.2.6', 'v2.3.3', 'v2.4.5', 'v2.4.6', 'v2.5.1']
     env:
       CLI_VERSION: ${{ matrix.version }}
       TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'

.github/workflows/release.yml

@@ -20,7 +20,6 @@ jobs:
   build:
     name: Release
     runs-on: ubuntu-latest
-    # TODO Share steps with the main workflow.
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -36,7 +35,10 @@ jobs:
         shell: bash
 
       - name: Build
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
+          echo "APP INSIGHTS KEY LENGTH: ${#APP_INSIGHTS_KEY}"
           cd extensions/ql-vscode
           npm run build -- --release
         shell: bash
@@ -90,6 +92,10 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
+      ###
+      # Do Post release work: version bump and changelog PR
+      # Only do this if we are running from a PR (ie- this is part of the release process)
+
       # The checkout action does not fetch the main branch.
       # Fetch the main branch so that we can base the version bump PR against main.
       - name: Fetch main branch

.vscode/extensions.json

@@ -1,5 +1,5 @@
 {
-  // See http://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
+  // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [

CONTRIBUTING.md

@@ -25,12 +25,11 @@ Here are a few things you can do that will increase the likelihood of your pull
 * Follow the [style guide][style].
 * Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
 * Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
-Make sure you have a fairly recent version of vscode (>1.32) and are using nodejs
-version >=v10.13.0. (Tested on v10.15.1 and v10.16.0).
+Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
 
 ### Installing all packages
 
@@ -94,7 +93,7 @@ Alternatively, you can run the tests inside of vscode. There are several vscode
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
-1. Double-check that the extension `package.json` has the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
+1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
 1. Create a PR for this release:
     * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
     * Create a new branch for the release named after the new version. For example: `v1.3.6`

README.md

@@ -16,7 +16,6 @@ To see what has changed in the last few versions of the extension, see the [Chan
 * Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
-
 ## Project goals and scope
 
 This project will track new feature development in CodeQL and, whenever appropriate, bring that functionality to the Visual Studio Code experience.

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,57 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.4.7 - 23 April 2021
+
+- Fix a bug that prevented the results view from being loaded. [#842](https://github.com/github/vscode-codeql/pull/842)
+
+## 1.4.6 - 21 April 2021
+
+- Avoid showing an error popup when running a query with `@kind table` metadata. [#814](https://github.com/github/vscode-codeql/pull/814)
+- Add an option to jump from a .qlref file to the .ql file it references. [#815](https://github.com/github/vscode-codeql/pull/815)
+- Avoid opening the results panel when a database is deleted. [#831](https://github.com/github/vscode-codeql/pull/831)
+- Forward all query metadata to the CLI when interpreting results. [#838](https://github.com/github/vscode-codeql/pull/838)
+
+## 1.4.5 - 22 March 2021
+
+- Avoid showing an error popup when user runs a query without `@kind` metadata. [#801](https://github.com/github/vscode-codeql/pull/801)
+- Fix running of tests when the `ms-python` extension is installed. [#803](https://github.com/github/vscode-codeql/pull/803)
+
+## 1.4.4 - 19 March 2021
+
+- Introduce evaluator options for saving intermediate results to the disk cache (`codeQL.runningQueries.saveCache`) and for limiting the size of this cache (`codeQL.runningQueries.cacheSize`). [#778](https://github.com/github/vscode-codeql/pull/778)
+- Respect the `codeQL.runningQueries.numberOfThreads` setting when creating SARIF files during result interpretation. [#771](https://github.com/github/vscode-codeql/pull/771)
+- Allow using raw LGTM project slugs for fetching LGTM databases. [#769](https://github.com/github/vscode-codeql/pull/769)
+- Better error messages when BQRS interpretation fails to produce SARIF. [#770](https://github.com/github/vscode-codeql/pull/770)
+- Implement sorting of the query history view by name, date, and results count. [#777](https://github.com/github/vscode-codeql/pull/777)
+- Add a configuration option to pass additional arguments to the CLI when running tests. [#785](https://github.com/github/vscode-codeql/pull/785)
+- Introduce option to view query results as CSV. [#784](https://github.com/github/vscode-codeql/pull/784)
+- Add some snippets for commonly used QL statements. [#782](https://github.com/github/vscode-codeql/pull/782)
+- More descriptive error messages on QL test failures. [#788](https://github.com/github/vscode-codeql/pull/788)
+
+## 1.4.3 - 22 February 2021
+
+- Avoid displaying an error when removing orphaned databases and the storage folder does not exist. [#748](https://github.com/github/vscode-codeql/pull/748)
+- Add better error messages when AST Viewer is unable to create an AST. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Cache AST viewing operations so that subsequent calls to view the AST of a single file will be extremely fast. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Ensure CodeQL version in status bar updates correctly when version changes. [#754](https://github.com/github/vscode-codeql/pull/754)
+- Avoid deleting the quick query file when it is re-opened. [#747](https://github.com/github/vscode-codeql/pull/747)
+
+## 1.4.2 - 2 February 2021
+
+- Add a status bar item for the CodeQL CLI to show the current version. [#741](https://github.com/github/vscode-codeql/pull/741)
+- Fix version constraint for flagging CLI support of non-destructive updates. [#744](https://github.com/github/vscode-codeql/pull/744)
+- Add a _More Information_ button in the telemetry popup that opens the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) in a browser tab. [#742](https://github.com/github/vscode-codeql/pull/742)
+
+## 1.4.1 - 29 January 2021
+
+- Reword the telemetry modal dialog box. [#738](https://github.com/github/vscode-codeql/pull/738)
+
+## 1.4.0 - 29 January 2021
+
+- Fix bug where databases are not reregistered when the query server restarts. [#734](https://github.com/github/vscode-codeql/pull/734)
+- Fix bug where upgrade requests were erroneously being marked as failed. [#734](https://github.com/github/vscode-codeql/pull/734)
+- On a strictly opt-in basis, collect anonymized usage data from the VS Code extension, helping improve CodeQL's usability and performance. See the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) for more information on exactly what data is collected and what it is used for. [#611](https://github.com/github/vscode-codeql/pull/611)
+
 ## 1.3.10 - 20 January 2021
 
 - Include the full stack in error log messages to help with debugging. [#726](https://github.com/github/vscode-codeql/pull/726)

extensions/ql-vscode/README.md

@@ -1,6 +1,6 @@
 # CodeQL extension for Visual Studio Code
 
-This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
+This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://codeql.github.com/docs/) and allows you to easily find problems in codebases. In particular, the extension:
 
 - Enables you to use CodeQL to query databases generated from source code.
 - Shows the flow of data through the results of path queries, which is essential for triaging security results.
@@ -12,7 +12,7 @@ To see what has changed in the last few versions of the extension, see the [Chan
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
-For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
+For information about other configurations, see the separate [CodeQL help](https://codeql.github.com/docs/codeql-for-visual-studio-code/).
 
 ### Quick start: Installing and configuring the extension
 
@@ -40,9 +40,9 @@ The CodeQL extension requires a minimum of Visual Studio Code 1.39. Older versio
 
 ### Checking access to the CodeQL CLI
 
-The extension uses the [CodeQL CLI](https://help.semmle.com/codeql/codeql-cli.html) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
+The extension uses the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
 
-If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#configuring-access-to-the-codeql-cli).
+If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#configuring-access-to-the-codeql-cli).
 
 If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Log** in the **Output** view for any error messages.
 
@@ -52,7 +52,7 @@ When you're working with CodeQL, you need access to the standard CodeQL librarie
 Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
-For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
+For information about configuring an existing workspace for CodeQL, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#updating-an-existing-workspace-for-codeql).
 
 ## Upgrading CodeQL standard libraries
 
@@ -75,7 +75,7 @@ You can find all the commands contributed by the extension in the Command Palett
 
 ### Importing a database from LGTM
 
-While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
+While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from LGTM.com.
 
 1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
 1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
@@ -100,13 +100,17 @@ If there are any problems running a query, a notification is displayed in the bo
 
 ## What next?
 
-For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
+For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:
 
-- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Create a database for a different codebase](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/).
 - [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Learn more about CodeQL](https://codeql.github.com/docs/).
 - [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
 
 The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the MIT License. The version of CodeQL used by the CodeQL extension is subject to the [GitHub CodeQL Terms & Conditions](https://securitylab.github.com/tools/codeql/license).
+
+## Data and Telemetry
+
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information about telemetry, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code).

extensions/ql-vscode/gulpfile.ts/appInsights.ts

@@ -0,0 +1,16 @@
+import * as gulp from 'gulp';
+import * as replace from 'gulp-replace';
+
+/** Inject the application insights key into the telemetry file */
+export function injectAppInsightsKey() {
+  if (!process.env.APP_INSIGHTS_KEY) {
+    // noop
+    console.log('APP_INSIGHTS_KEY environment variable is not set. So, cannot inject it into the application.');
+    return Promise.resolve();
+  }
+
+  // replace the key
+  return gulp.src(['out/telemetry.js'])
+    .pipe(replace(/REPLACE-APP-INSIGHTS-KEY/, process.env.APP_INSIGHTS_KEY))
+    .pipe(gulp.dest('out/'));
+}

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -13,6 +13,7 @@ const packageFiles = [
   'CHANGELOG.md',
   'README.md',
   'language-configuration.json',
+  'snippets.json',
   'media',
   'node_modules',
   'out'

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -4,7 +4,12 @@ import { compileTextMateGrammar } from './textmate';
 import { copyTestData } from './tests';
 import { compileView } from './webpack';
 import { packageExtension } from './package';
+import { injectAppInsightsKey } from './appInsights';
 
-export const buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
-export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData };
-exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);
+export const buildWithoutPackage =
+  gulp.parallel(
+    compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
+  );
+
+export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData, injectAppInsightsKey };
+export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json.schemastore.org/tsconfig",
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
     "strict": true,

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -13,7 +13,10 @@ export const config: webpack.Configuration = {
   },
   devtool: 'inline-source-map',
   resolve: {
-    extensions: ['.js', '.ts', '.tsx', '.json']
+    extensions: ['.js', '.ts', '.tsx', '.json'],
+    fallback: {
+      path: require.resolve('path-browserify')
+    }
   },
   module: {
     rules: [

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -6,21 +6,23 @@ export function compileView(cb: (err?: Error) => void) {
     if (error) {
       cb(error);
     }
-    console.log(stats.toString({
-      errorDetails: true,
-      colors: true,
-      assets: false,
-      builtAt: false,
-      version: false,
-      hash: false,
-      entrypoints: false,
-      timings: false,
-      modules: false,
-      errors: true
-    }));
-    if (stats.hasErrors()) {
-      cb(new Error('Compilation errors detected.'));
-      return;
+    if (stats) {
+      console.log(stats.toString({
+        errorDetails: true,
+        colors: true,
+        assets: false,
+        builtAt: false,
+        version: false,
+        hash: false,
+        entrypoints: false,
+        timings: false,
+        modules: false,
+        errors: true
+      }));
+      if (stats.hasErrors()) {
+        cb(new Error('Compilation errors detected.'));
+        return;
+      }
     }
 
     cb();

extensions/ql-vscode/media/dark/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" fill="#C5C5C5" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" fill="#C5C5C5"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/light/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" />
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" />
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" />
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.3.10",
+  "version": "1.4.7",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -34,6 +34,7 @@
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.openReferencedFile",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
@@ -106,6 +107,12 @@
         "path": "./out/syntaxes/dbscheme.tmLanguage.json"
       }
     ],
+    "snippets": [
+      {
+        "language": "ql",
+        "path": "./snippets.json"
+      }
+    ],
     "configuration": {
       "type": "object",
       "title": "CodeQL",
@@ -114,7 +121,7 @@
           "scope": "machine",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
+          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -123,6 +130,21 @@
           "maximum": 1024,
           "description": "Number of threads for running queries."
         },
+        "codeQL.runningQueries.saveCache": {
+          "type": "boolean",
+          "default": false,
+          "scope": "window",
+          "description": "Aggressively save intermediate results to the disk cache. This may speed up subsequent queries if they are similar. Be aware that using this option will greatly increase disk usage and initial evaluation time."
+        },
+        "codeQL.runningQueries.cacheSize": {
+          "type": [
+            "integer",
+            "null"
+          ],
+          "default": null,
+          "minimum": 1024,
+          "description": "Maximum size of the disk cache (in MB). Leave blank to allow the evaluator to automatically adjust the size of the disk cache based on the size of the codebase and the complexity of the queries being executed."
+        },
         "codeQL.runningQueries.timeout": {
           "type": [
             "integer",
@@ -160,12 +182,18 @@
         "codeQL.resultsDisplay.pageSize": {
           "type": "integer",
           "default": 200,
-          "description": "Number of queries displayed per page of the results view."
+          "description": "Max number of query results to display per page in the results view."
         },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "[%t] %q on %d - %s",
-          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+          "default": "%q on %d - %s, %r result count [%t]",
+          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, %r is the number of results, and %s is a status string."
+        },
+        "codeQL.runningTests.additionalTestArguments": {
+          "scope": "machine",
+          "type": "array",
+          "default": [],
+          "markdownDescription": "Additional command line arguments to pass to the CLI when [running tests](https://codeql.github.com/docs/codeql-cli/manual/test-run/). This setting should be an array of strings, each containing an argument to be passed."
         },
         "codeQL.runningTests.numberOfThreads": {
           "scope": "window",
@@ -174,6 +202,18 @@
           "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running CodeQL tests."
+        },
+        "codeQL.telemetry.enableTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "markdownDescription": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub. For more information, see the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code)"
+        },
+        "codeQL.telemetry.logTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "description": "Specifies whether or not to write telemetry events to the extension log."
         }
       }
     },
@@ -190,10 +230,18 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.openReferencedFile",
+        "title": "CodeQL: Open Referenced File"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
       },
+      {
+        "command": "codeQL.openDocumentation",
+        "title": "CodeQL: Open Documentation"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
@@ -326,6 +374,30 @@
           "dark": "media/dark/trash.svg"
         }
       },
+      {
+        "command": "codeQLQueryHistory.sortByName",
+        "title": "Sort by Name",
+        "icon": {
+          "light": "media/light/sort-alpha.svg",
+          "dark": "media/dark/sort-alpha.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByDate",
+        "title": "Sort by Query Date",
+        "icon": {
+          "light": "media/light/sort-date.svg",
+          "dark": "media/dark/sort-date.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByCount",
+        "title": "Sort by Results Count",
+        "icon": {
+          "light": "media/light/sort-num.svg",
+          "dark": "media/dark/sort-num.svg"
+        }
+      },
       {
         "command": "codeQLQueryHistory.showQueryLog",
         "title": "Show Query Log"
@@ -335,8 +407,12 @@
         "title": "Show Query Text"
       },
       {
-        "command": "codeQLQueryHistory.viewSarif",
-        "title": "View SARIF"
+        "command": "codeQLQueryHistory.viewCsvResults",
+        "title": "View Results (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarifResults",
+        "title": "View Results (SARIF)"
       },
       {
         "command": "codeQLQueryHistory.viewDil",
@@ -430,6 +506,21 @@
           "when": "view == codeQLQueryHistory",
           "group": "navigation"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
         {
           "command": "codeQLAstViewer.clear",
           "when": "view == codeQLAstViewer",
@@ -493,7 +584,12 @@
           "when": "view == codeQLQueryHistory"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifResults",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
         },
@@ -528,6 +624,11 @@
           "command": "codeQL.runQueries",
           "group": "9_qlCommands",
           "when": "resourceScheme != codeql-zip-archive"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .qlref"
         }
       ],
       "commandPalette": [
@@ -543,6 +644,10 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -620,7 +725,11 @@
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifResults",
           "when": "false"
         },
         {
@@ -635,6 +744,18 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "false"
+        },
         {
           "command": "codeQLAstViewer.gotoCode",
           "when": "false"
@@ -664,6 +785,10 @@
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
         }
       ]
     },
@@ -728,6 +853,7 @@
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.5",
     "node-fetch": "~2.6.0",
+    "path-browserify": "^1.0.1",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "semver": "~7.3.2",
@@ -735,6 +861,7 @@
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
+    "vscode-extension-telemetry": "^0.1.6",
     "vscode-jsonrpc": "^5.0.1",
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
@@ -750,10 +877,11 @@
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
+    "@types/gulp-replace": "0.0.31",
     "@types/gulp-sourcemaps": "0.0.32",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
-    "@types/mocha": "^8.0.4",
+    "@types/mocha": "^8.2.0",
     "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
@@ -772,6 +900,7 @@
     "@typescript-eslint/eslint-plugin": "~2.23.0",
     "@typescript-eslint/parser": "~2.23.0",
     "ansi-colors": "^4.1.1",
+    "applicationinsights": "^1.8.7",
     "chai": "^4.2.0",
     "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
@@ -779,6 +908,7 @@
     "eslint-plugin-react": "~7.19.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
+    "gulp-replace": "^1.0.0",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
     "husky": "~4.2.5",
@@ -793,15 +923,15 @@
     "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
-    "ts-loader": "^5.4.5",
+    "ts-loader": "^8.1.0",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
     "typescript": "~3.8.3",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
     "vscode-test": "^1.4.0",
-    "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2"
+    "webpack": "^5.28.0",
+    "webpack-cli": "^4.6.0"
   },
   "husky": {
     "hooks": {

extensions/ql-vscode/snippets.json

@@ -0,0 +1,134 @@
+{
+  "Query Metadata": {
+    "prefix": "querymetadata",
+    "body": [
+      "/**",
+      " * @name $1",
+      " * @description $2",
+      " * @kind $3",
+      " * @id $4",
+      " * @tags $5",
+      " */"
+    ],
+    "description": "Metadata for a query"
+  },
+  "Class": {
+    "prefix": "class",
+    "body": ["class $1 extends $2 {", "\t$0", "}"],
+    "description": "A class"
+  },
+  "From/Where/Select": {
+    "prefix": "from",
+    "body": ["from $1", "where $2", "select $3"],
+    "description": "A from/where/select statement"
+  },
+  "Predicate": {
+    "prefix": "predicate",
+    "body": ["predicate $1($2) {", "\t$0", "}"],
+    "description": "A predicate"
+  },
+  "Dataflow Tracking Class": {
+    "prefix": "dataflowtracking",
+    "body": [
+      "class $1 extends DataFlow::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a dataflow tracking class"
+  },
+  "Taint Tracking Class": {
+    "prefix": "tainttracking",
+    "body": [
+      "class $1 extends TaintTracking::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a taint tracking class"
+  },
+  "Count": {
+    "prefix": "count",
+    "body": ["count($1 | $2 | $3)"],
+    "description": "A count aggregate"
+  },
+  "Max": {
+    "prefix": "max",
+    "body": ["max($1 | $2 | $3)"],
+    "description": "A max aggregate"
+  },
+  "Min": {
+    "prefix": "min",
+    "body": ["min($1 | $2 | $3)"],
+    "description": "A min aggregate"
+  },
+  "Average": {
+    "prefix": "avg",
+    "body": ["avg($1 | $2 | $3)"],
+    "description": "An average aggregate"
+  },
+  "Sum": {
+    "prefix": "sum",
+    "body": ["sum($1 | $2 | $3)"],
+    "description": "A sum aggregate"
+  },
+  "Concatenation": {
+    "prefix": "concat",
+    "body": ["concat($1 | $2 | $3)"],
+    "description": "A concatenation aggregate"
+  },
+  "Rank": {
+    "prefix": "rank",
+    "body": ["rank[$1]($2 | $3 | $4)"],
+    "description": "A rank aggregate"
+  },
+  "Strict Sum": {
+    "prefix": "strictsum",
+    "body": ["strictsum($1 | $2 | $3)"],
+    "description": "A strict sum aggregate"
+  },
+  "Strict Concatenation": {
+    "prefix": "strictconcat",
+    "body": ["strictconcat($1 | $2 | $3)"],
+    "description": "A strict concatenation aggregate"
+  },
+  "Strict Count": {
+    "prefix": "strictcount",
+    "body": ["strictcount($1 | $2 | $3)"],
+    "description": "A strict count aggregate"
+  },
+  "Unique": {
+    "prefix": "unique",
+    "body": ["unique($1 | $2 | $3)"],
+    "description": "A unique aggregate"
+  },
+  "Exists": {
+    "prefix": "exists",
+    "body": ["exists($1 | $2 | $3)"],
+    "description": "An exists quantifier"
+  },
+  "For All": {
+    "prefix": "forall",
+    "body": ["forall($1 | $2 | $3)"],
+    "description": "A for all quantifier"
+  },
+  "For All and Exists": {
+    "prefix": "forex",
+    "body": ["forex($1 | $2 | $3)"],
+    "description": "A for all and exists quantifier"
+  }
+}

extensions/ql-vscode/src/astViewer.ts

@@ -19,7 +19,8 @@ import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
 import { showLocation } from './interface-utils';
 import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
 import { commandRunner } from './commandRunner';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
 
 export interface AstItem {
   id: BqrsId;
@@ -129,8 +130,13 @@ export class AstViewer extends DisposableObject {
     this.treeDataProvider.db = db;
     this.treeDataProvider.refresh();
     this.treeView.message = `AST for ${path.basename(fileName)}`;
-    this.treeView.reveal(roots[0], { focus: false });
     this.currentFile = fileName;
+    // Handle error on reveal. This could happen if
+    // the tree view is disposed during the reveal.
+    this.treeView.reveal(roots[0], { focus: false })?.then(
+      () => { /**/ },
+      err => showAndLogErrorMessage(err)
+    );
   }
 
   private updateTreeSelection(e: TextEditorSelectionChangeEvent) {
@@ -178,7 +184,12 @@ export class AstViewer extends DisposableObject {
 
       const targetItem = findBest(range, this.treeDataProvider.roots);
       if (targetItem) {
-        this.treeView.reveal(targetItem);
+        // Handle error on reveal. This could happen if
+        // the tree view is disposed during the reveal.
+        this.treeView.reveal(targetItem)?.then(
+          () => { /**/ },
+          err => showAndLogErrorMessage(err)
+        );
       }
     }
   }

extensions/ql-vscode/src/cli.ts

@@ -24,6 +24,11 @@ import { CompilationMessage } from './pure/messages';
  */
 const SARIF_FORMAT = 'sarifv2.1.0';
 
+/**
+ * The string used to specify CSV format.
+ */
+const CSV_FORMAT = 'csv';
+
 /**
  * Flags to pass to all cli commands.
  */
@@ -59,6 +64,7 @@ export interface DbInfo {
 export interface UpgradesInfo {
   scripts: string[];
   finalDbscheme: string;
+  matchesTarget?: boolean;
 }
 
 /**
@@ -66,6 +72,11 @@ export interface UpgradesInfo {
  */
 export type QlpacksInfo = { [name: string]: string[] };
 
+/**
+ * The expected output of `codeql resolve qlref`.
+ */
+export type QlrefInfo = { resolvedPath: string };
+
 // `codeql bqrs interpret` requires both of these to be present or
 // both absent.
 export interface SourceInfo {
@@ -97,6 +108,7 @@ export interface TestCompleted {
   evaluationMs: number;
   expected: string;
   diff: string[] | undefined;
+  failureDescription?: string;
 }
 
 /**
@@ -119,15 +131,6 @@ interface BqrsDecodeOptions {
  */
 export class CodeQLCliServer implements Disposable {
 
-  /**
-   * CLI version where --kind=DIL was introduced
-   */
-  private static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
-
-  /**
-   * CLI version where languages are exposed during a `codeql resolve database` command.
-   */
-  private static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
 
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
@@ -144,6 +147,8 @@ export class CodeQLCliServer implements Disposable {
   /** Path to current codeQL executable, or undefined if not running yet. */
   codeQlPath: string | undefined;
 
+  cliConstraints = new CliVersionConstraint(this);
+
   /**
    * When set to true, ignore some modal popups and assume user has clicked "yes".
    */
@@ -160,6 +165,7 @@ export class CodeQLCliServer implements Disposable {
     if (this.distributionProvider.onDidChangeDistribution) {
       this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
+        this._version = undefined;
       });
     }
     if (this.cliConfig.onDidChangeConfiguration) {
@@ -446,12 +452,15 @@ export class CodeQLCliServer implements Disposable {
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
    * @param commandArgs The arguments to pass to the `codeql` command.
    * @param description Description of the action being run, to be shown in log and error messages.
+   * @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
    * @param progressReporter Used to output progress messages, e.g. to the status bar.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
-  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, progressReporter?: ProgressReporter): Promise<OutputType> {
-    // Add format argument first, in case commandArgs contains positional parameters.
-    const args = ['--format', 'json'].concat(commandArgs);
+  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, addFormat = true, progressReporter?: ProgressReporter): Promise<OutputType> {
+    let args: string[] = [];
+    if (addFormat) // Add format argument first, in case commandArgs contains positional parameters.
+      args = args.concat(['--format', 'json']);
+    args = args.concat(commandArgs);
     const result = await this.runCodeQlCliCommand(command, args, description, progressReporter);
     try {
       return JSON.parse(result) as OutputType;
@@ -490,6 +499,18 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  public async resolveQlref(qlref: string): Promise<QlrefInfo> {
+    const subcommandArgs = [
+      qlref
+    ];
+    return await this.runJsonCodeQlCliCommand<QlrefInfo>(
+      ['resolve', 'qlref'],
+      subcommandArgs,
+      'Resolving qlref',
+      false
+    );
+  }
+
   /**
    * Runs QL tests.
    * @param testPaths Full paths of the tests to run.
@@ -500,12 +521,12 @@ export class CodeQLCliServer implements Disposable {
     testPaths: string[], workspaces: string[], options: TestRunOptions
   ): AsyncGenerator<TestCompleted, void, unknown> {
 
-    const subcommandArgs = [
+    const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
       '--additional-packs', workspaces.join(path.delimiter),
       '--threads',
       this.cliConfig.numberTestThreads.toString(),
       ...testPaths
-    ];
+    ]);
 
     for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
       subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
@@ -534,7 +555,7 @@ export class CodeQLCliServer implements Disposable {
     if (queryMemoryMb !== undefined) {
       args.push('--ram', queryMemoryMb.toString());
     }
-    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', progressReporter);
+    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', true, progressReporter);
   }
   /**
    * Gets the headers (and optionally pagination info) of a bqrs.
@@ -573,32 +594,48 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
-      `-t=kind=${metadata.kind}`,
-      `-t=id=${metadata.id}`,
       '--output', interpretedResultsPath,
-      '--format', SARIF_FORMAT,
+      '--format', format,
+      // Forward all of the query metadata.
+      ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`)
+    ];
+    if (format == SARIF_FORMAT) {
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      '--no-group-results',
-    ];
+      args.push('--no-group-results');
+    }
     if (sourceInfo !== undefined) {
       args.push(
         '--source-archive', sourceInfo.sourceArchive,
         '--source-location-prefix', sourceInfo.sourceLocationPrefix
       );
     }
+
+    args.push(
+      '--threads',
+      this.cliConfig.numberThreads.toString(),
+    );
+
     args.push(resultsPath);
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
+  }
+
+  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
 
     let output: string;
     try {
       output = await fs.readFile(interpretedResultsPath, 'utf8');
-    } catch (err) {
-      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
+    } catch (e) {
+      const rawMessage = e.stderr || e.message;
+      const errorMessage = rawMessage.startsWith('Cannot create a string')
+        ? `SARIF too large. ${rawMessage}`
+        : rawMessage;
+      throw new Error(`Reading output of interpretation failed: ${errorMessage}`);
     }
     try {
       return JSON.parse(output) as sarif.Log;
@@ -607,6 +644,9 @@ export class CodeQLCliServer implements Disposable {
     }
   }
 
+  async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
+    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+  }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
     const sortDirectionStrings = sortDirections.map(direction => {
@@ -646,12 +686,19 @@ export class CodeQLCliServer implements Disposable {
    * Gets information necessary for upgrading a database.
    * @param dbScheme the path to the dbscheme of the database to be upgraded.
    * @param searchPath A list of directories to search for upgrade scripts.
+   * @param allowDowngradesIfPossible Whether we should try and include downgrades of we can.
+   * @param targetDbScheme The dbscheme to try to upgrade to.
    * @returns A list of database upgrade script directories
    */
-  resolveUpgrades(dbScheme: string, searchPath: string[]): Promise<UpgradesInfo> {
+  async resolveUpgrades(dbScheme: string, searchPath: string[], allowDowngradesIfPossible: boolean, targetDbScheme?: string): Promise<UpgradesInfo> {
     const args = ['--additional-packs', searchPath.join(path.delimiter), '--dbscheme', dbScheme];
-
-    return this.runJsonCodeQlCliCommand<UpgradesInfo>(
+    if (targetDbScheme) {
+      args.push('--target-dbscheme', targetDbScheme);
+      if (allowDowngradesIfPossible && await this.cliConstraints.supportsDowngrades()) {
+        args.push('--allow-downgrades');
+      }
+    }
+    return await this.runJsonCodeQlCliCommand<UpgradesInfo>(
       ['resolve', 'upgrades'],
       args,
       'Resolving database upgrade scripts',
@@ -700,7 +747,7 @@ export class CodeQLCliServer implements Disposable {
   }
 
   async generateDil(qloFile: string, outFile: string): Promise<void> {
-    const extraArgs = await this.supportsDecompileDil()
+    const extraArgs = await this.cliConstraints.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
       : ['-o', outFile, qloFile];
     await this.runCodeQlCliCommand(
@@ -717,14 +764,6 @@ export class CodeQLCliServer implements Disposable {
     return this._version;
   }
 
-  private async supportsDecompileDil() {
-    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0;
-  }
-
-  public async supportsLanguageName() {
-    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_LANGUAGE) >= 0;
-  }
-
   private async refreshVersion() {
     const distribution = await this.distributionProvider.getDistribution();
     switch (distribution.kind) {
@@ -858,6 +897,20 @@ class SplitBuffer {
     this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
   }
 
+  /**
+   * A version of startsWith that isn't overriden by a broken version of ms-python.
+   *
+   * The definition comes from
+   * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
+   * which is CC0/public domain
+   *
+   * See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
+   */
+  private static startsWith(s: string, searchString: string, position: number): boolean {
+    const pos = position > 0 ? position | 0 : 0;
+    return s.substring(pos, pos + searchString.length) === searchString;
+  }
+
   /**
    * Extract the next full line from the buffer, if one is available.
    * @returns The text of the next available full line (without the separator), or `undefined` if no
@@ -866,7 +919,7 @@ class SplitBuffer {
   public getNextLine(): string | undefined {
     while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
       for (const separator of this.separators) {
-        if (this.buffer.startsWith(separator, this.searchIndex)) {
+        if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
           const line = this.buffer.substr(0, this.searchIndex);
           this.buffer = this.buffer.substr(this.searchIndex + separator.length);
           this.searchIndex = 0;
@@ -939,3 +992,60 @@ export function shouldDebugQueryServer() {
     && process.env.QUERY_SERVER_JAVA_DEBUG !== '0'
     && process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
 }
+
+export class CliVersionConstraint {
+
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  public static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
+
+  /**
+   * CLI version where languages are exposed during a `codeql resolve database` command.
+   */
+  public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
+
+  /**
+   * CLI version where `codeql resolve upgrades` supports
+   * the `--allow-downgrades` flag
+   */
+  public static CLI_VERSION_WITH_DOWNGRADES = new SemVer('2.4.4');
+
+  /**
+   * CLI version where the `codeql resolve qlref` command is available.
+   */
+  public static CLI_VERSION_WITH_RESOLVE_QLREF = new SemVer('2.5.1');
+
+  /**
+   * CLI version where database registration was introduced
+   */
+  public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
+
+  constructor(private readonly cli: CodeQLCliServer) {
+    /**/
+  }
+
+  private async isVersionAtLeast(v: SemVer) {
+    return (await this.cli.getVersion()).compare(v) >= 0;
+  }
+
+  public async supportsDecompileDil() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DECOMPILE_KIND_DIL);
+  }
+
+  public async supportsLanguageName() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
+  }
+
+  public async supportsDowngrades() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
+  }
+
+  public async supportsResolveQlref() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF);
+  }
+
+  async supportsDatabaseRegistration() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
+  }
+}

extensions/ql-vscode/src/commandRunner.ts

@@ -8,6 +8,7 @@ import {
 } from 'vscode';
 import { showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
 import { logger } from './logging';
+import { telemetryListener } from './telemetry';
 
 export class UserCancellationException extends Error {
   /**
@@ -114,9 +115,13 @@ export function commandRunner(
   task: NoProgressTask,
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+
     try {
       return await task(...args);
     } catch (e) {
+      error = e;
       const errorMessage = `${e.message || e} (${commandId})`;
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
@@ -135,6 +140,9 @@ export function commandRunner(
         });
       }
       return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
     }
   });
 }
@@ -155,6 +163,8 @@ export function commandRunnerWithProgress<R>(
   progressOptions: Partial<ProgressOptions>
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
     const progressOptionsWithDefaults = {
       location: ProgressLocation.Notification,
       ...progressOptions
@@ -162,6 +172,7 @@ export function commandRunnerWithProgress<R>(
     try {
       return await withProgress(progressOptionsWithDefaults, task, ...args);
     } catch (e) {
+      error = e;
       const errorMessage = `${e.message || e} (${commandId})`;
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
@@ -180,6 +191,9 @@ export function commandRunnerWithProgress<R>(
         });
       }
       return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
     }
   });
 }

extensions/ql-vscode/src/compare/compare-interface.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from '../vscode-utils/disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import {
   WebviewPanel,
   ExtensionContext,

extensions/ql-vscode/src/config.ts

@@ -1,10 +1,10 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
 
 /** Helper class to look up a labelled (and possibly nested) setting. */
-class Setting {
+export class Setting {
   name: string;
   parent?: Setting;
 
@@ -39,10 +39,19 @@ class Setting {
 
 const ROOT_SETTING = new Setting('codeQL');
 
-// Distribution configuration
+// Global configuration
+const TELEMETRY_SETTING = new Setting('telemetry', ROOT_SETTING);
+const AST_VIEWER_SETTING = new Setting('astViewer', ROOT_SETTING);
+const GLOBAL_TELEMETRY_SETTING = new Setting('telemetry');
 
+export const LOG_TELEMETRY = new Setting('logTelemetry', TELEMETRY_SETTING);
+export const ENABLE_TELEMETRY = new Setting('enableTelemetry', TELEMETRY_SETTING);
+
+export const GLOBAL_ENABLE_TELEMETRY = new Setting('enableTelemetry', GLOBAL_TELEMETRY_SETTING);
+
+// Distribution configuration
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
-const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
+export const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
 const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
@@ -65,24 +74,29 @@ export interface DistributionConfig {
 
 const RUNNING_QUERIES_SETTING = new Setting('runningQueries', ROOT_SETTING);
 const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES_SETTING);
+const SAVE_CACHE_SETTING = new Setting('saveCache', RUNNING_QUERIES_SETTING);
+const CACHE_SIZE_SETTING = new Setting('cacheSize', RUNNING_QUERIES_SETTING);
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
 const RUNNING_TESTS_SETTING = new Setting('runningTests', ROOT_SETTING);
 const RESULTS_DISPLAY_SETTING = new Setting('resultsDisplay', ROOT_SETTING);
 
+export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting('additionalTestArguments', RUNNING_TESTS_SETTING);
 export const NUMBER_OF_TEST_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_TESTS_SETTING);
 export const MAX_QUERIES = new Setting('maxQueries', RUNNING_QUERIES_SETTING);
 export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
 export const PAGE_SIZE = new Setting('pageSize', RESULTS_DISPLAY_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
-const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
+const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING, DEBUG_SETTING];
 
 export interface QueryServerConfig {
   codeQlPath: string;
   debug: boolean;
   numThreads: number;
+  saveCache: boolean;
+  cacheSize: number;
   queryMemoryMb?: number;
   timeoutSecs: number;
   onDidChangeConfiguration?: Event<void>;
@@ -96,15 +110,17 @@ export interface QueryHistoryConfig {
   onDidChangeConfiguration: Event<void>;
 }
 
-const CLI_SETTINGS = [NUMBER_OF_TEST_THREADS_SETTING];
+const CLI_SETTINGS = [ADDITIONAL_TEST_ARGUMENTS_SETTING, NUMBER_OF_TEST_THREADS_SETTING, NUMBER_OF_THREADS_SETTING];
 
 export interface CliConfig {
+  additionalTestArguments: string[];
   numberTestThreads: number;
+  numberThreads: number;
   onDidChangeConfiguration?: Event<void>;
 }
 
 
-abstract class ConfigListener extends DisposableObject {
+export abstract class ConfigListener extends DisposableObject {
   protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());
 
   constructor() {
@@ -185,6 +201,14 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     return NUMBER_OF_THREADS_SETTING.getValue<number>();
   }
 
+  public get saveCache(): boolean {
+    return SAVE_CACHE_SETTING.getValue<boolean>();
+  }
+
+  public get cacheSize(): number {
+    return CACHE_SIZE_SETTING.getValue<number | null>() || 0;
+  }
+
   /** Gets the configured query timeout, in seconds. This looks up the setting at the time of access. */
   public get timeoutSecs(): number {
     return TIMEOUT_SETTING.getValue<number | null>() || 0;
@@ -222,11 +246,18 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
 }
 
 export class CliConfigListener extends ConfigListener implements CliConfig {
+  public get additionalTestArguments(): string[] {
+    return ADDITIONAL_TEST_ARGUMENTS_SETTING.getValue();
+  }
 
   public get numberTestThreads(): number {
     return NUMBER_OF_TEST_THREADS_SETTING.getValue();
   }
 
+  public get numberThreads(): number {
+    return NUMBER_OF_THREADS_SETTING.getValue<number>();
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
   }
@@ -240,3 +271,17 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
  * want to enable experimental features, they can add them directly in
  * their vscode settings json file.
  */
+
+/**
+ * Enables canary features of this extension. Recommended for all internal users.
+ */
+export const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);
+
+export function isCanary() {
+  return !!CANARY_FEATURES.getValue<boolean>();
+}
+
+/**
+ * Avoids caching in the AST viewer if the user is also a canary user.
+ */
+export const NO_CACHE_AST_VIEWER = new Setting('disableCache', AST_VIEWER_SETTING);

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -1,4 +1,15 @@
-import * as vscode from 'vscode';
+import {
+  CancellationToken,
+  DefinitionProvider,
+  Location,
+  LocationLink,
+  Position,
+  ProgressLocation,
+  ReferenceContext,
+  ReferenceProvider,
+  TextDocument,
+  Uri
+} from 'vscode';
 
 import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
 import { CodeQLCliServer } from '../cli';
@@ -14,6 +25,7 @@ import {
 } from './keyType';
 import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
 
 /**
  * Run templated CodeQL queries to find definitions and references in
@@ -22,20 +34,20 @@ import { qlpackOfDatabase, resolveQueries } from './queryResolver';
  * or from a selected identifier.
  */
 
-export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
-  private cache: CachedOperation<vscode.LocationLink[]>;
+export class TemplateQueryDefinitionProvider implements DefinitionProvider {
+  private cache: CachedOperation<LocationLink[]>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
   ) {
-    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+    this.cache = new CachedOperation<LocationLink[]>(this.getDefinitions.bind(this));
   }
 
-  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+  async provideDefinition(document: TextDocument, position: Position, _token: CancellationToken): Promise<LocationLink[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.LocationLink[] = [];
+    const locLinks: LocationLink[] = [];
     for (const link of fileLinks) {
       if (link.originSelectionRange!.contains(position)) {
         locLinks.push(link);
@@ -44,9 +56,9 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
     return locLinks;
   }
 
-  private async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+  private async getDefinitions(uriString: string): Promise<LocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding definitions'
     }, async (progress, token) => {
@@ -64,7 +76,7 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
   }
 }
 
-export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+export class TemplateQueryReferenceProvider implements ReferenceProvider {
   private cache: CachedOperation<FullLocationLink[]>;
 
   constructor(
@@ -76,13 +88,13 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
   }
 
   async provideReferences(
-    document: vscode.TextDocument,
-    position: vscode.Position,
-    _context: vscode.ReferenceContext,
-    _token: vscode.CancellationToken
-  ): Promise<vscode.Location[]> {
+    document: TextDocument,
+    position: Position,
+    _context: ReferenceContext,
+    _token: CancellationToken
+  ): Promise<Location[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.Location[] = [];
+    const locLinks: Location[] = [];
     for (const link of fileLinks) {
       if (link.targetRange!.contains(position)) {
         locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
@@ -93,7 +105,7 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 
   private async getReferences(uriString: string): Promise<FullLocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding references'
     }, async (progress, token) => {
@@ -112,40 +124,47 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 }
 
 export class TemplatePrintAstProvider {
-  private cache: CachedOperation<QueryWithResults | undefined>;
+  private cache: CachedOperation<QueryWithResults>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
-
-    // Note: progress and token are only used if a cached value is not available
-    private progress: ProgressCallback,
-    private token: vscode.CancellationToken
   ) {
-    this.cache = new CachedOperation<QueryWithResults | undefined>(this.getAst.bind(this));
+    this.cache = new CachedOperation<QueryWithResults>(this.getAst.bind(this));
   }
 
-  async provideAst(document?: vscode.TextDocument): Promise<AstBuilder | undefined> {
+  async provideAst(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    document?: TextDocument
+  ): Promise<AstBuilder | undefined> {
     if (!document) {
-      return;
-    }
-    const queryResults = await this.cache.get(document.uri.toString());
-    if (!queryResults) {
-      return;
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
+    const queryResults = this.shouldCache()
+      ? await this.cache.get(document.uri.toString(), progress, token)
+      : await this.getAst(document.uri.toString(), progress, token);
 
     return new AstBuilder(
       queryResults, this.cli,
-      this.dbm.findDatabaseItem(vscode.Uri.parse(queryResults.database.databaseUri!, true))!,
+      this.dbm.findDatabaseItem(Uri.parse(queryResults.database.databaseUri!, true))!,
       document.fileName
     );
   }
 
-  private async getAst(uriString: string): Promise<QueryWithResults> {
-    const uri = vscode.Uri.parse(uriString, true);
+  private shouldCache() {
+    return !(isCanary() && NO_CACHE_AST_VIEWER.getValue<boolean>());
+  }
+
+  private async getAst(
+    uriString: string,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<QueryWithResults> {
+    const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
-      throw new Error('AST Viewing is only available for databases with zipped source archives.');
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
 
     const zippedArchive = decodeSourceArchiveUri(uri);
@@ -181,9 +200,9 @@ export class TemplatePrintAstProvider {
       this.qs,
       db,
       false,
-      vscode.Uri.file(query),
-      this.progress,
-      this.token,
+      Uri.file(query),
+      progress,
+      token,
       templates
     );
   }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -74,7 +74,7 @@ export async function promptImportLgtmDatabase(
 ): Promise<DatabaseItem | undefined> {
   const lgtmUrl = await window.showInputBox({
     prompt:
-      'Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)',
+      'Enter the project slug or URL on LGTM (e.g., g/github/codeql or https://lgtm.com/projects/g/github/codeql)',
   });
   if (!lgtmUrl) {
     return;
@@ -352,13 +352,14 @@ export async function findDirWithFile(
 
 /**
  * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
- * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * There are several possibilities for the provider: in addition to GitHub.com (g),
  * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
  *
- * After the {provider}/{org}/{name} path components, there may be the components
- * related to sub pages.
+ * This function accepts any url that matches the pattern above. It also accepts the
+ * raw project slug, e.g., `g/myorg/myproject`
  *
- * This function accepts any url that matches the patter above
+ * After the `{provider}/{org}/{name}` path components, there may be the components
+ * related to sub pages.
  *
  * @param lgtmUrl The URL to the lgtm project
  *
@@ -370,6 +371,10 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
     return false;
   }
 
+  if (convertRawLgtmSlug(lgtmUrl)) {
+    return true;
+  }
+
   try {
     const uri = Uri.parse(lgtmUrl, true);
     if (uri.scheme !== 'https') {
@@ -387,9 +392,23 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
   }
 }
 
+function convertRawLgtmSlug(maybeSlug: string): string | undefined {
+  if (!maybeSlug) {
+    return;
+  }
+  const segments = maybeSlug.split('/');
+  const providers = ['g', 'gl', 'b', 'git'];
+  if (segments.length === 3 && providers.includes(segments[0])) {
+    return `https://lgtm.com/projects/${maybeSlug}`;
+  }
+  return;
+}
+
 // exported for testing
 export async function convertToDatabaseUrl(lgtmUrl: string) {
   try {
+    lgtmUrl = convertRawLgtmSlug(lgtmUrl) || lgtmUrl;
+
     const uri = Uri.parse(lgtmUrl, true);
     const paths = ['api', 'v1.0'].concat(
       uri.path.split('/').filter((segment) => segment)

extensions/ql-vscode/src/databases-ui.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import {
   Event,
   EventEmitter,
@@ -12,12 +12,10 @@ import {
 } from 'vscode';
 import * as fs from 'fs-extra';
 
-import * as cli from './cli';
 import {
   DatabaseChangedEvent,
   DatabaseItem,
   DatabaseManager,
-  getUpgradesDirectories,
 } from './databases';
 import {
   commandRunner,
@@ -25,15 +23,14 @@ import {
   ProgressCallback,
 } from './commandRunner';
 import {
-  getOnDiskWorkspaceFolders,
-  showAndLogErrorMessage,
   isLikelyDatabaseRoot,
-  isLikelyDbLanguageFolder
+  isLikelyDbLanguageFolder,
+  showAndLogErrorMessage
 } from './helpers';
 import { logger } from './logging';
 import { clearCacheInDatabase } from './run-queries';
 import * as qsClient from './queryserver-client';
-import { upgradeDatabase } from './upgrades';
+import { upgradeDatabaseExplicit } from './upgrades';
 import {
   importArchiveDatabase,
   promptImportInternetDatabase,
@@ -218,7 +215,6 @@ export class DatabaseUI extends DisposableObject {
   private treeDataProvider: DatabaseTreeDataProvider;
 
   public constructor(
-    private cliserver: cli.CodeQLCliServer,
     private databaseManager: DatabaseManager,
     private readonly queryServer: qsClient.QueryServerClient | undefined,
     private readonly storagePath: string,
@@ -383,8 +379,8 @@ export class DatabaseUI extends DisposableObject {
     let dbDirs = undefined;
 
     if (
-      !(await fs.pathExists(this.storagePath) ||
-        !(await fs.stat(this.storagePath)).isDirectory())
+      !(await fs.pathExists(this.storagePath)) ||
+      !(await fs.stat(this.storagePath)).isDirectory()
     ) {
       logger.log('Missing or invalid storage directory. Not trying to remove orphaned databases.');
       return;
@@ -427,9 +423,9 @@ export class DatabaseUI extends DisposableObject {
     if (failures.length) {
       const dirname = path.dirname(failures[0]);
       showAndLogErrorMessage(
-        `Failed to delete unused databases:\n  ${
-        failures.join('\n  ')
-        }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.`
+        `Failed to delete unused databases (${
+        failures.join(', ')
+        }).\nTo delete unused databases, please remove them manually from the storage folder ${dirname}.`
       );
     }
   };
@@ -540,25 +536,10 @@ export class DatabaseUI extends DisposableObject {
     }
 
     // Search for upgrade scripts in any workspace folders available
-    const searchPath: string[] = getOnDiskWorkspaceFolders();
 
-    const upgradeInfo = await this.cliserver.resolveUpgrades(
-      databaseItem.contents.dbSchemeUri.fsPath,
-      searchPath
-    );
-
-    const { scripts, finalDbscheme } = upgradeInfo;
-
-    if (finalDbscheme === undefined) {
-      throw new Error('Could not determine target dbscheme to upgrade to.');
-    }
-    const targetDbSchemeUri = Uri.file(finalDbscheme);
-
-    await upgradeDatabase(
+    await upgradeDatabaseExplicit(
       this.queryServer,
       databaseItem,
-      targetDbSchemeUri,
-      getUpgradesDirectories(scripts),
       progress,
       token
     );

extensions/ql-vscode/src/databases.ts

@@ -8,14 +8,14 @@ import {
   showAndLogErrorMessage,
   showAndLogWarningMessage,
   showAndLogInformationMessage,
-  isLikelyDatabaseRoot,
+  isLikelyDatabaseRoot
 } from './helpers';
 import {
   ProgressCallback,
   withProgress
 } from './commandRunner';
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { Logger, logger } from './logging';
 import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
 import { QueryServerClient } from './queryserver-client';
@@ -514,7 +514,10 @@ export class DatabaseManager extends DisposableObject {
   ) {
     super();
 
-    this.loadPersistedState();  // Let this run async.
+    qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
+
+    // Let this run async.
+    this.loadPersistedState();
   }
 
   public async openDatabase(
@@ -542,6 +545,22 @@ export class DatabaseManager extends DisposableObject {
     return databaseItem;
   }
 
+  private async reregisterDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken
+  ) {
+    let completed = 0;
+    await Promise.all(this._databaseItems.map(async (databaseItem) => {
+      await this.registerDatabase(progress, token, databaseItem);
+      completed++;
+      progress({
+        maxStep: this._databaseItems.length,
+        step: completed,
+        message: 'Re-registering databases'
+      });
+    }));
+  }
+
   private async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
     // The folder may already be in workspace state from a previous
     // session. If not, add it.
@@ -789,7 +808,7 @@ export class DatabaseManager extends DisposableObject {
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.qs.supportsDatabaseRegistration())) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
       const databases: Dataset[] = [{
         dbDir: dbItem.contents.datasetUri.fsPath,
         workingSet: 'default'
@@ -803,7 +822,7 @@ export class DatabaseManager extends DisposableObject {
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.qs.supportsDatabaseRegistration())) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
       const databases: Dataset[] = [{
         dbDir: dbItem.contents.datasetUri.fsPath,
         workingSet: 'default'
@@ -833,7 +852,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async getPrimaryLanguage(dbPath: string) {
-    if (!(await this.cli.supportsLanguageName())) {
+    if (!(await this.cli.cliConstraints.supportsLanguageName())) {
       // return undefined so that we recalculate on restart until the cli is at a version that
       // supports this feature. This recalculation is cheap since we avoid calling into the cli
       // unless we know it can return the langauges property.

extensions/ql-vscode/src/discovery.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { logger } from './logging';
 
 /**

extensions/ql-vscode/src/distribution.ts

@@ -225,9 +225,11 @@ export class DistributionManager implements DistributionProvider {
    *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
-  public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: ProgressCallback): Promise<void> {
-    return this.extensionSpecificDistributionManager!.installDistributionRelease(release, progressCallback);
+  public installExtensionManagedDistributionRelease(
+    release: Release,
+    progressCallback?: ProgressCallback
+  ): Promise<void> {
+    return this.extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
   }
 
   public get onDidChangeDistribution(): Event<void> | undefined {

extensions/ql-vscode/src/extension.ts

@@ -18,7 +18,7 @@ import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
 
 import { AstViewer } from './astViewer';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { CodeQLCliServer } from './cli';
+import { CodeQLCliServer, CliVersionConstraint } from './cli';
 import {
   CliConfigListener,
   DistributionConfigListener,
@@ -45,7 +45,6 @@ import {
   GithubRateLimitedError
 } from './distribution';
 import * as helpers from './helpers';
-import { commandRunner, commandRunnerWithProgress, ProgressCallback, ProgressUpdate, withProgress } from './commandRunner';
 import { assertNever } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
 import { InterfaceManager } from './interface';
@@ -60,6 +59,15 @@ import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
 import { CompareInterfaceManager } from './compare/compare-interface';
 import { gatherQlFiles } from './pure/files';
+import { initializeTelemetry } from './telemetry';
+import {
+  commandRunner,
+  commandRunnerWithProgress,
+  ProgressCallback,
+  withProgress,
+  ProgressUpdate
+} from './commandRunner';
+import { CodeQlStatusBarHandler } from './status-bar';
 
 /**
  * extension.ts
@@ -88,6 +96,9 @@ const errorStubs: Disposable[] = [];
  */
 let isInstallingOrUpdatingDistribution = false;
 
+const extensionId = 'GitHub.vscode-codeql';
+const extension = extensions.getExtension(extensionId);
+
 /**
  * If the user tries to execute vscode commands after extension activation is failed, give
  * a sensible error message.
@@ -98,8 +109,6 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
-  const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
-  const extension = extensions.getExtension(extensionId);
   if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
   }
@@ -139,10 +148,14 @@ export interface CodeQLExtensionInterface {
  * @returns CodeQLExtensionInterface
  */
 export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | {}> {
-  logger.log('Starting CodeQL extension');
+  logger.log(`Starting ${extensionId} extension`);
+  if (extension === undefined) {
+    throw new Error(`Can't find extension ${extensionId}`);
+  }
 
   const distributionConfigListener = new DistributionConfigListener();
   initializeLogging(ctx);
+  await initializeTelemetry(extension, ctx);
   languageSupport.install();
 
   ctx.subscriptions.push(distributionConfigListener);
@@ -278,14 +291,22 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<CodeQLExtensionInterface | {}> {
+  async function installOrUpdateThenTryActivate(
+    config: DistributionUpdateConfig
+  ): Promise<CodeQLExtensionInterface | {}> {
+
     await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
     let extensionInterface: CodeQLExtensionInterface | {} = {};
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      extensionInterface = await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(
+        ctx,
+        distributionManager,
+        distributionConfigListener
+      );
+
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = 'Install CodeQL CLI';
@@ -327,7 +348,8 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
 
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
-  distributionManager: DistributionManager
+  distributionManager: DistributionManager,
+  distributionConfigListener: DistributionConfigListener
 ): Promise<CodeQLExtensionInterface> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
@@ -348,6 +370,9 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cliServer);
 
+  const statusBar = new CodeQlStatusBarHandler(cliServer, distributionConfigListener);
+  ctx.subscriptions.push(statusBar);
+
   logger.log('Initializing query server client.');
   const qs = new qsClient.QueryServerClient(
     qlConfigurationListener,
@@ -369,7 +394,6 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(dbm);
   logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(
-    cliServer,
     dbm,
     qs,
     getContextStoragePath(ctx),
@@ -448,12 +472,30 @@ async function activateWithInstalledDistribution(
         progress,
         token
       );
-      const item = qhm.addQuery(info);
+      const item = qhm.buildCompletedQuery(info);
       await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      // The call to showResults potentially creates SARIF file;
-      // Update the tree item context value to allow viewing that
-      // SARIF file from context menu.
-      await qhm.refreshTreeView(item);
+      // Note we must update the query history view after showing results as the
+      // display and sorting might depend on the number of results
+      await qhm.addCompletedQuery(item);
+    }
+  }
+
+  async function openReferencedFile(
+    selectedQuery: Uri
+  ): Promise<void> {
+    if (qs !== undefined) {
+      if (await cliServer.cliConstraints.supportsResolveQlref()) {
+        const resolved = await cliServer.resolveQlref(selectedQuery.path);
+        const uri = Uri.file(resolved.resolvedPath);
+        await window.showTextDocument(uri, { preview: false });
+      } else {
+        helpers.showAndLogErrorMessage(
+          'Jumping from a .qlref file to the .ql file it references is not '
+          + 'supported with the CLI version you are running.\n'
+          + `Please upgrade your CLI to version ${
+          CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF
+          } or later to use this feature.`);
+      }
     }
   }
 
@@ -593,30 +635,44 @@ async function activateWithInstalledDistribution(
       }
     )
   );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.openReferencedFile',
+      openReferencedFile
+    )
+  );
 
   ctx.subscriptions.push(
-    commandRunner('codeQL.restartQueryServer', async () => {
-      await qs.restartQueryServer();
+    commandRunnerWithProgress('codeQL.restartQueryServer', async (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) => {
+      await qs.restartQueryServer(progress, token);
       helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
       });
+    }, {
+      title: 'Restarting Query Server'
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.chooseDatabaseFolder', (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseFolder(progress, token), {
+      title: 'Choose a Database from a Folder'
     })
   );
   ctx.subscriptions.push(
-    commandRunner('codeQL.chooseDatabaseFolder', (
+    commandRunnerWithProgress('codeQL.chooseDatabaseArchive', (
       progress: ProgressCallback,
       token: CancellationToken
     ) =>
-      databaseUI.handleChooseDatabaseFolder(progress, token)
-    )
-  );
-  ctx.subscriptions.push(
-    commandRunner('codeQL.chooseDatabaseArchive', (
-      progress: ProgressCallback,
-      token: CancellationToken
-    ) =>
-      databaseUI.handleChooseDatabaseArchive(progress, token)
-    )
+      databaseUI.handleChooseDatabaseArchive(progress, token), {
+      title: 'Choose a Database from an Archive'
+    })
   );
   ctx.subscriptions.push(
     commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
@@ -640,6 +696,10 @@ async function activateWithInstalledDistribution(
       })
   );
 
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openDocumentation', async () =>
+      env.openExternal(Uri.parse('https://codeql.github.com/docs/'))));
+
   logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
@@ -656,13 +716,18 @@ async function activateWithInstalledDistribution(
   );
 
   const astViewer = new AstViewer();
+  const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm);
+
   ctx.subscriptions.push(astViewer);
   ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
     progress: ProgressCallback,
     token: CancellationToken
   ) => {
-    const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)
-      .provideAst(window.activeTextEditor?.document);
+    const ast = await templateProvider.provideAst(
+      progress,
+      token,
+      window.activeTextEditor?.document,
+    );
     if (ast) {
       astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
     }

extensions/ql-vscode/src/helpers.ts

@@ -4,8 +4,10 @@ import * as yaml from 'js-yaml';
 import * as path from 'path';
 import {
   ExtensionContext,
+  Uri,
   window as Window,
-  workspace
+  workspace,
+  env
 } from 'vscode';
 import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
@@ -27,8 +29,13 @@ export async function showAndLogErrorMessage(message: string, {
   items = [] as string[],
   fullMessage = undefined as (string | undefined)
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage, fullMessage);
+  return internalShowAndLog(dropLinesExceptInitial(message), items, outputLogger, Window.showErrorMessage, fullMessage);
 }
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join('\n');
+}
+
 /**
  * Show a warning message and log it to the console
  *
@@ -80,17 +87,61 @@ async function internalShowAndLog(
 
 /**
  * Opens a modal dialog for the user to make a yes/no choice.
- * @param message The message to show.
  *
- * @return `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
  */
-export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
+export async function showBinaryChoiceDialog(message: string, modal = true): Promise<boolean | undefined> {
   const yesItem = { title: 'Yes', isCloseAffordance: false };
   const noItem = { title: 'No', isCloseAffordance: true };
-  const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
+  const chosenItem = await Window.showInformationMessage(message, { modal }, yesItem, noItem);
+  if (!chosenItem) {
+    return undefined;
+  }
   return chosenItem?.title === yesItem.title;
 }
 
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(message: string, url: string): Promise<boolean | undefined> {
+  const urlItem = { title: 'More Information', isCloseAffordance: false };
+  const yesItem = { title: 'Yes', isCloseAffordance: false };
+  const noItem = { title: 'No', isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await Window.showInformationMessage(message, { modal: true }, urlItem, yesItem, noItem);
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
 /**
  * Show an information message with a customisable action.
  * @param message The message to show.
@@ -250,19 +301,19 @@ export async function getPrimaryDbscheme(datasetFolder: string): Promise<string>
  * A cached mapping from strings to value of type U.
  */
 export class CachedOperation<U> {
-  private readonly operation: (t: string) => Promise<U>;
+  private readonly operation: (t: string, ...args: any[]) => Promise<U>;
   private readonly cached: Map<string, U>;
   private readonly lru: string[];
   private readonly inProgressCallbacks: Map<string, [(u: U) => void, (reason?: any) => void][]>;
 
-  constructor(operation: (t: string) => Promise<U>, private cacheSize = 100) {
+  constructor(operation: (t: string, ...args: any[]) => Promise<U>, private cacheSize = 100) {
     this.operation = operation;
     this.lru = [];
     this.inProgressCallbacks = new Map<string, [(u: U) => void, (reason?: any) => void][]>();
     this.cached = new Map<string, U>();
   }
 
-  async get(t: string): Promise<U> {
+  async get(t: string, ...args: any[]): Promise<U> {
     // Try and retrieve from the cache
     const fromCache = this.cached.get(t);
     if (fromCache !== undefined) {
@@ -283,7 +334,7 @@ export class CachedOperation<U> {
     const callbacks: [(u: U) => void, (reason?: any) => void][] = [];
     this.inProgressCallbacks.set(t, callbacks);
     try {
-      const result = await this.operation(t);
+      const result = await this.operation(t, ...args);
       callbacks.forEach(f => f[0](result));
       this.inProgressCallbacks.delete(t);
       if (this.lru.length > this.cacheSize) {
@@ -316,7 +367,7 @@ export class CachedOperation<U> {
  * `cli.CodeQLCliServer.resolveDatabase` and use the first entry in the
  * `languages` property.
  *
- * @see cli.CodeQLCliServer.supportsLanguageName
+ * @see cli.CliVersionConstraint.supportsLanguageName
  * @see cli.CodeQLCliServer.resolveDatabase
  */
 const dbSchemeToLanguage = {

extensions/ql-vscode/src/interface.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as Sarif from 'sarif';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as vscode from 'vscode';
 import {
   Diagnostic,
@@ -30,8 +30,8 @@ import {
   RawResultsSortState,
 } from './pure/interface-types';
 import { Logger } from './logging';
-import { commandRunner } from './commandRunner';
 import * as messages from './pure/messages';
+import { commandRunner } from './commandRunner';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
@@ -137,9 +137,11 @@ export class InterfaceManager extends DisposableObject {
       this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
         if (kind === DatabaseEventKind.Remove) {
           this._diagnosticCollection.clear();
-          this.postMessage({
-            t: 'untoggleShowProblems'
-          });
+          if (this.isShowingPanel()) {
+            this.postMessage({
+              t: 'untoggleShowProblems'
+            });
+          }
         }
       })
     );
@@ -149,6 +151,10 @@ export class InterfaceManager extends DisposableObject {
     this.postMessage({ t: 'navigatePath', direction });
   }
 
+  private isShowingPanel() {
+    return !!this._panel;
+  }
+
   // Returns the webview panel, creating it if it doesn't already
   // exist.
   getPanel(): vscode.WebviewPanel {
@@ -168,6 +174,7 @@ export class InterfaceManager extends DisposableObject {
           ]
         }
       ));
+
       this._panel.onDidDispose(
         () => {
           this._panel = undefined;
@@ -236,61 +243,67 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
-    switch (msg.t) {
-      case 'viewSourceFile': {
-        await jumpToLocation(msg, this.databaseManager, this.logger);
-        break;
-      }
-      case 'toggleDiagnostics': {
-        if (msg.visible) {
-          const databaseItem = this.databaseManager.findDatabaseItem(
-            Uri.parse(msg.databaseUri)
-          );
-          if (databaseItem !== undefined) {
-            await this.showResultsAsDiagnostics(
-              msg.origResultsPaths,
-              msg.metadata,
-              databaseItem
+    try {
+      switch (msg.t) {
+        case 'viewSourceFile': {
+          await jumpToLocation(msg, this.databaseManager, this.logger);
+          break;
+        }
+        case 'toggleDiagnostics': {
+          if (msg.visible) {
+            const databaseItem = this.databaseManager.findDatabaseItem(
+              Uri.parse(msg.databaseUri)
+            );
+            if (databaseItem !== undefined) {
+              await this.showResultsAsDiagnostics(
+                msg.origResultsPaths,
+                msg.metadata,
+                databaseItem
+              );
+            }
+          } else {
+            // TODO: Only clear diagnostics on the same database.
+            this._diagnosticCollection.clear();
+          }
+          break;
+        }
+        case 'resultViewLoaded':
+          this._panelLoaded = true;
+          this._panelLoadedCallBacks.forEach((cb) => cb());
+          this._panelLoadedCallBacks = [];
+          break;
+        case 'changeSort':
+          await this.changeRawSortState(msg.resultSetName, msg.sortState);
+          break;
+        case 'changeInterpretedSort':
+          await this.changeInterpretedSortState(msg.sortState);
+          break;
+        case 'changePage':
+          if (msg.selectedTable === ALERTS_TABLE_NAME) {
+            await this.showPageOfInterpretedResults(msg.pageNumber);
+          }
+          else {
+            await this.showPageOfRawResults(
+              msg.selectedTable,
+              msg.pageNumber,
+              // When we are in an unsorted state, we guarantee that
+              // sortedResultsInfo doesn't have an entry for the current
+              // result set. Use this to determine whether or not we use
+              // the sorted bqrs file.
+              this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
             );
           }
-        } else {
-          // TODO: Only clear diagnostics on the same database.
-          this._diagnosticCollection.clear();
-        }
-        break;
+          break;
+        case 'openFile':
+          await this.openFile(msg.filePath);
+          break;
+        default:
+          assertNever(msg);
       }
-      case 'resultViewLoaded':
-        this._panelLoaded = true;
-        this._panelLoadedCallBacks.forEach((cb) => cb());
-        this._panelLoadedCallBacks = [];
-        break;
-      case 'changeSort':
-        await this.changeRawSortState(msg.resultSetName, msg.sortState);
-        break;
-      case 'changeInterpretedSort':
-        await this.changeInterpretedSortState(msg.sortState);
-        break;
-      case 'changePage':
-        if (msg.selectedTable === ALERTS_TABLE_NAME) {
-          await this.showPageOfInterpretedResults(msg.pageNumber);
-        }
-        else {
-          await this.showPageOfRawResults(
-            msg.selectedTable,
-            msg.pageNumber,
-            // When we are in an unsorted state, we guarantee that
-            // sortedResultsInfo doesn't have an entry for the current
-            // result set. Use this to determine whether or not we use
-            // the sorted bqrs file.
-            this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
-          );
-        }
-        break;
-      case 'openFile':
-        await this.openFile(msg.filePath);
-        break;
-      default:
-        assertNever(msg);
+    } catch (e) {
+      showAndLogErrorMessage(e.message, {
+        fullMessage: e.stack
+      });
     }
   }
 
@@ -394,6 +407,7 @@ export class InterfaceManager extends DisposableObject {
       }
     );
     const resultSet = transformBqrsResultSet(schema, chunk);
+    results.setResultCount(interpretationPage?.numTotalResults || resultSet.schema.rows);
     const parsedResultSets: ParsedResultSets = {
       pageNumber: 0,
       pageSize,
@@ -540,24 +554,26 @@ export class InterfaceManager extends DisposableObject {
     sourceInfo: cli.SourceInfo | undefined,
     sourceLocationPrefix: string,
     sortState: InterpretedResultsSortState | undefined
-  ): Promise<Interpretation> {
+  ): Promise<Interpretation | undefined> {
+    if (!resultsPaths) {
+      this.logger.log('No results path. Cannot display interpreted results.');
+      return undefined;
+    }
+
     const sarif = await interpretResults(
       this.cliServer,
       metadata,
       resultsPaths,
       sourceInfo
     );
+
     sarif.runs.forEach(run => {
       if (run.results !== undefined) {
         sortInterpretedResults(run.results, sortState);
       }
     });
 
-    const numTotalResults = (() => {
-      if (sarif.runs.length === 0) return 0;
-      if (sarif.runs[0].results === undefined) return 0;
-      return sarif.runs[0].results.length;
-    })();
+    const numTotalResults = sarif.runs[0]?.results?.length || 0;
 
     const interpretation: Interpretation = {
       sarif,
@@ -626,8 +642,8 @@ export class InterfaceManager extends DisposableObject {
       } catch (e) {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
-        this.logger.log(
-          `Exception during results interpretation: ${e.message}. Will show raw results instead.`
+        showAndLogErrorMessage(
+          `Showing raw results instead of interpreted ones due to an error. ${e.message}`
         );
       }
     }
@@ -659,6 +675,10 @@ export class InterfaceManager extends DisposableObject {
       undefined
     );
 
+    if (!interpretation) {
+      return;
+    }
+
     try {
       await this.showProblemResultsAsDiagnostics(interpretation, database);
     } catch (e) {

extensions/ql-vscode/src/logging.ts

@@ -1,5 +1,5 @@
 import { window as Window, OutputChannel, Progress, Disposable } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 

extensions/ql-vscode/src/pure/disposable-object.ts

@@ -1,4 +1,11 @@
-import { Disposable } from 'vscode';
+
+// Avoid explicitly referencing Disposable type in vscode.
+// This file cannot have dependencies on the vscode API.
+interface Disposable {
+  dispose(): any;
+}
+
+export type DisposeHandler = (disposable: Disposable) => void;
 
 /**
  * Base class to make it easier to implement a `Disposable` that owns other disposable object.
@@ -40,21 +47,39 @@ export abstract class DisposableObject implements Disposable {
    * @param obj The object to stop tracking.
    */
   protected disposeAndStopTracking(obj: Disposable): void {
-    if (obj !== undefined) {
-      this.tracked!.delete(obj);
+    if (obj && this.tracked) {
+      this.tracked.delete(obj);
       obj.dispose();
     }
   }
 
-  public dispose() {
+  /**
+   * Dispose this object and all contained objects
+   *
+   * @param disposeHandler An optional dispose handler that gets
+   *      passed each element to dispose. The dispose handler
+   *      can choose how (and if) to dispose the object. The
+   *      primary usage is for tests that should not dispose
+   *      all items of a disposable.
+   */
+  public dispose(disposeHandler?: DisposeHandler) {
     if (this.tracked !== undefined) {
       for (const trackedObject of this.tracked.values()) {
-        trackedObject.dispose();
+        if (disposeHandler) {
+          disposeHandler(trackedObject);
+        } else {
+          trackedObject.dispose();
+        }
       }
       this.tracked = undefined;
     }
     while (this.disposables.length > 0) {
-      this.disposables.pop()!.dispose();
+      const disposable = this.disposables.pop()!;
+      if (disposeHandler) {
+        disposeHandler(disposable);
+      } else {
+        disposable.dispose();
+      }
     }
   }
 }

extensions/ql-vscode/src/pure/interface-types.ts

@@ -34,6 +34,7 @@ export interface QueryMetadata {
   description?: string;
   id?: string;
   kind?: string;
+  scored?: string;
 }
 
 export interface PreviousExecution {

extensions/ql-vscode/src/pure/messages.ts

@@ -413,6 +413,20 @@ export interface CompileUpgradeParams {
   singleFileUpgrades: true;
 }
 
+/**
+ * Parameters for compiling an upgrade.
+ */
+export interface CompileUpgradeSequenceParams {
+  /**
+   * The sequence of upgrades to compile
+   */
+  upgradePaths: string[];
+  /**
+   * A directory to store parts of the compiled upgrade
+   */
+  upgradeTempDir: string;
+}
+
 /**
  * Parameters describing an upgrade
  */
@@ -460,6 +474,19 @@ export interface CompileUpgradeResult {
    */
   error?: string;
 }
+
+export interface CompileUpgradeSequenceResult {
+  /**
+   * The compiled upgrades as a single file.
+   */
+  compiledUpgrade?: string;
+  /**
+   * Any errors that occurred when checking the scripts.
+   */
+  error?: string;
+}
+
+
 /**
  * A description of a upgrade process
  */
@@ -498,7 +525,7 @@ export interface UpgradeDescription {
 }
 
 
-export type CompiledUpgrades = MultiFileCompiledUpgrades | SingleFileCompiledUpgrade
+export type CompiledUpgrades = MultiFileCompiledUpgrades | SingleFileCompiledUpgrades
 
 /**
  * The parts shared by all compiled upgrades
@@ -543,7 +570,7 @@ interface MultiFileCompiledUpgrades extends CompiledUpgradesBase {
  * A compiled upgrade.
  * The upgrade is in a single file.
  */
-export interface SingleFileCompiledUpgrade extends CompiledUpgradesBase {
+export interface SingleFileCompiledUpgrades extends CompiledUpgradesBase {
   /**
    * The steps in the upgrade path
    */
@@ -696,6 +723,10 @@ export interface QueryToRun {
    * A uri pointing to the qlo to run.
    */
   qlo: string;
+  /**
+   * A uri pointing to the compiled upgrade file.
+   */
+  compiledUpgrade?: string;
   /**
    * The path where we should save this queries results
    */
@@ -972,7 +1003,10 @@ export const checkUpgrade = new rpc.RequestType<WithProgressId<UpgradeParams>, C
  * Compile an upgrade script to upgrade a dataset.
  */
 export const compileUpgrade = new rpc.RequestType<WithProgressId<CompileUpgradeParams>, CompileUpgradeResult, void, void>('compilation/compileUpgrade');
-
+/**
+ * Compile an upgrade script to upgrade a dataset.
+ */
+export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<CompileUpgradeSequenceParams>, CompileUpgradeSequenceResult, void, void>('compilation/compileUpgradeSequence');
 
 /**
  * Clear the cache of a dataset

extensions/ql-vscode/src/query-history.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as vscode from 'vscode';
-import { window as Window } from 'vscode';
+import { window as Window, env } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
@@ -10,13 +10,12 @@ import {
   showAndLogWarningMessage,
   showBinaryChoiceDialog
 } from './helpers';
-import {
-  commandRunner
-} from './commandRunner';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
+import { commandRunner } from './commandRunner';
+import { assertNever } from './pure/helpers-pure';
 
 /**
  * query-history.ts
@@ -60,10 +59,21 @@ const SHOW_QUERY_TEXT_QUICK_EVAL_MSG = `\
  */
 const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
 
+enum SortOrder {
+  NameAsc = 'NameAsc',
+  NameDesc = 'NameDesc',
+  DateAsc = 'DateAsc',
+  DateDesc = 'DateDesc',
+  CountAsc = 'CountAsc',
+  CountDesc = 'CountDesc',
+}
+
 /**
  * Tree data provider for the query history view.
  */
 export class HistoryTreeDataProvider extends DisposableObject {
+  private _sortOrder = SortOrder.DateAsc;
+
   private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
 
   readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
@@ -113,7 +123,24 @@ export class HistoryTreeDataProvider extends DisposableObject {
   getChildren(
     element?: CompletedQuery
   ): vscode.ProviderResult<CompletedQuery[]> {
-    return element ? [] : this.history;
+    return element ? [] : this.history.sort((q1, q2) => {
+      switch (this.sortOrder) {
+        case SortOrder.NameAsc:
+          return q1.toString().localeCompare(q2.toString(), env.language);
+        case SortOrder.NameDesc:
+          return q2.toString().localeCompare(q1.toString(), env.language);
+        case SortOrder.DateAsc:
+          return q1.date.getTime() - q2.date.getTime();
+        case SortOrder.DateDesc:
+          return q2.date.getTime() - q1.date.getTime();
+        case SortOrder.CountAsc:
+          return q1.resultCount - q2.resultCount;
+        case SortOrder.CountDesc:
+          return q2.resultCount - q1.resultCount;
+        default:
+          assertNever(this.sortOrder);
+      }
+    });
   }
 
   getParent(_element: CompletedQuery): vscode.ProviderResult<CompletedQuery> {
@@ -159,6 +186,15 @@ export class HistoryTreeDataProvider extends DisposableObject {
   find(queryId: number): CompletedQuery | undefined {
     return this.allHistory.find((query) => query.query.queryID === queryId);
   }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire();
+  }
 }
 
 /**
@@ -226,6 +262,24 @@ export class QueryHistoryManager extends DisposableObject {
         this.handleRemoveHistoryItem.bind(this)
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByName',
+        this.handleSortByName.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByDate',
+        this.handleSortByDate.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByCount',
+        this.handleSortByCount.bind(this)
+      )
+    );
     this.push(
       commandRunner(
         'codeQLQueryHistory.setLabel',
@@ -252,8 +306,14 @@ export class QueryHistoryManager extends DisposableObject {
     );
     this.push(
       commandRunner(
-        'codeQLQueryHistory.viewSarif',
-        this.handleViewSarif.bind(this)
+        'codeQLQueryHistory.viewCsvResults',
+        this.handleViewCsvResults.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.viewSarifResults',
+        this.handleViewSarifResults.bind(this)
       )
     );
     this.push(
@@ -347,6 +407,30 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
+  async handleSortByName() {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  }
+
+  async handleSortByDate() {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAsc;
+    }
+  }
+
+  async handleSortByCount() {
+    if (this.treeDataProvider.sortOrder === SortOrder.CountAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.CountDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.CountAsc;
+    }
+  }
+
   async handleSetLabel(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
@@ -364,7 +448,12 @@ export class QueryHistoryManager extends DisposableObject {
     if (response !== undefined) {
       // Interpret empty string response as 'go back to using default'
       singleItem.options.label = response === '' ? undefined : response;
-      this.treeDataProvider.refresh(singleItem);
+      if (this.treeDataProvider.sortOrder === SortOrder.NameAsc ||
+        this.treeDataProvider.sortOrder === SortOrder.NameDesc) {
+        this.treeDataProvider.refresh();
+      } else {
+        this.treeDataProvider.refresh(singleItem);
+      }
     }
   }
 
@@ -461,7 +550,7 @@ export class QueryHistoryManager extends DisposableObject {
     await vscode.window.showTextDocument(doc, { preview: false });
   }
 
-  async handleViewSarif(
+  async handleViewSarifResults(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
   ) {
@@ -482,6 +571,19 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
+  async handleViewCsvResults(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    await this.tryOpenExternalFile(
+      await singleItem.query.ensureCsvProduced(this.qs)
+    );
+  }
+
   async handleViewDil(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[],
@@ -513,11 +615,14 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
-  addQuery(info: QueryWithResults): CompletedQuery {
+  buildCompletedQuery(info: QueryWithResults): CompletedQuery {
     const item = new CompletedQuery(info, this.queryHistoryConfigListener);
+    return item;
+  }
+
+  addCompletedQuery(item: CompletedQuery) {
     this.treeDataProvider.pushQuery(item);
     this.updateTreeViewSelectionIfVisible();
-    return item;
   }
 
   find(queryId: number): CompletedQuery | undefined {
@@ -692,3 +797,4 @@ the file in the file explorer and dragging it into the workspace.`
     this.treeDataProvider.refresh(completedQuery);
   }
 }
+

extensions/ql-vscode/src/query-results.ts

@@ -11,12 +11,14 @@ import { QueryHistoryConfig } from './config';
 import { QueryHistoryItemOptions } from './query-history';
 
 export class CompletedQuery implements QueryWithResults {
+  readonly date: Date;
   readonly time: string;
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
   readonly database: DatabaseInfo;
   readonly logFileLocation?: string;
   options: QueryHistoryItemOptions;
+  resultCount: number;
   dispose: () => void;
 
   /**
@@ -44,8 +46,14 @@ export class CompletedQuery implements QueryWithResults {
     this.options = evaluation.options;
     this.dispose = evaluation.dispose;
 
-    this.time = new Date().toLocaleString(env.language);
+    this.date = new Date();
+    this.time = this.date.toLocaleString(env.language);
     this.sortedResultsInfo = new Map();
+    this.resultCount = 0;
+  }
+
+  setResultCount(value: number) {
+    this.resultCount = value;
   }
 
   get databaseName(): string {
@@ -80,11 +88,12 @@ export class CompletedQuery implements QueryWithResults {
   }
 
   interpolate(template: string): string {
-    const { databaseName, queryName, time, statusString } = this;
+    const { databaseName, queryName, time, resultCount, statusString } = this;
     const replacements: { [k: string]: string } = {
       t: time,
       q: queryName,
       d: databaseName,
+      r: resultCount.toString(),
       s: statusString,
       '%': '%',
     };
@@ -170,17 +179,20 @@ export async function interpretResults(
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }
+  return await server.interpretBqrs(ensureMetadataIsComplete(metadata), resultsPath, interpretedResultsPath, sourceInfo);
+}
+
+export function ensureMetadataIsComplete(metadata: QueryMetadata | undefined) {
   if (metadata === undefined) {
     throw new Error('Can\'t interpret results without query metadata');
   }
-  let { kind, id } = metadata;
-  if (kind === undefined) {
+  if (metadata.kind === undefined) {
     throw new Error('Can\'t interpret results without query metadata including kind');
   }
-  if (id === undefined) {
+  if (metadata.id === undefined) {
     // Interpretation per se doesn't really require an id, but the
     // SARIF format does, so in the absence of one, we use a dummy id.
-    id = 'dummy-id';
+    metadata.id = 'dummy-id';
   }
-  return await server.interpretBqrs({ kind, id }, resultsPath, interpretedResultsPath, sourceInfo);
+  return metadata;
 }

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,14 +1,14 @@
 import * as cp from 'child_process';
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { Disposable } from 'vscode';
-import { CancellationToken, createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
+import { DisposableObject } from './pure/disposable-object';
+import { Disposable, CancellationToken, commands } from 'vscode';
+import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
 import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
 import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
 import * as messages from './pure/messages';
-import { SemVer } from 'semver';
+import { ProgressCallback, ProgressTask } from './commandRunner';
 
 type ServerOpts = {
   logger: Logger;
@@ -49,17 +49,22 @@ type WithProgressReporting = (task: (progress: ProgressReporter, token: Cancella
  */
 export class QueryServerClient extends DisposableObject {
 
-  /**
-   * Query Server version where database registration was introduced
-   */
-  private static VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
-
   serverProcess?: ServerProcess;
   evaluationResultCallbacks: { [key: number]: (res: EvaluationResult) => void };
   progressCallbacks: { [key: number]: ((res: ProgressMessage) => void) | undefined };
   nextCallback: number;
   nextProgress: number;
   withProgressReporting: WithProgressReporting;
+
+  private readonly queryServerStartListeners = [] as ProgressTask<void>[];
+
+  // Can't use standard vscode EventEmitter here since they do not cause the calling
+  // function to fail if one of the event handlers fail. This is something that
+  // we need here.
+  readonly onDidStartQueryServer = (e: ProgressTask<void>) => {
+    this.queryServerStartListeners.push(e);
+  }
+
   public activeQueryName: string | undefined;
 
   constructor(
@@ -71,10 +76,8 @@ export class QueryServerClient extends DisposableObject {
     super();
     // When the query server configuration changes, restart the query server.
     if (config.onDidChangeConfiguration !== undefined) {
-      this.push(config.onDidChangeConfiguration(async () => {
-        this.logger.log('Restarting query server due to configuration changes...');
-        await this.restartQueryServer();
-      }, this));
+      this.push(config.onDidChangeConfiguration(() =>
+        commands.executeCommand('codeQL.restartQueryServer')));
     }
     this.withProgressReporting = withProgressReporting;
     this.nextCallback = 0;
@@ -97,9 +100,19 @@ export class QueryServerClient extends DisposableObject {
   }
 
   /** Restarts the query server by disposing of the current server process and then starting a new one. */
-  async restartQueryServer(): Promise<void> {
+  async restartQueryServer(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<void> {
     this.stopQueryServer();
     await this.startQueryServer();
+
+    // Ensure we await all responses from event handlers so that
+    // errors can be properly reported to the user.
+    await Promise.all(this.queryServerStartListeners.map(handler => handler(
+      progress,
+      token
+    )));
   }
 
   showLog(): void {
@@ -117,7 +130,16 @@ export class QueryServerClient extends DisposableObject {
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
 
-    if (await this.supportsDatabaseRegistration()) {
+    if (this.config.saveCache) {
+      args.push('--save-cache');
+    }
+
+    if (this.config.cacheSize > 0) {
+      args.push('--max-disk-cache');
+      args.push(this.config.cacheSize.toString());
+    }
+
+    if (await this.cliServer.cliConstraints.supportsDatabaseRegistration()) {
       args.push('--require-db-registration');
     }
 
@@ -174,10 +196,6 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
-  async supportsDatabaseRegistration() {
-    return (await this.cliServer.getVersion()).compare(QueryServerClient.VERSION_WITH_DB_REGISTRATION) >= 0;
-  }
-
   registerCallback(callback: (res: EvaluationResult) => void): number {
     const id = this.nextCallback++;
     this.evaluationResultCallbacks[id] = callback;

extensions/ql-vscode/src/quick-query.ts

@@ -1,16 +1,20 @@
 import * as fs from 'fs-extra';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
-import { CancellationToken, ExtensionContext, window as Window, workspace, Uri } from 'vscode';
+import {
+  CancellationToken,
+  ExtensionContext,
+  window as Window,
+  workspace,
+  Uri
+} from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import { CodeQLCliServer } from './cli';
 import { DatabaseUI } from './databases-ui';
-import { logger } from './logging';
 import {
   getInitialQueryContents,
   getPrimaryDbscheme,
   getQlPackForDbscheme,
-  showAndLogErrorMessage,
   showBinaryChoiceDialog,
 } from './helpers';
 import {
@@ -21,23 +25,35 @@ import {
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
 const QUICK_QUERY_WORKSPACE_FOLDER_NAME = 'Quick Queries';
+const QLPACK_FILE_HEADER = '# This is an automatically generated file.\n\n';
 
 export function isQuickQueryPath(queryPath: string): boolean {
   return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
 }
 
-function getQuickQueriesDir(ctx: ExtensionContext): string {
+async function getQuickQueriesDir(ctx: ExtensionContext): Promise<string> {
   const storagePath = ctx.storagePath;
   if (storagePath === undefined) {
     throw new Error('Workspace storage path is undefined');
   }
   const queriesPath = path.join(storagePath, QUICK_QUERIES_DIR_NAME);
-  fs.ensureDir(queriesPath, { mode: 0o700 });
+  await fs.ensureDir(queriesPath, { mode: 0o700 });
   return queriesPath;
 }
 
+function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
+  workspace.updateWorkspaceFolders(
+    index,
+    len,
+    { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
+  );
+}
 
-
+function findExistingQuickQueryEditor() {
+  return Window.visibleTextEditors.find(editor =>
+    path.basename(editor.document.uri.fsPath) === QUICK_QUERY_QUERY_NAME
+  );
+}
 
 /**
  * Show a buffer the user can enter a simple query into.
@@ -50,26 +66,18 @@ export async function displayQuickQuery(
   token: CancellationToken
 ) {
 
-  function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
-    workspace.updateWorkspaceFolders(
-      index,
-      len,
-      { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
-    );
-  }
-
   try {
-    const workspaceFolders = workspace.workspaceFolders || [];
-    const queriesDir = await getQuickQueriesDir(ctx);
-
     // If there is already a quick query open, don't clobber it, just
     // show it.
-    const existing = workspace.textDocuments.find(doc => path.basename(doc.uri.fsPath) === QUICK_QUERY_QUERY_NAME);
-    if (existing !== undefined) {
-      Window.showTextDocument(existing);
+    const existing = findExistingQuickQueryEditor();
+    if (existing) {
+      await Window.showTextDocument(existing.document);
       return;
     }
 
+    const workspaceFolders = workspace.workspaceFolders || [];
+    const queriesDir = await getQuickQueriesDir(ctx);
+
     // We need to have a multi-root workspace to make quick query work
     // at all. Changing the workspace from single-root to multi-root
     // causes a restart of the whole extension host environment, so we
@@ -88,10 +96,11 @@ export async function displayQuickQuery(
     }
 
     const index = workspaceFolders.findIndex(folder => folder.name === QUICK_QUERY_WORKSPACE_FOLDER_NAME);
-    if (index === -1)
+    if (index === -1) {
       updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
-    else
+    } else {
       updateQuickQueryDir(queriesDir, index, 1);
+    }
 
     // We're going to infer which qlpack to use from the current database
     const dbItem = await databaseUI.getDatabaseItem(progress, token);
@@ -102,31 +111,38 @@ export async function displayQuickQuery(
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
     const dbscheme = await getPrimaryDbscheme(datasetFolder);
     const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
-
-    const quickQueryQlpackYaml: any = {
-      name: 'quick-query',
-      version: '1.0.0',
-      libraryPathDependencies: [qlpack]
-    };
-
-    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
     const qlPackFile = path.join(queriesDir, 'qlpack.yml');
-    await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
-    await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8');
-    Window.showTextDocument(await workspace.openTextDocument(qlFile));
-  }
+    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const shouldRewrite = await checkShouldRewrite(qlPackFile, qlpack);
 
-  // TODO: clean up error handling for top-level commands like this
-  catch (e) {
-    if (e instanceof UserCancellationException) {
-      logger.log(e.message);
+    // Only rewrite the qlpack file if the database has changed
+    if (shouldRewrite) {
+      const quickQueryQlpackYaml: any = {
+        name: 'quick-query',
+        version: '1.0.0',
+        libraryPathDependencies: [qlpack]
+      };
+      await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.safeDump(quickQueryQlpackYaml), 'utf8');
     }
-    else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      logger.log(e.message);
+
+    if (shouldRewrite || !(await fs.pathExists(qlFile))) {
+      await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
     }
-    else if (e instanceof Error)
-      showAndLogErrorMessage(e.message);
-    else
+
+    await Window.showTextDocument(await workspace.openTextDocument(qlFile));
+  } catch (e) {
+    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      throw new UserCancellationException(e.message);
+    } else {
       throw e;
+    }
   }
 }
+
+async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
+  if (!(await fs.pathExists(qlPackFile))) {
+    return true;
+  }
+  const qlPackContents: any = yaml.safeLoad(await fs.readFile(qlPackFile, 'utf8'));
+  return qlPackContents.libraryPathDependencies?.[0] !== newDependency;
+}

extensions/ql-vscode/src/run-queries.ts

@@ -1,7 +1,7 @@
 import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import * as tmp from 'tmp';
+import * as tmp from 'tmp-promise';
 import {
   CancellationToken,
   ConfigurationTarget,
@@ -14,7 +14,7 @@ import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 
 import * as cli from './cli';
 import * as config from './config';
-import { DatabaseItem, getUpgradesDirectories } from './databases';
+import { DatabaseItem } from './databases';
 import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
 import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { DatabaseInfo, QueryMetadata, ResultsPaths } from './pure/interface-types';
@@ -23,7 +23,8 @@ import * as messages from './pure/messages';
 import { QueryHistoryItemOptions } from './query-history';
 import * as qsClient from './queryserver-client';
 import { isQuickQueryPath } from './quick-query';
-import { upgradeDatabase } from './upgrades';
+import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
+import { ensureMetadataIsComplete } from './query-results';
 
 /**
  * run-queries.ts
@@ -53,6 +54,7 @@ export class QueryInfo {
 
   readonly compiledQueryPath: string;
   readonly dilPath: string;
+  readonly csvPath: string;
   readonly resultsPaths: ResultsPaths;
   readonly dataset: Uri; // guarantee the existence of a well-defined dataset dir at this point
   readonly queryID: number;
@@ -68,6 +70,7 @@ export class QueryInfo {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
     this.dilPath = path.join(tmpDir.name, `results${this.queryID}.dil`);
+    this.csvPath = path.join(tmpDir.name, `results${this.queryID}.csv`);
     this.resultsPaths = {
       resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`),
       interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`)
@@ -80,6 +83,7 @@ export class QueryInfo {
 
   async run(
     qs: qsClient.QueryServerClient,
+    upgradeQlo: string | undefined,
     progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.EvaluationResult> {
@@ -90,6 +94,7 @@ export class QueryInfo {
     const queryToRun: messages.QueryToRun = {
       resultsPath: this.resultsPaths.resultsPath,
       qlo: Uri.file(this.compiledQueryPath).toString(),
+      compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
       allowUnknownTemplates: true,
       templateValues: this.templates,
       id: callbackId,
@@ -164,7 +169,15 @@ export class QueryInfo {
     if (!hasMetadataFile) {
       logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
     }
-    return hasMetadataFile;
+
+    const hasKind = !!this.metadata?.kind;
+    if (!hasKind) {
+      logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
+    }
+
+    const isTable = hasKind && this.metadata?.kind === 'table';
+
+    return hasMetadataFile && hasKind && !isTable;
   }
 
   /**
@@ -181,6 +194,13 @@ export class QueryInfo {
     return fs.pathExists(this.dilPath);
   }
 
+  /**
+   * Holds if this query already has CSV results produced
+   */
+  async hasCsv(): Promise<boolean> {
+    return fs.pathExists(this.csvPath);
+  }
+
   async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
     if (await this.hasDil()) {
       return this.dilPath;
@@ -196,8 +216,27 @@ export class QueryInfo {
     return this.dilPath;
   }
 
+  async ensureCsvProduced(qs: qsClient.QueryServerClient): Promise<string> {
+    if (await this.hasCsv()) {
+      return this.csvPath;
+    }
+
+    let sourceInfo;
+    if (this.dbItem.sourceArchive !== undefined) {
+      sourceInfo = {
+        sourceArchive: this.dbItem.sourceArchive.fsPath,
+        sourceLocationPrefix: await this.dbItem.getSourceLocationPrefix(
+          qs.cliServer
+        ),
+      };
+    }
+
+    await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
+    return this.csvPath;
+  }
 }
 
+
 export interface QueryWithResults {
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
@@ -292,7 +331,7 @@ async function checkDbschemeCompatibility(
   const searchPath = getOnDiskWorkspaceFolders();
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
-    const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
+    const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
     const hash = async function(filename: string): Promise<string> {
       return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
     };
@@ -311,18 +350,15 @@ async function checkDbschemeCompatibility(
     const upgradableTo = await hash(finalDbscheme);
 
     if (upgradableTo != dbschemeOfLib) {
-      logger.log(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but database has scheme ${query.program.dbschemePath}, and no upgrade path found`);
-      throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace. Please try using a newer version of the query libraries.`);
+      reportNoUpgradePath(query);
     }
 
     if (upgradableTo == dbschemeOfLib &&
       dbschemeOfDb != dbschemeOfLib) {
       // Try to upgrade the database
-      await upgradeDatabase(
+      await upgradeDatabaseExplicit(
         qs,
         query.dbItem,
-        Uri.file(finalDbscheme),
-        getUpgradesDirectories(scripts),
         progress,
         token
       );
@@ -330,6 +366,42 @@ async function checkDbschemeCompatibility(
   }
 }
 
+function reportNoUpgradePath(query: QueryInfo) {
+  throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`);
+}
+
+/**
+ * Compile a non-destructive upgrade.
+ */
+async function compileNonDestructiveUpgrade(
+  qs: qsClient.QueryServerClient,
+  upgradeTemp: tmp.DirectoryResult,
+  query: QueryInfo,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<string> {
+  const searchPath = getOnDiskWorkspaceFolders();
+
+  if (!query.dbItem?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
+  }
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme);
+
+  if (!matchesTarget) {
+    reportNoUpgradePath(query);
+  }
+  const result = await compileDatabaseUpgradeSequence(qs, query.dbItem, scripts, upgradeTemp, progress, token);
+  if (result.compiledUpgrade === undefined) {
+    const error = result.error || '[no error message available]';
+    throw new Error(error);
+  }
+  // We can upgrade to the actual target
+  query.program.dbschemePath = query.queryDbscheme;
+  // We are new enough that we will always support single file upgrades.
+  return result.compiledUpgrade;
+
+}
+
 /**
  * Prompts the user to save `document` if it has unsaved changes.
  *
@@ -516,67 +588,85 @@ export async function compileAndRunQueryAgainstDatabase(
   }
 
   const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
-  await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
 
-  let errors;
+  const upgradeDir = await tmp.dir({ dir: upgradesTmpDir.name, unsafeCleanup: true });
   try {
-    errors = await query.compile(qs, progress, token);
-  } catch (e) {
-    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+    let upgradeQlo;
+    if (await hasNondestructiveUpgradeCapabilities(qs)) {
+      upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, progress, token);
     } else {
-      throw e;
+      await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
     }
-  }
-
-  if (errors.length == 0) {
-    const result = await query.run(qs, progress, token);
-    if (result.resultType !== messages.QueryResultType.SUCCESS) {
-      const message = result.message || 'Failed to run query';
-      logger.log(message);
-      showAndLogErrorMessage(message);
-    }
-    return {
-      query,
-      result,
-      database: {
-        name: db.name,
-        databaseUri: db.databaseUri.toString(true)
-      },
-      options: historyItemOptions,
-      logFileLocation: result.logFileLocation,
-      dispose: () => {
-        qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+    let errors;
+    try {
+      errors = await query.compile(qs, progress, token);
+    } catch (e) {
+      if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+        return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+      } else {
+        throw e;
       }
-    };
-  } else {
-    // Error dialogs are limited in size and scrollability,
-    // so we include a general description of the problem,
-    // and direct the user to the output window for the detailed compilation messages.
-    // However we don't show quick eval errors there so we need to display them anyway.
-    qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
-
-    const formattedMessages: string[] = [];
-
-    for (const error of errors) {
-      const message = error.message || '[no error message available]';
-      const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
-      formattedMessages.push(formatted);
-      qs.logger.log(formatted);
     }
-    if (quickEval && formattedMessages.length <= 3) {
-      showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
+
+    if (errors.length === 0) {
+      const result = await query.run(qs, upgradeQlo, progress, token);
+      if (result.resultType !== messages.QueryResultType.SUCCESS) {
+        const message = result.message || 'Failed to run query';
+        logger.log(message);
+        showAndLogErrorMessage(message);
+      }
+      return {
+        query,
+        result,
+        database: {
+          name: db.name,
+          databaseUri: db.databaseUri.toString(true)
+        },
+        options: historyItemOptions,
+        logFileLocation: result.logFileLocation,
+        dispose: () => {
+          qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+        }
+      };
     } else {
-      showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
-        ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
-        ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
-        ' and choose CodeQL Query Server from the dropdown.');
-    }
+      // Error dialogs are limited in size and scrollability,
+      // so we include a general description of the problem,
+      // and direct the user to the output window for the detailed compilation messages.
+      // However we don't show quick eval errors there so we need to display them anyway.
+      qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
 
-    return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
+      const formattedMessages: string[] = [];
+
+      for (const error of errors) {
+        const message = error.message || '[no error message available]';
+        const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
+        formattedMessages.push(formatted);
+        qs.logger.log(formatted);
+      }
+      if (quickEval && formattedMessages.length <= 2) {
+        // If there are more than 2 error messages, they will not be displayed well in a popup
+        // and will be trimmed by the function displaying the error popup. Accordingly, we only
+        // try to show the errors if there are 2 or less, otherwise we direct the user to the log.
+        showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
+      } else {
+        showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
+      }
+
+      return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
+    }
+  } finally {
+    try {
+      upgradeDir.cleanup();
+    } catch (e) {
+      qs.logger.log(`Could not clean up the upgrades dir. Reason: ${e.message || e}`);
+    }
   }
 }
 
+const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
+  ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
+  ' and choose CodeQL Query Server from the dropdown.';
+
 function createSyntheticResult(
   query: QueryInfo,
   db: DatabaseItem,

extensions/ql-vscode/src/status-bar.ts

@@ -0,0 +1,48 @@
+import { ConfigurationChangeEvent, StatusBarAlignment, StatusBarItem, window, workspace } from 'vscode';
+import { CodeQLCliServer } from './cli';
+import { CANARY_FEATURES, CUSTOM_CODEQL_PATH_SETTING, DistributionConfigListener } from './config';
+import { DisposableObject } from './pure/disposable-object';
+
+/**
+ * Creates and manages a status bar item for codeql. THis item contains
+ * the current codeQL cli version as well as a notification if you are
+ * in canary mode
+ *
+ */
+export class CodeQlStatusBarHandler extends DisposableObject {
+
+  private readonly item: StatusBarItem;
+
+  constructor(private cli: CodeQLCliServer, distributionConfigListener: DistributionConfigListener) {
+    super();
+    this.item = window.createStatusBarItem(StatusBarAlignment.Right);
+    this.push(this.item);
+    this.push(workspace.onDidChangeConfiguration(this.handleDidChangeConfiguration, this));
+    this.push(distributionConfigListener.onDidChangeConfiguration(() => this.updateStatusItem()));
+    this.item.command = 'codeQL.openDocumentation';
+    this.updateStatusItem();
+  }
+
+  private handleDidChangeConfiguration(e: ConfigurationChangeEvent) {
+    if (
+      e.affectsConfiguration(CANARY_FEATURES.qualifiedName) ||
+      e.affectsConfiguration(CUSTOM_CODEQL_PATH_SETTING.qualifiedName)
+    ) {
+      // Wait a few seconds before updating the status item.
+      // This avoids a race condition where the cli's version
+      // is not updated before the status bar is refreshed.
+      setTimeout(() => this.updateStatusItem(), 3000);
+    }
+  }
+
+  private async updateStatusItem() {
+    const canary = CANARY_FEATURES.getValue() ? ' (Canary)' : '';
+    // since getting the verison may take a few seconds, initialize with some
+    // meaningful text.
+    this.item.text = `CodeQL${canary}`;
+
+    const version = await this.cli.getVersion();
+    this.item.text = `CodeQL CLI v${version}${canary}`;
+    this.item.show();
+  }
+}

extensions/ql-vscode/src/telemetry.ts

@@ -0,0 +1,214 @@
+import { ConfigurationTarget, Extension, ExtensionContext, ConfigurationChangeEvent } from 'vscode';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY } from './config';
+import * as appInsights from 'applicationinsights';
+import { logger } from './logging';
+import { UserCancellationException } from './commandRunner';
+import { showBinaryChoiceWithUrlDialog } from './helpers';
+
+// Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
+const key = 'REPLACE-APP-INSIGHTS-KEY';
+
+export enum CommandCompletion {
+  Success = 'Success',
+  Failed = 'Failed',
+  Cancelled = 'Cancelled'
+}
+
+// Avoid sending the following data to App insights since we don't need it.
+const tagsToRemove = [
+  'ai.application.ver',
+  'ai.device.id',
+  'ai.cloud.roleInstance',
+  'ai.cloud.role',
+  'ai.device.id',
+  'ai.device.osArchitecture',
+  'ai.device.osPlatform',
+  'ai.device.osVersion',
+  'ai.internal.sdkVersion',
+  'ai.session.id'
+];
+
+const baseDataPropertiesToRemove = [
+  'common.os',
+  'common.platformversion',
+  'common.remotename',
+  'common.uikind',
+  'common.vscodesessionid'
+];
+
+export class TelemetryListener extends ConfigListener {
+
+  static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
+
+  private reporter?: TelemetryReporter;
+
+  constructor(
+    private readonly id: string,
+    private readonly version: string,
+    private readonly key: string,
+    private readonly ctx: ExtensionContext
+  ) {
+    super();
+  }
+
+  /**
+   * This function handles changes to relevant configuration elements. There are 2 configuration
+   * ids that this function cares about:
+   *
+   *     * `codeQL.telemetry.enableTelemetry`: If this one has changed, then we need to re-initialize
+   *        the reporter and the reporter may wind up being removed.
+   *     * `codeQL.canary`: A change here could possibly re-trigger a dialog popup.
+   *
+   * Note that the global telemetry setting also gate-keeps whether or not to send telemetry events
+   * to Application Insights. However, this gatekeeping happens inside of the vscode-extension-telemetry
+   * package. So, this does not need to be handled here.
+   *
+   * @param e the configuration change event
+   */
+  async handleDidChangeConfiguration(e: ConfigurationChangeEvent): Promise<void> {
+    if (
+      e.affectsConfiguration('codeQL.telemetry.enableTelemetry') ||
+      e.affectsConfiguration('telemetry.enableTelemetry')
+    ) {
+      await this.initialize();
+    }
+
+    // Re-request telemetry so that users can see the dialog again.
+    // Re-request if codeQL.canary is being set to `true` and telemetry
+    // is not currently enabled.
+    if (
+      e.affectsConfiguration('codeQL.canary') &&
+      CANARY_FEATURES.getValue() &&
+      !ENABLE_TELEMETRY.getValue()
+    ) {
+      await Promise.all([
+        this.setTelemetryRequested(false),
+        this.requestTelemetryPermission()
+      ]);
+    }
+  }
+
+  async initialize() {
+    await this.requestTelemetryPermission();
+
+    this.disposeReporter();
+
+    if (ENABLE_TELEMETRY.getValue<boolean>()) {
+      this.createReporter();
+    }
+  }
+
+  private createReporter() {
+    this.reporter = new TelemetryReporter(
+      this.id,
+      this.version,
+      this.key,
+      /* anonymize stack traces */ true
+    );
+    this.push(this.reporter);
+
+    const client = (this.reporter as any).appInsightsClient as appInsights.TelemetryClient;
+    if (client) {
+      // add a telemetry processor to delete unwanted properties
+      client.addTelemetryProcessor((envelope: any) => {
+        tagsToRemove.forEach(tag => delete envelope.tags[tag]);
+        const baseDataProperties = (envelope.data as any)?.baseData?.properties;
+        if (baseDataProperties) {
+          baseDataPropertiesToRemove.forEach(prop => delete baseDataProperties[prop]);
+        }
+
+        if (LOG_TELEMETRY.getValue<boolean>()) {
+          logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
+        }
+        return true;
+      });
+    }
+  }
+
+  dispose() {
+    super.dispose();
+    this.reporter?.dispose();
+  }
+
+  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+    if (!this.reporter) {
+      return;
+    }
+    const status = !error
+      ? CommandCompletion.Success
+      : error instanceof UserCancellationException
+        ? CommandCompletion.Cancelled
+        : CommandCompletion.Failed;
+
+    const isCanary = (!!CANARY_FEATURES.getValue<boolean>()).toString();
+
+    this.reporter.sendTelemetryEvent(
+      'command-usage',
+      {
+        name,
+        status,
+        isCanary
+      },
+      { executionTime }
+    );
+  }
+
+  /**
+   * Displays a popup asking the user if they want to enable telemetry
+   * for this extension.
+   */
+  async requestTelemetryPermission() {
+    if (!this.wasTelemetryRequested()) {
+      // if global telemetry is disabled, avoid showing the dialog or making any changes
+      let result = undefined;
+      if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
+        // Extension won't start until this completes.
+        result = await showBinaryChoiceWithUrlDialog(
+          'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?',
+          'https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code'
+        );
+      }
+      if (result !== undefined) {
+        await Promise.all([
+          this.setTelemetryRequested(true),
+          ENABLE_TELEMETRY.updateValue<boolean>(result, ConfigurationTarget.Global),
+        ]);
+      }
+    }
+  }
+
+  /**
+   * Exposed for testing
+   */
+  get _reporter() {
+    return this.reporter;
+  }
+
+  private disposeReporter() {
+    if (this.reporter) {
+      this.reporter.dispose();
+      this.reporter = undefined;
+    }
+  }
+
+  private wasTelemetryRequested(): boolean {
+    return !!this.ctx.globalState.get<boolean>('telemetry-request-viewed');
+  }
+
+  private async setTelemetryRequested(newValue: boolean): Promise<void> {
+    await this.ctx.globalState.update('telemetry-request-viewed', newValue);
+  }
+}
+
+
+/**
+ * The global Telemetry instance
+ */
+export let telemetryListener: TelemetryListener;
+
+export async function initializeTelemetry(extension: Extension<any>, ctx: ExtensionContext): Promise<void> {
+  telemetryListener = new TelemetryListener(extension.id, extension.packageJSON.version, key, ctx);
+  telemetryListener.initialize();
+  ctx.subscriptions.push(telemetryListener);
+}

extensions/ql-vscode/src/test-adapter.ts

@@ -15,7 +15,7 @@ import {
 import { TestAdapterRegistrar } from 'vscode-test-adapter-util';
 import { QLTestFile, QLTestNode, QLTestDirectory, QLTestDiscovery } from './qltest-discovery';
 import { Event, EventEmitter, CancellationTokenSource, CancellationToken } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { CodeQLCliServer } from './cli';
 import { getOnDiskWorkspaceFolders } from './helpers';
 import { testLogger } from './logging';
@@ -223,8 +223,8 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
           ? 'errored'
           : 'failed';
       let message: string | undefined;
-      if (event.diff?.length) {
-        message = ['', `${state}: ${event.test}`, ...event.diff, ''].join('\n');
+      if (event.failureDescription || event.diff?.length) {
+        message = ['', `${state}: ${event.test}`, event.failureDescription || event.diff?.join('\n'), ''].join('\n');
         testLogger.log(message);
       }
       this._testStates.fire({

extensions/ql-vscode/src/test-ui.ts

@@ -13,7 +13,7 @@ import {
 
 import { showAndLogWarningMessage } from './helpers';
 import { TestTreeNode } from './test-tree-node';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { UIService } from './vscode-utils/ui-service';
 import { QLTestAdapter, getExpectedFile, getActualFile } from './test-adapter';
 import { logger } from './logging';

extensions/ql-vscode/src/upgrades.ts

@@ -1,11 +1,14 @@
 import * as vscode from 'vscode';
-import { DatabaseItem } from './databases';
-import { showAndLogErrorMessage } from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
 import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 import * as messages from './pure/messages';
 import * as qsClient from './queryserver-client';
 import { upgradesTmpDir } from './run-queries';
+import * as tmp from 'tmp-promise';
+import * as path from 'path';
+import * as semver from 'semver';
+import { DatabaseItem } from './databases';
 
 /**
  * Maximum number of lines to include from database upgrade message,
@@ -15,79 +18,97 @@ import { upgradesTmpDir } from './run-queries';
 const MAX_UPGRADE_MESSAGE_LINES = 10;
 
 /**
- * Checks whether the given database can be upgraded to the given target DB scheme,
- * and whether the user wants to proceed with the upgrade.
- * Reports errors to both the user and the console.
- * @returns the `UpgradeParams` needed to start the upgrade, if the upgrade is possible and was confirmed by the user, or `undefined` otherwise.
+ * Check that we support non-destructive upgrades.
+ *
+ * This requires 3 features. The ability to compile an upgrade sequence; The ability to
+ * run a non-destructive upgrades as a query; the ability to specify a target when
+ * resolving upgrades. We check for a version of codeql that has all three features.
  */
-async function checkAndConfirmDatabaseUpgrade(
+export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QueryServerClient): Promise<boolean> {
+  return semver.gte(await qs.cliServer.getVersion(), '2.4.2');
+}
+
+
+/**
+ * Compile a database upgrade sequence.
+ * Callers must check that this is valid with the current queryserver first.
+ */
+export async function compileDatabaseUpgradeSequence(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
-  targetDbScheme: vscode.Uri,
-  upgradesDirectories: vscode.Uri[],
+  resolvedSequence: string[],
+  currentUpgradeTmp: tmp.DirectoryResult,
   progress: ProgressCallback,
-  token: vscode.CancellationToken,
-): Promise<messages.UpgradeParams | undefined> {
+  token: vscode.CancellationToken
+): Promise<messages.CompileUpgradeSequenceResult> {
   if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
-  const params: messages.UpgradeParams = {
-    fromDbscheme: db.contents.dbSchemeUri.fsPath,
-    toDbscheme: targetDbScheme.fsPath,
-    additionalUpgrades: upgradesDirectories.map(uri => uri.fsPath)
-  };
+  if (!await hasNondestructiveUpgradeCapabilities(qs)) {
+    throw new Error('The version of codeql is too old to run non-destructive upgrades.');
+  }
+  // If possible just compile the upgrade sequence
+  return await qs.sendRequest(messages.compileUpgradeSequence, {
+    upgradeTempDir: currentUpgradeTmp.path,
+    upgradePaths: resolvedSequence
+  }, token, progress);
+}
 
-  let checkUpgradeResult: messages.CheckUpgradeResult;
-  try {
-    qs.logger.log('Checking database upgrade...');
-    checkUpgradeResult = await checkDatabaseUpgrade(qs, params, progress, token);
-  }
-  catch (e) {
-    throw new Error(`Database cannot be upgraded: ${e}`);
-  }
-  finally {
-    qs.logger.log('Done checking database upgrade.');
+async function compileDatabaseUpgrade(
+  qs: qsClient.QueryServerClient,
+  db: DatabaseItem,
+  targetDbScheme: string,
+  resolvedSequence: string[],
+  currentUpgradeTmp: tmp.DirectoryResult,
+  progress: ProgressCallback,
+  token: vscode.CancellationToken
+): Promise<messages.CompileUpgradeResult> {
+  if (!db.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
   }
+  // We have the upgrades we want but compileUpgrade
+  // requires searching for them.  So we use the parent directories of the upgrades
+  // as the upgrade path.
+  const parentDirs = resolvedSequence.map(dir => path.dirname(dir));
+  const uniqueParentDirs = new Set(parentDirs);
+  progress({
+    step: 1,
+    maxStep: 3,
+    message: 'Checking for database upgrades'
+  });
+  return qs.sendRequest(messages.compileUpgrade, {
+    upgrade: {
+      fromDbscheme: db.contents.dbSchemeUri.fsPath,
+      toDbscheme: targetDbScheme,
+      additionalUpgrades: Array.from(uniqueParentDirs)
+    },
+    upgradeTempDir: currentUpgradeTmp.path,
+    singleFileUpgrades: true,
+  }, token, progress);
+}
 
-  const checkedUpgrades = checkUpgradeResult.checkedUpgrades;
-  if (checkedUpgrades === undefined) {
-    const error = checkUpgradeResult.upgradeError || '[no error message available]';
-    throw new Error(`Database cannot be upgraded: ${error}`);
-  }
+/**
+ * Checks whether the user wants to proceed with the upgrade.
+ * Reports errors to both the user and the console.
+ */
+async function checkAndConfirmDatabaseUpgrade(
+  compiled: messages.CompiledUpgrades,
+  db: DatabaseItem,
+  quiet: boolean
+): Promise<void> {
 
-  if (checkedUpgrades.scripts.length === 0) {
-    progress({
-      step: 3,
-      maxStep: 3,
-      message: 'Database is already up to date; nothing to do.'
-    });
-    return;
-  }
-
-  let curSha = checkedUpgrades.initialSha;
   let descriptionMessage = '';
-  for (const script of checkedUpgrades.scripts) {
+  const descriptions = getUpgradeDescriptions(compiled);
+  for (const script of descriptions) {
     descriptionMessage += `Would perform upgrade: ${script.description}\n`;
     descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
-    curSha = script.newSha;
   }
-
-  const targetSha = checkedUpgrades.targetSha;
-  if (curSha != targetSha) {
-    // Newlines aren't rendered in notifications: https://github.com/microsoft/vscode/issues/48900
-    // A modal dialog would be rendered better, but is more intrusive.
-    await showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
-    Can upgrade from ${checkedUpgrades.initialSha} (current) to ${curSha}, but cannot reach ${targetSha} (target).`);
-    // TODO: give a more informative message if we think the DB is ahead of the target DB scheme
-    return;
-  }
-
   logger.log(descriptionMessage);
 
 
   // If the quiet flag is set, do the upgrade without a popup.
-  if (qs.cliServer.quiet) {
-    return params;
+  if (quiet) {
+    return;
   }
 
   // Ask the user to confirm the upgrade.
@@ -111,106 +132,91 @@ async function checkAndConfirmDatabaseUpgrade(
     logger.outputChannel.show();
   }
 
-  if (chosenItem === yesItem) {
-    return params;
-  }
-  else {
+  if (chosenItem !== yesItem) {
     throw new UserCancellationException('User cancelled the database upgrade.');
   }
 }
 
+/**
+ * Get the descriptions from a compiled upgrade
+ */
+function getUpgradeDescriptions(compiled: messages.CompiledUpgrades): messages.UpgradeDescription[] {
+  // We use the presence of compiledUpgradeFile to check
+  // if it is multifile or not. We need to explicitly check undefined
+  // as the types claim the empty string is a valid value
+  if (compiled.compiledUpgradeFile === undefined) {
+    return compiled.scripts.map(script => script.description);
+  } else {
+    return compiled.descriptions;
+  }
+}
+
 /**
  * Command handler for 'Upgrade Database'.
  * Attempts to upgrade the given database to the given target DB scheme, using the given directory of upgrades.
  * First performs a dry-run and prompts the user to confirm the upgrade.
  * Reports errors during compilation and evaluation of upgrades to the user.
  */
-export async function upgradeDatabase(
+export async function upgradeDatabaseExplicit(
   qs: qsClient.QueryServerClient,
-  db: DatabaseItem, targetDbScheme: vscode.Uri,
-  upgradesDirectories: vscode.Uri[],
+  db: DatabaseItem,
   progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.RunUpgradeResult | undefined> {
-  const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories, progress, token);
 
-  if (upgradeParams === undefined) {
-    return;
+  const searchPath: string[] = getOnDiskWorkspaceFolders();
+
+  if (!db?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
   }
+  const upgradeInfo = await qs.cliServer.resolveUpgrades(
+    db.contents.dbSchemeUri.fsPath,
+    searchPath,
+    false
+  );
 
-  let compileUpgradeResult: messages.CompileUpgradeResult;
+  const { scripts, finalDbscheme } = upgradeInfo;
+
+  if (finalDbscheme === undefined) {
+    throw new Error('Could not determine target dbscheme to upgrade to.');
+  }
+  const currentUpgradeTmp = await tmp.dir({ dir: upgradesTmpDir.name, prefix: 'upgrade_', keep: false, unsafeCleanup: true });
   try {
-    compileUpgradeResult = await compileDatabaseUpgrade(qs, upgradeParams, progress, token);
-  }
-  catch (e) {
-    showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done compiling database upgrade.');
-  }
-
-  if (compileUpgradeResult.compiledUpgrades === undefined) {
-    const error = compileUpgradeResult.error || '[no error message available]';
-    (`Compilation of database upgrades failed: ${error}`);
-    return;
-  }
-
-  try {
-    qs.logger.log('Running the following database upgrade:');
-    // We use the presence of compiledUpgradeFile to check
-    // if it is multifile or not. We need to explicitly check undefined
-    // as the types claim the empty string is a valid value
-    if (compileUpgradeResult.compiledUpgrades.compiledUpgradeFile === undefined) {
-      qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
-    } else {
-      qs.logger.log(compileUpgradeResult.compiledUpgrades.descriptions.map(s => s.description).join('\n'));
+    let compileUpgradeResult: messages.CompileUpgradeResult;
+    try {
+      compileUpgradeResult = await compileDatabaseUpgrade(qs, db, finalDbscheme, scripts, currentUpgradeTmp, progress, token);
     }
-    return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
+    catch (e) {
+      showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
+      return;
+    }
+    finally {
+      qs.logger.log('Done compiling database upgrade.');
+    }
+
+    if (!compileUpgradeResult.compiledUpgrades) {
+      const error = compileUpgradeResult.error || '[no error message available]';
+      showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
+      return;
+    }
+
+    await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, db, qs.cliServer.quiet);
+
+    try {
+      qs.logger.log('Running the following database upgrade:');
+
+      getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n');
+      return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
+    }
+    catch (e) {
+      showAndLogErrorMessage(`Database upgrade failed: ${e}`);
+      return;
+    } finally {
+      qs.logger.log('Done running database upgrade.');
+    }
+  } finally {
+    currentUpgradeTmp.cleanup();
   }
-  catch (e) {
-    showAndLogErrorMessage(`Database upgrade failed: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done running database upgrade.');
-  }
-}
-
-async function checkDatabaseUpgrade(
-  qs: qsClient.QueryServerClient,
-  upgradeParams: messages.UpgradeParams,
-  progress: ProgressCallback,
-  token: vscode.CancellationToken,
-): Promise<messages.CheckUpgradeResult> {
-  progress({
-    step: 1,
-    maxStep: 3,
-    message: 'Checking for database upgrades'
-  });
-
-  return qs.sendRequest(messages.checkUpgrade, upgradeParams, token, progress);
-}
-
-async function compileDatabaseUpgrade(
-  qs: qsClient.QueryServerClient,
-  upgradeParams: messages.UpgradeParams,
-  progress: ProgressCallback,
-  token: vscode.CancellationToken,
-): Promise<messages.CompileUpgradeResult> {
-  const params: messages.CompileUpgradeParams = {
-    upgrade: upgradeParams,
-    upgradeTempDir: upgradesTmpDir.name,
-    singleFileUpgrades: true
-  };
-
-  progress({
-    step: 2,
-    maxStep: 3,
-    message: 'Compiling database upgrades'
-  });
-
-  return qs.sendRequest(messages.compileUpgrade, params, token, progress);
 }
 
 async function runDatabaseUpgrade(

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -44,7 +44,6 @@ describe('Databases', function() {
 
   afterEach(() => {
     try {
-      // dispose();
       sandbox.restore();
     } catch (e) {
       fail(e);

extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts

@@ -16,14 +16,12 @@ export const DB_URL = 'https://github.com/github/vscode-codeql/files/5586722/sim
 export const dbLoc = path.join(fs.realpathSync(path.join(__dirname, '../../../')), 'build/tests/db.zip');
 export let storagePath: string;
 
-// See https://github.com/DefinitelyTyped/DefinitelyTyped/pull/49860
-// Should be of type Mocha
-export default function(mocha: /*Mocha*/ any) {
+export default function(mocha: Mocha) {
   // create an extension storage location
   let removeStorage: tmp.DirResult['removeCallback'] | undefined;
 
-  mocha.globalSetup([
-    // ensure the test database is downloaded
+  // ensure the test database is downloaded
+  (mocha.options as any).globalSetup.push(
     async () => {
       fs.mkdirpSync(path.dirname(dbLoc));
       if (!fs.existsSync(dbLoc)) {
@@ -44,14 +42,18 @@ export default function(mocha: /*Mocha*/ any) {
           fail('Failed to download test database: ' + e);
         }
       }
-    },
+    }
+  );
 
-    // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+  (mocha.options as any).globalSetup.push(
     async () => {
       await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
-    },
+    }
+  );
 
-    // Create the temp directory to be used as extension local storage.
+  // Create the temp directory to be used as extension local storage.
+  (mocha.options as any).globalSetup.push(
     () => {
       const dir = tmp.dirSync();
       storagePath = fs.realpathSync(dir.name);
@@ -61,10 +63,10 @@ export default function(mocha: /*Mocha*/ any) {
 
       removeStorage = dir.removeCallback;
     }
-  ]);
+  );
 
-  mocha.globalTeardown([
-    // ensure etension is cleaned up.
+  // ensure etension is cleaned up.
+  (mocha.options as any).globalTeardown.push(
     async () => {
       const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
       // This shuts down the extension and can only be run after all tests have completed.
@@ -72,10 +74,13 @@ export default function(mocha: /*Mocha*/ any) {
       if ('dispose' in extension) {
         extension.dispose();
       }
-    },
-    // ensure temp directory is cleaned up.
+    }
+  );
+
+  // ensure temp directory is cleaned up.
+  (mocha.options as any).globalTeardown.push(
     () => {
       removeStorage?.();
     }
-  ]);
+  );
 }

extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts

@@ -1,10 +1,11 @@
 import { fail } from 'assert';
-import { CancellationToken, extensions, Uri } from 'vscode';
+import { CancellationToken, commands, ExtensionContext, extensions, Uri } from 'vscode';
 import * as sinon from 'sinon';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import 'mocha';
 import { expect } from 'chai';
+import * as yaml from 'js-yaml';
 
 import { DatabaseItem, DatabaseManager } from '../../databases';
 import { CodeQLExtensionInterface } from '../../extension';
@@ -14,6 +15,7 @@ import { compileAndRunQueryAgainstDatabase } from '../../run-queries';
 import { CodeQLCliServer } from '../../cli';
 import { QueryServerClient } from '../../queryserver-client';
 import { skipIfNoCodeQL } from '../ensureCli';
+import { QueryResultType } from '../../pure/messages';
 
 
 /**
@@ -33,6 +35,11 @@ describe('Queries', function() {
   let sandbox: sinon.SinonSandbox;
   let progress: sinon.SinonSpy;
   let token: CancellationToken;
+  let ctx: ExtensionContext;
+
+  let qlpackFile: string;
+  let qlFile: string;
+
 
   beforeEach(async () => {
     sandbox = sinon.createSandbox();
@@ -44,6 +51,9 @@ describe('Queries', function() {
         cli = extension.cliServer;
         qs = extension.qs;
         cli.quiet = true;
+        ctx = extension.ctx;
+        qlpackFile = `${ctx.storagePath}/quick-queries/qlpack.yml`;
+        qlFile = `${ctx.storagePath}/quick-queries/quick-query.ql`;
       } else {
         throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
       }
@@ -94,10 +104,73 @@ describe('Queries', function() {
       // just check that the query was successful
       expect(result.database.name).to.eq('db');
       expect(result.options.queryText).to.eq(fs.readFileSync(queryPath, 'utf8'));
+      expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
     } catch (e) {
       console.error('Test Failed');
       fail(e);
     }
   });
 
+  // Asserts a fix for bug https://github.com/github/vscode-codeql/issues/733
+  it('should restart the database and run a query', async () => {
+    try {
+      await commands.executeCommand('codeQL.restartQueryServer');
+      const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
+      const result = await compileAndRunQueryAgainstDatabase(
+        cli,
+        qs,
+        dbItem,
+        false,
+        Uri.file(queryPath),
+        progress,
+        token
+      );
+
+      // this message would indicate that the databases were not properly reregistered
+      expect(result.result.message).not.to.eq('No result from server');
+      expect(result.options.queryText).to.eq(fs.readFileSync(queryPath, 'utf8'));
+      expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
+    } catch (e) {
+      console.error('Test Failed');
+      fail(e);
+    }
+  });
+
+  it('should create a quick query', async () => {
+    safeDel(qlFile);
+    safeDel(qlpackFile);
+
+    await commands.executeCommand('codeQL.quickQuery');
+
+    // should have created the quick query file and query pack file
+    expect(fs.pathExistsSync(qlFile)).to.be.true;
+    expect(fs.pathExistsSync(qlpackFile)).to.be.true;
+
+    const qlpackContents: any = await yaml.safeLoad(
+      fs.readFileSync(qlpackFile, 'utf8')
+    );
+    // Should have chosen the js libraries
+    expect(qlpackContents.libraryPathDependencies[0]).to.eq('codeql-javascript');
+  });
+
+  it('should avoid creating a quick query', async () => {
+    fs.writeFileSync(qlpackFile, yaml.safeDump({
+      name: 'quick-query',
+      version: '1.0.0',
+      libraryPathDependencies: ['codeql-javascript']
+    }));
+    fs.writeFileSync(qlFile, 'xxx');
+    await commands.executeCommand('codeQL.quickQuery');
+
+    // should not have created the quick query file because database schema hasn't changed
+    expect(fs.readFileSync(qlFile, 'utf8')).to.eq('xxx');
+  });
+
+  function safeDel(file: string) {
+    try {
+      fs.unlinkSync(file);
+    } catch (e) {
+      // ignore
+    }
+  }
 });

extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts

@@ -33,7 +33,10 @@ class Checkpoint<T> {
   constructor() {
     this.res = () => { /**/ };
     this.rej = () => { /**/ };
-    this.promise = new Promise((res, rej) => { this.res = res; this.rej = rej; });
+    this.promise = new Promise((res, rej) => {
+      this.res = res as () => {};
+      this.rej = rej;
+    });
   }
 
   async done(): Promise<T> {
@@ -81,6 +84,11 @@ const queryTestCases: QueryTestCase[] = [
   }
 ];
 
+const db: messages.Dataset = {
+  dbDir: path.join(__dirname, '../test-db'),
+  workingSet: 'default',
+};
+
 describe('using the query server', function() {
   before(function() {
     skipIfNoCodeQL(this);
@@ -120,6 +128,12 @@ describe('using the query server', function() {
     const evaluationSucceeded = new Checkpoint<void>();
     const parsedResults = new Checkpoint<void>();
 
+    it('should register the database if necessary', async () => {
+      if (await cliServer.cliConstraints.supportsDatabaseRegistration()) {
+        await qs.sendRequest(messages.registerDatabases, { databases: [db] }, token, (() => { /**/ }) as any);
+      }
+    });
+
     it(`should be able to compile query ${queryName}`, async function() {
       await queryServerStarted.done();
       expect(fs.existsSync(queryTestCase.queryPath)).to.be.true;
@@ -166,15 +180,11 @@ describe('using the query server', function() {
           id: callbackId,
           timeoutSecs: 1000,
         };
-        const db: messages.Dataset = {
-          dbDir: path.join(__dirname, '../test-db'),
-          workingSet: 'default',
-        };
         const params: messages.EvaluateQueriesParams = {
           db,
           evaluateId: callbackId,
           queries: [queryToRun],
-          stopOnError: false,
+          stopOnError: true,
           useSequenceHint: false
         };
         await qs.sendRequest(messages.runQueries, params, token, () => { /**/ });

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -28,7 +28,10 @@ import { workspace } from 'vscode';
 process.on('unhandledRejection', e => {
   console.error('Unhandled rejection.');
   console.error(e);
-  process.exit(-1);
+  // Must use a setTimeout in order to ensure the log is fully flushed before exiting
+  setTimeout(() => {
+    process.exit(-1);
+  }, 2000);
 });
 
 const _1MB = 1024 * 1024;
@@ -36,7 +39,7 @@ const _10MB = _1MB * 10;
 
 // CLI version to test. Hard code the latest as default. And be sure
 // to update the env if it is not otherwise set.
-const CLI_VERSION = process.env.CLI_VERSION || 'v2.4.0';
+const CLI_VERSION = process.env.CLI_VERSION || 'v2.4.2';
 process.env.CLI_VERSION = CLI_VERSION;
 
 // Base dir where CLIs will be downloaded into

extensions/ql-vscode/src/vscode-tests/index-template.ts

@@ -2,6 +2,7 @@ import * as path from 'path';
 import * as Mocha from 'mocha';
 import * as glob from 'glob';
 import { ensureCli } from './ensureCli';
+import { env } from 'vscode';
 
 
 // Use this handler to avoid swallowing unhandled rejections.
@@ -42,8 +43,17 @@ export async function runTestsInDirectory(testsRoot: string, useCli = false): Pr
   // Create the mocha test
   const mocha = new Mocha({
     ui: 'bdd',
-    color: true
-  });
+    color: true,
+    globalSetup: [],
+    globalTeardown: [],
+  } as any);
+
+  (mocha.options as any).globalSetup.push(
+    // convert this function into an noop since it should not run during tests.
+    // If it does run during tests, then it can cause some testing environments
+    // to hang.
+    (env as any).openExternal = () => { /**/ }
+  );
 
   await ensureCli(useCli);
 

extensions/ql-vscode/src/vscode-tests/minimal-workspace/config.test.ts

@@ -15,7 +15,10 @@ import {
 chai.use(chaiAsPromised);
 const expect = chai.expect;
 
-describe('config listeners', () => {
+describe('config listeners', function() {
+  // Because we are adding some extra waiting, need to bump the test timeouts.
+  this.timeout(5000);
+
   let sandbox: Sinon.SinonSandbox;
   beforeEach(() => {
     sandbox = Sinon.createSandbox();
@@ -26,7 +29,7 @@ describe('config listeners', () => {
   });
 
   interface TestConfig<T> {
-    clazz: new() => {};
+    clazz: new () => {};
     settings: {
       name: string;
       property: string;
@@ -38,6 +41,10 @@ describe('config listeners', () => {
     {
       clazz: CliConfigListener,
       settings: [{
+        name: 'codeQL.runningQueries.numberOfThreads',
+        property: 'numberThreads',
+        values: [0, 1]
+      }, {
         name: 'codeQL.runningTests.numberOfThreads',
         property: 'numberTestThreads',
         values: [1, 0]
@@ -57,6 +64,14 @@ describe('config listeners', () => {
         name: 'codeQL.runningQueries.numberOfThreads',
         property: 'numThreads',
         values: [0, 1]
+      }, {
+        name: 'codeQL.runningQueries.saveCache',
+        property: 'saveCache',
+        values: [false, true]
+      }, {
+        name: 'codeQL.runningQueries.cacheSize',
+        property: 'cacheSize',
+        values: [0, 1]
       }, {
         name: 'codeQL.runningQueries.memory',
         property: 'queryMemoryMb',
@@ -84,19 +99,31 @@ describe('config listeners', () => {
         beforeEach(async () => {
           origValue = workspace.getConfiguration().get(setting.name);
           await workspace.getConfiguration().update(setting.name, setting.values[0]);
+          await wait();
           spy.resetHistory();
         });
 
         afterEach(async () => {
           await workspace.getConfiguration().update(setting.name, origValue);
+          await wait();
         });
 
         it(`should listen for changes to '${setting.name}'`, async () => {
           await workspace.getConfiguration().update(setting.name, setting.values[1]);
-          expect(spy.calledOnce).to.be.true;
+          await wait();
           expect(listener[setting.property]).to.eq(setting.values[1]);
+          expect(spy).to.have.been.calledOnce;
         });
       });
     });
   });
+
+  // Need to wait some time since the onDidChangeConfiguration listeners fire
+  // asynchronously and we sometimes need to wait for them to complete in
+  // order to have as successful test.
+  async function wait(ms = 50) {
+    return new Promise(resolve =>
+      setTimeout(resolve, ms)
+    );
+  }
 });

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -20,6 +20,7 @@ import { registerDatabases } from '../../pure/messages';
 import { ProgressCallback } from '../../commandRunner';
 import { CodeQLCliServer } from '../../cli';
 import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 describe('databases', () => {
 
@@ -66,10 +67,13 @@ describe('databases', () => {
       } as unknown as ExtensionContext,
       {
         sendRequest: sendRequestSpy,
-        supportsDatabaseRegistration: supportsDatabaseRegistrationSpy
+        onDidStartQueryServer: () => { /**/ }
       } as unknown as QueryServerClient,
       {
-        supportsLanguageName: supportsLanguageNameSpy,
+        cliConstraints: {
+          supportsLanguageName: supportsLanguageNameSpy,
+          supportsDatabaseRegistration: supportsDatabaseRegistrationSpy,
+        },
         resolveDatabase: resolveDatabaseSpy
       } as unknown as CodeQLCliServer,
       {} as Logger,
@@ -84,7 +88,7 @@ describe('databases', () => {
 
   afterEach(async () => {
     dir.removeCallback();
-    databaseManager.dispose();
+    databaseManager.dispose(testDisposeHandler);
     sandbox.restore();
   });
 

extensions/ql-vscode/src/vscode-tests/no-workspace/astViewer.test.ts

@@ -7,6 +7,7 @@ import * as yaml from 'js-yaml';
 import { AstViewer, AstItem } from '../../astViewer';
 import { commands, Range } from 'vscode';
 import { DatabaseItem } from '../../databases';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 chai.use(chaiAsPromised);
 const expect = chai.expect;
@@ -31,7 +32,7 @@ describe('AstViewer', () => {
   afterEach(() => {
     sandbox.restore();
     if (viewer) {
-      viewer.dispose();
+      viewer.dispose(testDisposeHandler);
       viewer = undefined;
     }
   });

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts

@@ -37,7 +37,6 @@ describe('AstBuilder', () => {
   let mockCli: CodeQLCliServer;
   let overrides: Record<string, object | undefined>;
 
-
   beforeEach(() => {
     mockCli = {
       bqrsDecode: sinon.stub().callsFake((_: string, resultSet: 'nodes' | 'edges' | 'graphProperties') => {

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -21,17 +21,19 @@ describe('databaseFetcher', function() {
   this.timeout(10000);
 
   describe('convertToDatabaseUrl', () => {
+    let sandbox: sinon.SinonSandbox;
     let quickPickSpy: sinon.SinonStub;
     beforeEach(() => {
-      quickPickSpy = sinon.stub(window, 'showQuickPick');
+      sandbox = sinon.createSandbox();
+      quickPickSpy = sandbox.stub(window, 'showQuickPick');
     });
 
     afterEach(() => {
-      (window.showQuickPick as sinon.SinonStub).restore();
+      sandbox.restore();
     });
 
     it('should convert a project url to a database url', async () => {
-      quickPickSpy.returns('javascript' as any);
+      quickPickSpy.resolves('javascript');
       const lgtmUrl = 'https://lgtm.com/projects/g/github/codeql';
       const dbUrl = await convertToDatabaseUrl(lgtmUrl);
 
@@ -43,7 +45,7 @@ describe('databaseFetcher', function() {
     });
 
     it('should convert a project url to a database url with extra path segments', async () => {
-      quickPickSpy.returns('python' as any);
+      quickPickSpy.resolves('python');
       const lgtmUrl =
         'https://lgtm.com/projects/g/github/codeql/subpage/subpage2?query=xxx';
       const dbUrl = await convertToDatabaseUrl(lgtmUrl);
@@ -53,8 +55,19 @@ describe('databaseFetcher', function() {
       );
     });
 
-    it('should fail on a nonexistant prohect', async () => {
-      quickPickSpy.returns('javascript' as any);
+    it('should convert a raw slug to a database url with extra path segments', async () => {
+      quickPickSpy.resolves('python');
+      const lgtmUrl =
+        'g/github/codeql';
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+
+      expect(dbUrl).to.equal(
+        'https://lgtm.com/api/v1.0/snapshots/1506465042581/python'
+      );
+    });
+
+    it('should fail on a nonexistent project', async () => {
+      quickPickSpy.resolves('javascript');
       const lgtmUrl = 'https://lgtm.com/projects/g/github/hucairz';
       expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
     });
@@ -69,6 +82,10 @@ describe('databaseFetcher', function() {
         .to.be.false;
       expect(looksLikeLgtmUrl('https://ww.lgtm.com/projects/g/github')).to.be
         .false;
+      expect(looksLikeLgtmUrl('g/github')).to.be
+        .false;
+      expect(looksLikeLgtmUrl('ggg/github/myproj')).to.be
+        .false;
     });
 
     it('should handle valid urls', () => {
@@ -84,6 +101,10 @@ describe('databaseFetcher', function() {
           'https://lgtm.com/projects/g/github/codeql/sub/pages?query=string'
         )
       ).to.be.true;
+      expect(looksLikeLgtmUrl('g/github/myproj')).to.be
+        .true;
+      expect(looksLikeLgtmUrl('git/github/myproj')).to.be
+        .true;
     });
   });
 

extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts

@@ -7,6 +7,7 @@ import { expect } from 'chai';
 import { Uri } from 'vscode';
 
 import { DatabaseUI } from '../../databases-ui';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 describe('databases-ui', () => {
   describe('fixDbUri', () => {
@@ -68,7 +69,6 @@ describe('databases-ui', () => {
     const db5 = createDatabase(storageDir, 'db2-notimported-with-codeql-database.yml', 'cpp', 'codeql-database.yml');
 
     const databaseUI = new DatabaseUI(
-      {} as any,
       {
         databaseItems: [
           { databaseUri: Uri.file(db1) }
@@ -90,7 +90,7 @@ describe('databases-ui', () => {
     expect(fs.pathExistsSync(db4)).to.be.false;
     expect(fs.pathExistsSync(db5)).to.be.false;
 
-    databaseUI.dispose();
+    databaseUI.dispose(testDisposeHandler);
   });
 
   function createDatabase(storageDir: string, dbName: string, language: string, extraFile?: string) {

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -190,10 +190,11 @@ describe('Launcher path', () => {
   let launcherThatExists = '';
 
   beforeEach(() => {
+    sandbox = sinon.createSandbox();
     getExecutableFromDirectory = createModule().getExecutableFromDirectory;
   });
 
-  beforeEach(() => {
+  afterEach(() => {
     sandbox.restore();
   });
 
@@ -300,7 +301,6 @@ describe('Launcher path', () => {
   });
 
   function createModule() {
-    sandbox = sinon.createSandbox();
     warnSpy = sandbox.spy();
     errorSpy = sandbox.spy();
     logSpy = sandbox.spy();

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -1,14 +1,23 @@
 import { expect } from 'chai';
 import 'mocha';
-import { ExtensionContext, Memento } from 'vscode';
+import { ExtensionContext, Memento, window } from 'vscode';
 import * as yaml from 'js-yaml';
 import * as tmp from 'tmp';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as sinon from 'sinon';
 
-import { getInitialQueryContents, InvocationRateLimiter, isLikelyDbLanguageFolder } from '../../helpers';
+import {
+  getInitialQueryContents,
+  InvocationRateLimiter,
+  isLikelyDbLanguageFolder,
+  showBinaryChoiceDialog,
+  showBinaryChoiceWithUrlDialog,
+  showInformationMessageWithAction
+} from '../../helpers';
 import { reportStreamProgress } from '../../commandRunner';
+import Sinon = require('sinon');
+import { fail } from 'assert';
 
 describe('helpers', () => {
   let sandbox: sinon.SinonSandbox;
@@ -225,4 +234,91 @@ describe('helpers', () => {
       message: 'My prefix (Size unknown)',
     });
   });
+
+  describe('open dialog', () => {
+    let showInformationMessageSpy: Sinon.SinonStub;
+    beforeEach(() => {
+      showInformationMessageSpy = sandbox.stub(window, 'showInformationMessage');
+    });
+
+    it('should show a binary choice dialog and return `yes`', (done) => {
+      // pretend user chooses 'yes'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      const res = showBinaryChoiceDialog('xxx');
+      res.then((val) => {
+        expect(val).to.eq(true);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog and return `no`', (done) => {
+      // pretend user chooses 'no'
+      showInformationMessageSpy.onCall(0).resolvesArg(3);
+      const res = showBinaryChoiceDialog('xxx');
+      res.then((val) => {
+        expect(val).to.eq(false);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show an info dialog and confirm the action', (done) => {
+      // pretend user chooses to run action
+      showInformationMessageSpy.onCall(0).resolvesArg(1);
+      const res = showInformationMessageWithAction('xxx', 'yyy');
+      res.then((val) => {
+        expect(val).to.eq(true);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show an action dialog and avoid choosing the action', (done) => {
+      // pretend user does not choose to run action
+      showInformationMessageSpy.onCall(0).resolves(undefined);
+      const res = showInformationMessageWithAction('xxx', 'yyy');
+      res.then((val) => {
+        expect(val).to.eq(false);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog with a url and return `yes`', (done) => {
+      // pretend user clicks on the url twice and then clicks 'yes'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      showInformationMessageSpy.onCall(1).resolvesArg(2);
+      showInformationMessageSpy.onCall(2).resolvesArg(3);
+      const res = showBinaryChoiceWithUrlDialog('xxx', 'invalid:url');
+      res.then((val) => {
+        expect(val).to.eq(true);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog with a url and return `no`', (done) => {
+      // pretend user clicks on the url twice and then clicks 'no'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      showInformationMessageSpy.onCall(1).resolvesArg(2);
+      showInformationMessageSpy.onCall(2).resolvesArg(4);
+      const res = showBinaryChoiceWithUrlDialog('xxx', 'invalid:url');
+      res.then((val) => {
+        expect(val).to.eq(false);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog and exit after clcking `more info` 5 times', (done) => {
+      // pretend user clicks on the url twice and then clicks 'no'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      showInformationMessageSpy.onCall(1).resolvesArg(2);
+      showInformationMessageSpy.onCall(2).resolvesArg(2);
+      showInformationMessageSpy.onCall(3).resolvesArg(2);
+      showInformationMessageSpy.onCall(4).resolvesArg(2);
+      const res = showBinaryChoiceWithUrlDialog('xxx', 'invalid:url');
+      res.then((val) => {
+        // No choie was made
+        expect(val).to.eq(undefined);
+        expect(showInformationMessageSpy.getCalls().length).to.eq(5);
+        done();
+      }).catch(e => fail(e));
+    });
+  });
 });

extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts

@@ -152,7 +152,8 @@ describe('CompletedQuery', () => {
     const sourceInfo = {};
     const metadata = {
       kind: 'my-kind',
-      id: 'my-id' as string | undefined
+      id: 'my-id' as string | undefined,
+      scored: undefined
     };
     const results1 = await interpretResults(
       mockServer,
@@ -183,7 +184,7 @@ describe('CompletedQuery', () => {
     );
     expect(results2).to.eq('1234');
     expect(spy).to.have.been.calledWith(
-      { kind: 'my-kind', id: 'dummy-id' },
+      { kind: 'my-kind', id: 'dummy-id', scored: undefined },
       resultsPath, interpretedResultsPath, sourceInfo
     );
 

extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts

@@ -24,6 +24,23 @@ describe('run-queries', () => {
     expect(info.dataset).to.eq('file:///abc');
   });
 
+  it('should check if interpreted results can be created', async () => {
+    const info = createMockQueryInfo();
+    (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(true);
+
+    expect(await info.canHaveInterpretedResults()).to.eq(true);
+
+    (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(false);
+    expect(await info.canHaveInterpretedResults()).to.eq(false);
+
+    (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(true);
+    info.metadata!.kind = undefined;
+    expect(await info.canHaveInterpretedResults()).to.eq(false);
+
+    info.metadata!.kind = 'table';
+    expect(await info.canHaveInterpretedResults()).to.eq(false);
+  });
+
   describe('compile', () => {
     it('should compile', async () => {
       const info = createMockQueryInfo();
@@ -73,9 +90,14 @@ describe('run-queries', () => {
       {
         contents: {
           datasetUri: 'file:///abc'
-        }
+        },
+        hasMetadataFile: sinon.stub()
       } as unknown as DatabaseItem,
-      'my-scheme' // queryDbscheme
+      'my-scheme', // queryDbscheme,
+      undefined,
+      {
+        kind: 'problem'
+      }
     );
   }
 

extensions/ql-vscode/src/vscode-tests/no-workspace/telemetry.test.ts

@@ -0,0 +1,380 @@
+import * as chai from 'chai';
+import 'mocha';
+import 'sinon-chai';
+import * as sinon from 'sinon';
+import * as chaiAsPromised from 'chai-as-promised';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ExtensionContext, workspace, ConfigurationTarget, window } from 'vscode';
+import { TelemetryListener, telemetryListener as globalTelemetryListener } from '../../telemetry';
+import { UserCancellationException } from '../../commandRunner';
+import { fail } from 'assert';
+import { ENABLE_TELEMETRY } from '../../config';
+
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
+const sandbox = sinon.createSandbox();
+
+describe('telemetry reporting', function() {
+  // setting preferences can trigger lots of background activity
+  // so need to bump up the timeout of this test.
+  this.timeout(10000);
+
+  let originalTelemetryExtension: boolean | undefined;
+  let originalTelemetryGlobal: boolean | undefined;
+  let isCanary: string;
+  let ctx: ExtensionContext;
+  let telemetryListener: TelemetryListener;
+
+  beforeEach(async () => {
+    try {
+      // in case a previous test has accidentally activated this extension,
+      // need to disable it first.
+      // Accidentaly activation may happen asynchronously due to activationEvents
+      // specified in the package.json.
+      globalTelemetryListener?.dispose();
+
+      ctx = createMockExtensionContext();
+
+      sandbox.stub(TelemetryReporter.prototype, 'sendTelemetryEvent');
+      sandbox.stub(TelemetryReporter.prototype, 'sendTelemetryException');
+      sandbox.stub(TelemetryReporter.prototype, 'dispose');
+
+      originalTelemetryExtension = workspace.getConfiguration().get<boolean>('codeQL.telemetry.enableTelemetry');
+      originalTelemetryGlobal = workspace.getConfiguration().get<boolean>('telemetry.enableTelemetry');
+      isCanary = (!!workspace.getConfiguration().get<boolean>('codeQL.canary')).toString();
+
+      // each test will default to telemetry being enabled
+      await enableTelemetry('telemetry', true);
+      await enableTelemetry('codeQL.telemetry', true);
+
+      telemetryListener = new TelemetryListener('my-id', '1.2.3', 'fake-key', ctx);
+      await wait(100);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+
+  afterEach(async () => {
+    telemetryListener?.dispose();
+    // await wait(100);
+    try {
+      sandbox.restore();
+      await enableTelemetry('telemetry', originalTelemetryGlobal);
+      await enableTelemetry('codeQL.telemetry', originalTelemetryExtension);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+
+  it('should initialize telemetry when both options are enabled', async () => {
+    await telemetryListener.initialize();
+
+    expect(telemetryListener._reporter).not.to.be.undefined;
+
+    const reporter: any = telemetryListener._reporter;
+    expect(reporter.extensionId).to.eq('my-id');
+    expect(reporter.extensionVersion).to.eq('1.2.3');
+    expect(reporter.userOptIn).to.eq(true); // enabled
+  });
+
+  it('should initialize telemetry when global option disabled', async () => {
+    try {
+      await enableTelemetry('telemetry', false);
+      await telemetryListener.initialize();
+      expect(telemetryListener._reporter).not.to.be.undefined;
+
+      const reporter: any = telemetryListener._reporter;
+      expect(reporter.userOptIn).to.eq(false); // disabled
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should not initialize telemetry when extension option disabled', async () => {
+    try {
+      await enableTelemetry('codeQL.telemetry', false);
+      await telemetryListener.initialize();
+
+      expect(telemetryListener._reporter).to.be.undefined;
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should not initialize telemetry when both options disabled', async () => {
+    await enableTelemetry('codeQL.telemetry', false);
+    await enableTelemetry('telemetry', false);
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).to.be.undefined;
+  });
+
+  it('should dispose telemetry object when re-initializing and should not add multiple', async () => {
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    const firstReporter = telemetryListener._reporter;
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    expect(telemetryListener._reporter).not.to.eq(firstReporter);
+
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+
+    // initializing a third time continues to dispose
+    await telemetryListener.initialize();
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledTwice;
+  });
+
+  it('should reinitialize reporter when extension setting changes', async () => {
+    await telemetryListener.initialize();
+
+    expect(TelemetryReporter.prototype.dispose).not.to.have.been.called;
+    expect(telemetryListener._reporter).not.to.be.undefined;
+
+    // this disables the reporter
+    await enableTelemetry('codeQL.telemetry', false);
+
+    expect(telemetryListener._reporter).to.be.undefined;
+
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+
+    // creates a new reporter, but does not dispose again
+    await enableTelemetry('codeQL.telemetry', true);
+
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+  });
+
+  it('should set userOprIn to false when global setting changes', async () => {
+    await telemetryListener.initialize();
+
+    const reporter: any = telemetryListener._reporter;
+    expect(reporter.userOptIn).to.eq(true); // enabled
+
+    await enableTelemetry('telemetry', false);
+    expect(reporter.userOptIn).to.eq(false); // disabled
+  });
+
+  it('should send an event', async () => {
+    await telemetryListener.initialize();
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Success',
+        isCanary
+      },
+      { executionTime: 1234 });
+
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should send a command usage event with an error', async () => {
+    await telemetryListener.initialize();
+
+    telemetryListener.sendCommandUsage('command-id', 1234, new UserCancellationException());
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Cancelled',
+        isCanary
+      },
+      { executionTime: 1234 });
+
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should avoid sending an event when telemetry is disabled', async () => {
+    await telemetryListener.initialize();
+    await enableTelemetry('codeQL.telemetry', false);
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+    telemetryListener.sendCommandUsage('command-id', 1234, new Error());
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).not.to.have.been.called;
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should send an event when telemetry is re-enabled', async () => {
+    await telemetryListener.initialize();
+    await enableTelemetry('codeQL.telemetry', false);
+    await enableTelemetry('codeQL.telemetry', true);
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Success',
+        isCanary
+      },
+      { executionTime: 1234 });
+  });
+
+  it('should filter undesired properties from telemetry payload', async () => {
+    await telemetryListener.initialize();
+    // Reach into the internal appInsights client to grab our telemetry processor.
+    const telemetryProcessor: Function =
+      ((telemetryListener._reporter as any).appInsightsClient._telemetryProcessors)[0];
+    const envelop = {
+      tags: {
+        'ai.cloud.roleInstance': true,
+        other: true
+      },
+      data: {
+        baseData: {
+          properties: {
+            'common.remotename': true,
+            other: true
+          }
+        }
+      }
+    };
+    const res = telemetryProcessor(envelop);
+    expect(res).to.eq(true);
+    expect(envelop).to.deep.eq({
+      tags: {
+        other: true
+      },
+      data: {
+        baseData: {
+          properties: {
+            other: true
+          }
+        }
+      }
+    });
+  });
+
+  it('should request permission if popup has never been seen before', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolvesArg(3 /* "yes" item */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    await enableTelemetry('codeQL.telemetry', false);
+
+    await telemetryListener.initialize();
+
+    // Dialog opened, user clicks "yes" and telemetry enabled
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(true);
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.true;
+  });
+
+  it('should prevent telemetry if permission is denied', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolvesArg(4 /* "no" item */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    await enableTelemetry('codeQL.telemetry', true);
+
+    await telemetryListener.initialize();
+
+    // Dialog opened, user clicks "no" and telemetry disabled
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(false);
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.true;
+  });
+
+  it('should unchange telemetry if permission dialog is dismissed', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+
+    // this causes requestTelemetryPermission to be called
+    await enableTelemetry('codeQL.telemetry', false);
+
+    // Dialog opened, and user closes without interacting with it
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(false);
+    // dialog was canceled, so should not have marked as viewed
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+  });
+
+  it('should unchange telemetry if permission dialog is cancelled if starting as true', async () => {
+    await enableTelemetry('codeQL.telemetry', false);
+
+    // as before, except start with telemetry enabled. It should _stay_ enabled if the
+    // dialog is canceled.
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+
+    // this causes requestTelemetryPermission to be called
+    await enableTelemetry('codeQL.telemetry', true);
+
+    // Dialog opened, and user closes without interacting with it
+    // Telemetry state should not have changed
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(true);
+    // dialog was canceled, so should not have marked as viewed
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+  });
+
+  it('should avoid showing dialog if global telemetry is disabled', async () => {
+    // when telemetry is disabled globally, we never want to show the
+    // opt in/out dialog. We just assume that codeql telemetry should
+    // remain disabled as well.
+    // If the user ever turns global telemetry back on, then we can
+    // show the dialog.
+
+    await enableTelemetry('telemetry', false);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    sandbox.stub(window, 'showInformationMessage');
+
+    await telemetryListener.initialize();
+
+    // popup should not be shown even though we have initialized telemetry
+    expect(window.showInformationMessage).not.to.have.been.called;
+  });
+
+  // This test is failing because codeQL.canary is not a registered configuration.
+  // We do not want to have it registered because we don't want this item
+  // appearing in the settings page. It needs to olny be set by users we tell
+  // about it to.
+  // At this point, I see no other way of testing re-requesting permission.
+  xit('should request permission again when user changes canary setting', async () => {
+    // initially, both canary and telemetry are false
+    await workspace.getConfiguration().update('codeQL.canary', false);
+    await enableTelemetry('codeQL.telemetry', false);
+    await ctx.globalState.update('telemetry-request-viewed', true);
+    await telemetryListener.initialize();
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+
+    // set canary to true
+    await workspace.getConfiguration().update('codeQL.canary', true);
+
+    // now, we should have to click through the telemetry requestor again
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+  });
+
+  function createMockExtensionContext(): ExtensionContext {
+    return {
+      globalState: {
+        _state: {
+          'telemetry-request-viewed': true
+        } as Record<string, any>,
+        get(key: string) {
+          return this._state[key];
+        },
+        update(key: string, val: any) {
+          this._state[key] = val;
+        }
+      }
+    } as any;
+  }
+
+  async function enableTelemetry(section: string, value: boolean | undefined) {
+    await workspace.getConfiguration(section).update(
+      'enableTelemetry', value, ConfigurationTarget.Global
+    );
+
+    // Need to wait some time since the onDidChangeConfiguration listeners fire
+    // asynchronously and we sometimes need to wait for them to complete in
+    // order to have as successful test.
+    await wait(50);
+  }
+
+  async function wait(ms = 0) {
+    return new Promise(resolve =>
+      setTimeout(resolve, ms)
+    );
+  }
+});

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -112,18 +112,21 @@ function getLaunchArgs(dir: TestDir) {
   switch (dir) {
     case TestDir.NoWorksspace:
       return [
-        '--disable-extensions'
+        '--disable-extensions',
+        '--disable-gpu'
       ];
 
     case TestDir.MinimalWorksspace:
       return [
         '--disable-extensions',
+        '--disable-gpu',
         path.resolve(__dirname, '../../test/data')
       ];
 
     case TestDir.CliIntegration:
       // CLI integration tests requires a multi-root workspace so that the data and the QL sources are accessible.
       return [
+        '--disable-gpu',
         path.resolve(__dirname, '../../test/data'),
         process.env.TEST_CODEQL_PATH!
       ];

extensions/ql-vscode/src/vscode-tests/test-dispose-handler.ts

@@ -0,0 +1,14 @@
+import { Disposable } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+
+export function testDisposeHandler(disposable: any & Disposable) {
+  if (disposable.onDidExpandElement && disposable.onDidCollapseElement && disposable.reveal) {
+    // This looks like a treeViewer. Don't dispose
+    return;
+  }
+  if (disposable instanceof DisposableObject) {
+    disposable.dispose(testDisposeHandler);
+  } else {
+    disposable.dispose();
+  }
+}

extensions/ql-vscode/src/vscode-utils/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from './disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from 'vscode';
 
 /**
@@ -62,4 +62,3 @@ export class MultiFileSystemWatcher extends DisposableObject {
     this._onDidChange.fire(uri);
   }
 }
-

extensions/ql-vscode/src/vscode-utils/ui-service.ts

@@ -1,5 +1,5 @@
 import { TreeDataProvider, window } from 'vscode';
-import { DisposableObject } from './disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import { commandRunner } from '../commandRunner';
 
 /**

extensions/ql-vscode/test/pure-tests/disposable-object.test.ts

@@ -0,0 +1,122 @@
+import 'chai';
+import 'chai/register-should';
+import 'sinon-chai';
+import * as sinon from 'sinon';
+import 'mocha';
+
+import { DisposableObject } from '../../src/pure/disposable-object';
+import { expect } from 'chai';
+
+describe('DisposableObject and DisposeHandler', () => {
+
+  let disposable1: { dispose: sinon.SinonSpy };
+  let disposable2: { dispose: sinon.SinonSpy };
+  let disposable3: { dispose: sinon.SinonSpy };
+  let disposable4: { dispose: sinon.SinonSpy };
+  let disposableObject: any;
+  let nestedDisposableObject: any;
+  const sandbox = sinon.createSandbox();
+
+  beforeEach(() => {
+    sandbox.restore();
+    disposable1 = { dispose: sandbox.spy() };
+    disposable2 = { dispose: sandbox.spy() };
+    disposable3 = { dispose: sandbox.spy() };
+    disposable4 = { dispose: sandbox.spy() };
+
+    disposableObject = new MyDisposableObject();
+    nestedDisposableObject = new MyDisposableObject();
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it('should dispose tracked and pushed objects', () => {
+    disposableObject.push(disposable1);
+    disposableObject.push(disposable2);
+    disposableObject.track(nestedDisposableObject);
+    nestedDisposableObject.track(disposable3);
+
+    disposableObject.dispose();
+
+    expect(disposable1.dispose).to.have.been.called;
+    expect(disposable2.dispose).to.have.been.called;
+    expect(disposable3.dispose).to.have.been.called;
+
+    // pushed items must be called in reverse order
+    sinon.assert.callOrder(disposable2.dispose, disposable1.dispose);
+
+    // now that disposableObject has been disposed, subsequent disposals are
+    // no-ops
+    disposable1.dispose.resetHistory();
+    disposable2.dispose.resetHistory();
+    disposable3.dispose.resetHistory();
+
+    disposableObject.dispose();
+
+    expect(disposable1.dispose).not.to.have.been.called;
+    expect(disposable2.dispose).not.to.have.been.called;
+    expect(disposable3.dispose).not.to.have.been.called;
+  });
+
+  it('should dispose and stop tracking objects', () => {
+    disposableObject.track(disposable1);
+    disposableObject.disposeAndStopTracking(disposable1);
+
+    expect(disposable1.dispose).to.have.been.called;
+    disposable1.dispose.resetHistory();
+
+    disposableObject.dispose();
+    expect(disposable1.dispose).not.to.have.been.called;
+  });
+
+  it('should avoid disposing an object that is not tracked', () => {
+    disposableObject.push(disposable1);
+    disposableObject.disposeAndStopTracking(disposable1);
+
+    expect(disposable1.dispose).not.to.have.been.called;
+
+    disposableObject.dispose();
+    expect(disposable1.dispose).to.have.been.called;
+  });
+
+  it('ahould use a dispose handler', () => {
+    const handler = (d: any) => (d === disposable1 || d === disposable3 || d === nestedDisposableObject)
+      ? d.dispose(handler)
+      : void (0);
+
+    disposableObject.push(disposable1);
+    disposableObject.push(disposable2);
+    disposableObject.track(nestedDisposableObject);
+    nestedDisposableObject.track(disposable3);
+    nestedDisposableObject.track(disposable4);
+
+    disposableObject.dispose(handler);
+
+    expect(disposable1.dispose).to.have.been.called;
+    expect(disposable2.dispose).not.to.have.been.called;
+    expect(disposable3.dispose).to.have.been.called;
+    expect(disposable4.dispose).not.to.have.been.called;
+
+    // now that disposableObject has been disposed, subsequent disposals are
+    // no-ops
+    disposable1.dispose.resetHistory();
+    disposable2.dispose.resetHistory();
+    disposable3.dispose.resetHistory();
+    disposable4.dispose.resetHistory();
+
+    disposableObject.dispose();
+
+    expect(disposable1.dispose).not.to.have.been.called;
+    expect(disposable2.dispose).not.to.have.been.called;
+    expect(disposable3.dispose).not.to.have.been.called;
+    expect(disposable4.dispose).not.to.have.been.called;
+  });
+
+  class MyDisposableObject extends DisposableObject {
+    constructor() {
+      super();
+    }
+  }
+});

extensions/ql-vscode/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json.schemastore.org/tsconfig",
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
     "strict": true,