v1.5.0

This version of VS Code has workspace trust enabled by default

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,7 +10,12 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
-**To Reproduce**
+**Version**
+The CodeQL and VS Code version in which the bug occurs.
+<!-- To copy version information for the CodeQL extension, click "CodeQL CLI vX.X.X" in the status bar at the bottom of the screen. 
+To copy detailed version information for VS Code itself, see https://code.visualstudio.com/docs/supporting/FAQ#_how-do-i-find-the-version. -->
+
+**To reproduce**
 Steps to reproduce the behavior.
 
 **Expected behavior**

.github/pull_request_template.md

@@ -9,4 +9,4 @@ Replace this with a description of the changes your pull request makes.
 
 - [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
 - [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] `@github/docs-content-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.
+- [ ] `@github/docs-content-codeql` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -2,24 +2,30 @@ name: "Code Scanning - CodeQL"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
-    - cron: '0 0 * * 0'
+    - cron: '21 17 * * 0'
 
 jobs:
   codeql:
-    strategy:
-      fail-fast: false
     runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      security-events: write
+      pull-requests: read
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@main
         with:
             languages: javascript
             config-file: ./.github/codeql/codeql-config.yml
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@main

.github/workflows/main.yml

@@ -1,5 +1,6 @@
 name: Build Extension
 on:
+  workflow_dispatch:
   pull_request:
   push:
     branches:
@@ -20,17 +21,19 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
       - name: Install dependencies
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm install
         shell: bash
 
       - name: Build
+        working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
-          cd extensions/ql-vscode
           npm run build
         shell: bash
 
@@ -61,24 +64,25 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
-      # We have to build the dependencies in `lib` before running any tests.
       - name: Install dependencies
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm install
         shell: bash
 
       - name: Build
+        working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
-          cd extensions/ql-vscode
           npm run build
         shell: bash
 
       - name: Lint
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run lint
 
       - name: Install CodeQL
@@ -91,27 +95,76 @@ jobs:
         shell: bash
 
       - name: Run unit tests (Linux)
+        working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
-          cd extensions/ql-vscode
           CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
 
       - name: Run unit tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
         if: matrix.os == 'ubuntu-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           sudo apt-get install xvfb
           /usr/bin/xvfb-run npm run integration
 
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run integration
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ['v2.2.6', 'v2.3.3', 'v2.4.6', 'v2.5.5']
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.14.0'
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Checkout QL
+        uses: actions/checkout@v2
+        with:
+          repository: github/codeql
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          /usr/bin/xvfb-run npm run cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run cli-integration

.github/workflows/release.yml

@@ -20,7 +20,6 @@ jobs:
   build:
     name: Release
     runs-on: ubuntu-latest
-    # TODO Share steps with the main workflow.
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -32,11 +31,14 @@ jobs:
       - name: Install dependencies
         run: |
           cd extensions/ql-vscode
-          npm install
+          npm ci
         shell: bash
 
       - name: Build
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
+          echo "APP INSIGHTS KEY LENGTH: ${#APP_INSIGHTS_KEY}"
           cd extensions/ql-vscode
           npm run build -- --release
         shell: bash
@@ -55,9 +57,6 @@ jobs:
           REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
-      # Uploading artifacts is not necessary to create a release.
-      # This is just in case the release itself fails and we want to access the built artifacts from Actions.
-      # TODO Remove if not useful.
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -93,6 +92,10 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
+      ###
+      # Do Post release work: version bump and changelog PR
+      # Only do this if we are running from a PR (ie- this is part of the release process)
+
       # The checkout action does not fetch the main branch.
       # Fetch the main branch so that we can base the version bump PR against main.
       - name: Fetch main branch
@@ -126,3 +129,40 @@ jobs:
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
           base: main
+
+  vscode-publish:
+    name: Publish to VS Code Marketplace
+    needs: build
+    environment: publish-vscode-marketplace
+    runs-on: ubuntu-latest
+    env:
+      VSCE_TOKEN: ${{ secrets.VSCE_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix || \
+            echo "Failed to publish to VS Code Marketplace. \
+                  If this was an authentication problem, please make sure the \
+                  auth token hasn't expired."
+
+  open-vsx-publish:
+    name: Publish to Open VSX Registry
+    needs: build
+    environment: publish-open-vsx
+    runs-on: ubuntu-latest
+    env:
+      OPEN_VSX_TOKEN: ${{ secrets.OPEN_VSX_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx ovsx publish -p $OPEN_VSX_TOKEN *.vsix

.gitignore

@@ -4,6 +4,7 @@
 # Generated files
 /dist/
 out/
+build/
 server/
 node_modules/
 gen/

.vscode/extensions.json

@@ -1,5 +1,5 @@
 {
-  // See http://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
+  // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [

.vscode/launch.json

@@ -8,7 +8,9 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        // Add a reference to a workspace to open. Eg-
+        // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -77,6 +79,25 @@
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
+    },
+    {
+      "name": "Launch Integration Tests - With CLI",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
+        // Add a path to a checked out instance of the codeql repository so the libraries are
+        // available in the workspace for the tests.
+        // "${workspaceRoot}/../codeql"
+      ],
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "outFiles": [
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+      ],
     }
   ]
 }

CONTRIBUTING.md

@@ -25,12 +25,11 @@ Here are a few things you can do that will increase the likelihood of your pull
 * Follow the [style guide][style].
 * Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
 * Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
-Make sure you have a fairly recent version of vscode (>1.32) and are using nodejs
-version >=v10.13.0. (Tested on v10.15.1 and v10.16.0).
+Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
 
 ### Installing all packages
 
@@ -57,7 +56,8 @@ We recommend that you keep `npm run watch` running in the backgound and you only
 
 1. on first checkout
 2. whenever any of the non-TypeScript resources have changed
-3. on any change to files included in the webview
+3. on any change to files included in one of the webviews
+   - **Important**: This is easy to forget. You must explicitly run `npm run build` whenever one of the files in the webview is changed. These are the files in the `src/view` and `src/compare/view` folders.
 
 ### Installing the extension
 
@@ -94,7 +94,7 @@ Alternatively, you can run the tests inside of vscode. There are several vscode
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
-1. Double-check that the extension `package.json` has the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
+1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
 1. Create a PR for this release:
     * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
     * Create a new branch for the release named after the new version. For example: `v1.3.6`
@@ -114,14 +114,32 @@ Alternatively, you can run the tests inside of vscode. There are several vscode
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
-1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
-1. Click the `...` menu in the CodeQL row and click **Update**.
-1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+    - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
+1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
+1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
 1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
 1. If documentation changes need to be published, notify documentation team that release has been made.
 1. Review and merge the version bump PR that is automatically created by Actions.
 
+## Secrets and authentication for publishing
+
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+
+To regenerate the Open VSX token:
+
+1. Log in to the [user settings page on Open VSX](https://open-vsx.org/user-settings/namespaces).
+1. Make sure you are a member of the GitHub namespace.
+1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
+1. Update the secret in the `publish-open-vsx` environment in the project settings.
+
+To regenerate the VSCode Marketplace token:
+
+1. Follow the instructions on [getting a PAT for Azure DevOps](https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token).
+1. Update the secret in the `publish-vscode-marketplace` environment in the project settings.
+
+Not that Azure DevOps PATs expire yearly and must be regenerated.
+
 ## Resources
 
 * [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)

README.md

@@ -16,7 +16,6 @@ To see what has changed in the last few versions of the extension, see the [Chan
 * Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
-
 ## Project goals and scope
 
 This project will track new feature development in CodeQL and, whenever appropriate, bring that functionality to the Visual Studio Code experience.

extensions/ql-vscode/.eslintrc.js

@@ -22,8 +22,10 @@ module.exports = {
       },
     ],
     "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/explicit-module-boundary-types": "off",
     "@typescript-eslint/no-non-null-assertion": "off",
     "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-floating-promises": [ "error", { ignoreVoid: true } ],
     "prefer-const": ["warn", { destructuring: "all" }],
     indent: "off",
     "@typescript-eslint/indent": "off",

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,94 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.5.0 - 14 June 2021
+
+- Display CodeQL CLI version being downloaded during an upgrade. [#862](https://github.com/github/vscode-codeql/pull/862)
+- Display a helpful message and link to documentation when a query produces no results. [#866](https://github.com/github/vscode-codeql/pull/866)
+- Refresh test databases automatically after a test run. [#868](https://github.com/github/vscode-codeql/pull/868)
+- Allow users to specify a custom directory for storing query server logs (`codeQL.runningQueries.customLogDirectory`). The extension will not delete these logs automatically. [#863](https://github.com/github/vscode-codeql/pull/863)
+- Support the VS Code [Workspace Trust feature](https://code.visualstudio.com/docs/editor/workspace-trust). This extension is now enabled in untrusted workspaces, but it restricts commands that contain arbitrary paths. [#861](https://github.com/github/vscode-codeql/pull/861)
+- Allow the `codeQL.cli.executablePath` configuration setting to be set in workspace-scoped configuration files. This means that each workspace can now specify its own CodeQL CLI compiler, a feature that is unblocked due to implementing Workspace Trust. [#861](https://github.com/github/vscode-codeql/pull/861)
+
+## 1.4.8 - 05 May 2021
+
+- Copy version information to the clipboard when a user clicks the CodeQL section of the status bar. [#845](https://github.com/github/vscode-codeql/pull/845)
+- Ensure changes in directories that contain tests will be properly updated in the test explorer. [#846](https://github.com/github/vscode-codeql/pull/846)
+- Remind users to choose a language when downloading a database from LGTM. [#852](https://github.com/github/vscode-codeql/pull/852)
+
+## 1.4.7 - 23 April 2021
+
+- Fix a bug that prevented the results view from being loaded. [#842](https://github.com/github/vscode-codeql/pull/842)
+
+## 1.4.6 - 21 April 2021
+
+- Avoid showing an error popup when running a query with `@kind table` metadata. [#814](https://github.com/github/vscode-codeql/pull/814)
+- Add an option to jump from a .qlref file to the .ql file it references. [#815](https://github.com/github/vscode-codeql/pull/815)
+- Avoid opening the results panel when a database is deleted. [#831](https://github.com/github/vscode-codeql/pull/831)
+- Forward all query metadata to the CLI when interpreting results. [#838](https://github.com/github/vscode-codeql/pull/838)
+
+## 1.4.5 - 22 March 2021
+
+- Avoid showing an error popup when user runs a query without `@kind` metadata. [#801](https://github.com/github/vscode-codeql/pull/801)
+- Fix running of tests when the `ms-python` extension is installed. [#803](https://github.com/github/vscode-codeql/pull/803)
+
+## 1.4.4 - 19 March 2021
+
+- Introduce evaluator options for saving intermediate results to the disk cache (`codeQL.runningQueries.saveCache`) and for limiting the size of this cache (`codeQL.runningQueries.cacheSize`). [#778](https://github.com/github/vscode-codeql/pull/778)
+- Respect the `codeQL.runningQueries.numberOfThreads` setting when creating SARIF files during result interpretation. [#771](https://github.com/github/vscode-codeql/pull/771)
+- Allow using raw LGTM project slugs for fetching LGTM databases. [#769](https://github.com/github/vscode-codeql/pull/769)
+- Better error messages when BQRS interpretation fails to produce SARIF. [#770](https://github.com/github/vscode-codeql/pull/770)
+- Implement sorting of the query history view by name, date, and results count. [#777](https://github.com/github/vscode-codeql/pull/777)
+- Add a configuration option to pass additional arguments to the CLI when running tests. [#785](https://github.com/github/vscode-codeql/pull/785)
+- Introduce option to view query results as CSV. [#784](https://github.com/github/vscode-codeql/pull/784)
+- Add some snippets for commonly used QL statements. [#782](https://github.com/github/vscode-codeql/pull/782)
+- More descriptive error messages on QL test failures. [#788](https://github.com/github/vscode-codeql/pull/788)
+
+## 1.4.3 - 22 February 2021
+
+- Avoid displaying an error when removing orphaned databases and the storage folder does not exist. [#748](https://github.com/github/vscode-codeql/pull/748)
+- Add better error messages when AST Viewer is unable to create an AST. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Cache AST viewing operations so that subsequent calls to view the AST of a single file will be extremely fast. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Ensure CodeQL version in status bar updates correctly when version changes. [#754](https://github.com/github/vscode-codeql/pull/754)
+- Avoid deleting the quick query file when it is re-opened. [#747](https://github.com/github/vscode-codeql/pull/747)
+
+## 1.4.2 - 2 February 2021
+
+- Add a status bar item for the CodeQL CLI to show the current version. [#741](https://github.com/github/vscode-codeql/pull/741)
+- Fix version constraint for flagging CLI support of non-destructive updates. [#744](https://github.com/github/vscode-codeql/pull/744)
+- Add a _More Information_ button in the telemetry popup that opens the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) in a browser tab. [#742](https://github.com/github/vscode-codeql/pull/742)
+
+## 1.4.1 - 29 January 2021
+
+- Reword the telemetry modal dialog box. [#738](https://github.com/github/vscode-codeql/pull/738)
+
+## 1.4.0 - 29 January 2021
+
+- Fix bug where databases are not reregistered when the query server restarts. [#734](https://github.com/github/vscode-codeql/pull/734)
+- Fix bug where upgrade requests were erroneously being marked as failed. [#734](https://github.com/github/vscode-codeql/pull/734)
+- On a strictly opt-in basis, collect anonymized usage data from the VS Code extension, helping improve CodeQL's usability and performance. See the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) for more information on exactly what data is collected and what it is used for. [#611](https://github.com/github/vscode-codeql/pull/611)
+
+## 1.3.10 - 20 January 2021
+
+- Include the full stack in error log messages to help with debugging. [#726](https://github.com/github/vscode-codeql/pull/726)
+
+## 1.3.9 - 12 January 2021
+
+- No changes visible to end users.
+
+## 1.3.8 - 17 December 2020
+
+- Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting the IDE. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)
+- Fix bug when removing databases where sometimes the source folder would not also be removed from the workspace or the database files would not be deleted from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692)
+- Query results with no string representation will now be displayed with placeholder text in query results. Previously, they were omitted. [#694](https://github.com/github/vscode-codeql/pull/694)
+- Add a label for the language of a database in the databases view. This will only take effect for new databases created with the CodeQL CLI v2.4.1 or later. [#697](https://github.com/github/vscode-codeql/pull/697)
+- Add clearer error message when running a query using a missing or invalid qlpack. [#702](https://github.com/github/vscode-codeql/pull/702)
+- Add clearer error message when trying to run a command from the query history view if no item in the history is selected. [#702](https://github.com/github/vscode-codeql/pull/702)
+- Fix a bug where it is not possible to download some database archives. This fix specifically addresses large archives and archives whose central directories do not align with file headers. [#700](https://github.com/github/vscode-codeql/pull/700)
+- Avoid error dialogs when QL test discovery or database cleanup encounters a missing directory. [#706](https://github.com/github/vscode-codeql/pull/706)
+- Add descriptive text and a link in the results view. [#711](https://github.com/github/vscode-codeql/pull/711)
+- Fix the _Set Label_ command in the query history view. [#710](https://github.com/github/vscode-codeql/pull/710)
+- Add the _CodeQL: View AST_ command to the right-click context menu when a source file in a database source archive is open in the editor. [#712](https://github.com/github/vscode-codeql/pull/712)
+
 ## 1.3.7 - 24 November 2020
 
 - Editors opened by navigating from the results view are no longer opened in _preview mode_. Now they are opened as a persistent editor. [#630](https://github.com/github/vscode-codeql/pull/630)

extensions/ql-vscode/README.md

@@ -1,6 +1,6 @@
 # CodeQL extension for Visual Studio Code
 
-This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
+This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://codeql.github.com/docs/) and allows you to easily find problems in codebases. In particular, the extension:
 
 - Enables you to use CodeQL to query databases generated from source code.
 - Shows the flow of data through the results of path queries, which is essential for triaging security results.
@@ -12,7 +12,7 @@ To see what has changed in the last few versions of the extension, see the [Chan
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
-For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
+For information about other configurations, see the separate [CodeQL help](https://codeql.github.com/docs/codeql-for-visual-studio-code/).
 
 ### Quick start: Installing and configuring the extension
 
@@ -40,9 +40,9 @@ The CodeQL extension requires a minimum of Visual Studio Code 1.39. Older versio
 
 ### Checking access to the CodeQL CLI
 
-The extension uses the [CodeQL CLI](https://help.semmle.com/codeql/codeql-cli.html) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
+The extension uses the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
 
-If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#configuring-access-to-the-codeql-cli).
+If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#configuring-access-to-the-codeql-cli).
 
 If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Log** in the **Output** view for any error messages.
 
@@ -52,7 +52,7 @@ When you're working with CodeQL, you need access to the standard CodeQL librarie
 Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
-For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
+For information about configuring an existing workspace for CodeQL, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#updating-an-existing-workspace-for-codeql).
 
 ## Upgrading CodeQL standard libraries
 
@@ -75,7 +75,7 @@ You can find all the commands contributed by the extension in the Command Palett
 
 ### Importing a database from LGTM
 
-While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
+While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from LGTM.com.
 
 1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
 1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
@@ -100,13 +100,17 @@ If there are any problems running a query, a notification is displayed in the bo
 
 ## What next?
 
-For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
+For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:
 
-- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Create a database for a different codebase](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/).
 - [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Learn more about CodeQL](https://codeql.github.com/docs/).
 - [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
 
 The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the MIT License. The version of CodeQL used by the CodeQL extension is subject to the [GitHub CodeQL Terms & Conditions](https://securitylab.github.com/tools/codeql/license).
+
+## Data and Telemetry
+
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information about telemetry, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code).

extensions/ql-vscode/gulpfile.ts/appInsights.ts

@@ -0,0 +1,16 @@
+import * as gulp from 'gulp';
+import * as replace from 'gulp-replace';
+
+/** Inject the application insights key into the telemetry file */
+export function injectAppInsightsKey() {
+  if (!process.env.APP_INSIGHTS_KEY) {
+    // noop
+    console.log('APP_INSIGHTS_KEY environment variable is not set. So, cannot inject it into the application.');
+    return Promise.resolve();
+  }
+
+  // replace the key
+  return gulp.src(['out/telemetry.js'])
+    .pipe(replace(/REPLACE-APP-INSIGHTS-KEY/, process.env.APP_INSIGHTS_KEY))
+    .pipe(gulp.dest('out/'));
+}

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -13,6 +13,7 @@ const packageFiles = [
   'CHANGELOG.md',
   'README.md',
   'language-configuration.json',
+  'snippets.json',
   'media',
   'node_modules',
   'out'

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -4,7 +4,12 @@ import { compileTextMateGrammar } from './textmate';
 import { copyTestData } from './tests';
 import { compileView } from './webpack';
 import { packageExtension } from './package';
+import { injectAppInsightsKey } from './appInsights';
 
-export const buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
-export { compileTextMateGrammar, watchTypeScript, compileTypeScript };
-exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);
+export const buildWithoutPackage =
+  gulp.parallel(
+    compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
+  );
+
+export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData, injectAppInsightsKey };
+export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

extensions/ql-vscode/gulpfile.ts/tests.ts

@@ -1,6 +1,17 @@
 import * as gulp from 'gulp';
 
 export function copyTestData() {
+  copyNoWorkspaceData();
+  copyCliIntegrationData();
+  return Promise.resolve();
+}
+
+function copyNoWorkspaceData() {
   return gulp.src('src/vscode-tests/no-workspace/data/**/*')
     .pipe(gulp.dest('out/vscode-tests/no-workspace/data'));
 }
+
+function copyCliIntegrationData() {
+  return gulp.src('src/vscode-tests/cli-integration/data/**/*')
+    .pipe(gulp.dest('out/vscode-tests/cli-integration/data'));
+}

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json.schemastore.org/tsconfig",
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
     "strict": true,

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -13,7 +13,10 @@ export const config: webpack.Configuration = {
   },
   devtool: 'inline-source-map',
   resolve: {
-    extensions: ['.js', '.ts', '.tsx', '.json']
+    extensions: ['.js', '.ts', '.tsx', '.json'],
+    fallback: {
+      path: require.resolve('path-browserify')
+    }
   },
   module: {
     rules: [

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -6,21 +6,23 @@ export function compileView(cb: (err?: Error) => void) {
     if (error) {
       cb(error);
     }
-    console.log(stats.toString({
-      errorDetails: true,
-      colors: true,
-      assets: false,
-      builtAt: false,
-      version: false,
-      hash: false,
-      entrypoints: false,
-      timings: false,
-      modules: false,
-      errors: true
-    }));
-    if (stats.hasErrors()) {
-      cb(new Error('Compilation errors detected.'));
-      return;
+    if (stats) {
+      console.log(stats.toString({
+        errorDetails: true,
+        colors: true,
+        assets: false,
+        builtAt: false,
+        version: false,
+        hash: false,
+        entrypoints: false,
+        timings: false,
+        modules: false,
+        errors: true
+      }));
+      if (stats.hasErrors()) {
+        cb(new Error('Compilation errors detected.'));
+        return;
+      }
     }
 
     cb();

extensions/ql-vscode/media/dark/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" fill="#C5C5C5" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" fill="#C5C5C5"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/light/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" />
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" />
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" />
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.3.7",
+  "version": "1.5.0",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,7 +13,7 @@
     "url": "https://github.com/github/vscode-codeql"
   },
   "engines": {
-    "vscode": "^1.43.0"
+    "vscode": "^1.57.0"
   },
   "categories": [
     "Programming Languages"
@@ -21,6 +21,16 @@
   "extensionDependencies": [
     "hbenl.vscode-test-explorer"
   ],
+  "capabilities": {
+    "untrustedWorkspaces": {
+      "supported": "limited",
+      "description": "Workspace trust is required to execute commands that can contain arbitrary paths.",
+      "restrictedConfigurations": [
+        "codeQL.cli.executablePath",
+        "codeQL.runningTests.additionalTestArguments"
+      ]
+    }
+  },
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
@@ -34,6 +44,7 @@
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.openReferencedFile",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
@@ -106,15 +117,21 @@
         "path": "./out/syntaxes/dbscheme.tmLanguage.json"
       }
     ],
+    "snippets": [
+      {
+        "language": "ql",
+        "path": "./snippets.json"
+      }
+    ],
     "configuration": {
       "type": "object",
       "title": "CodeQL",
       "properties": {
         "codeQL.cli.executablePath": {
-          "scope": "machine",
+          "scope": "window",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
+          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -123,6 +140,21 @@
           "maximum": 1024,
           "description": "Number of threads for running queries."
         },
+        "codeQL.runningQueries.saveCache": {
+          "type": "boolean",
+          "default": false,
+          "scope": "window",
+          "description": "Aggressively save intermediate results to the disk cache. This may speed up subsequent queries if they are similar. Be aware that using this option will greatly increase disk usage and initial evaluation time."
+        },
+        "codeQL.runningQueries.cacheSize": {
+          "type": [
+            "integer",
+            "null"
+          ],
+          "default": null,
+          "minimum": 1024,
+          "description": "Maximum size of the disk cache (in MB). Leave blank to allow the evaluator to automatically adjust the size of the disk cache based on the size of the codebase and the complexity of the queries being executed."
+        },
         "codeQL.runningQueries.timeout": {
           "type": [
             "integer",
@@ -157,15 +189,29 @@
           "default": 20,
           "description": "Max number of simultaneous queries to run using the 'CodeQL: Run Queries' command."
         },
+        "codeQL.runningQueries.customLogDirectory": {
+          "type": [
+            "string",
+            null
+          ],
+          "default": null,
+          "description": "Path to a directory where the CodeQL extension should store query server logs. If empty, the extension stores logs in a temporary workspace folder and deletes the contents after each run."
+        },
         "codeQL.resultsDisplay.pageSize": {
           "type": "integer",
           "default": 200,
-          "description": "Number of queries displayed per page of the results view."
+          "description": "Max number of query results to display per page in the results view."
         },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "[%t] %q on %d - %s",
-          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+          "default": "%q on %d - %s, %r result count [%t]",
+          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, %r is the number of results, and %s is a status string."
+        },
+        "codeQL.runningTests.additionalTestArguments": {
+          "scope": "window",
+          "type": "array",
+          "default": [],
+          "markdownDescription": "Additional command line arguments to pass to the CLI when [running tests](https://codeql.github.com/docs/codeql-cli/manual/test-run/). This setting should be an array of strings, each containing an argument to be passed."
         },
         "codeQL.runningTests.numberOfThreads": {
           "scope": "window",
@@ -174,6 +220,18 @@
           "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running CodeQL tests."
+        },
+        "codeQL.telemetry.enableTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "markdownDescription": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub. For more information, see the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code)"
+        },
+        "codeQL.telemetry.logTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "description": "Specifies whether or not to write telemetry events to the extension log."
         }
       }
     },
@@ -190,10 +248,22 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.openReferencedFile",
+        "title": "CodeQL: Open Referenced File"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
       },
+      {
+        "command": "codeQL.openDocumentation",
+        "title": "CodeQL: Open Documentation"
+      },
+      {
+        "command": "codeQL.copyVersion",
+        "title": "CodeQL: Copy Version Information"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
@@ -304,7 +374,7 @@
       },
       {
         "command": "codeQLQueryHistory.openQuery",
-        "title": "Open Query",
+        "title": "Open the query that produced these results",
         "icon": {
           "light": "media/light/edit.svg",
           "dark": "media/dark/edit.svg"
@@ -326,6 +396,30 @@
           "dark": "media/dark/trash.svg"
         }
       },
+      {
+        "command": "codeQLQueryHistory.sortByName",
+        "title": "Sort by Name",
+        "icon": {
+          "light": "media/light/sort-alpha.svg",
+          "dark": "media/dark/sort-alpha.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByDate",
+        "title": "Sort by Query Date",
+        "icon": {
+          "light": "media/light/sort-date.svg",
+          "dark": "media/dark/sort-date.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByCount",
+        "title": "Sort by Results Count",
+        "icon": {
+          "light": "media/light/sort-num.svg",
+          "dark": "media/dark/sort-num.svg"
+        }
+      },
       {
         "command": "codeQLQueryHistory.showQueryLog",
         "title": "Show Query Log"
@@ -335,8 +429,12 @@
         "title": "Show Query Text"
       },
       {
-        "command": "codeQLQueryHistory.viewSarif",
-        "title": "View SARIF"
+        "command": "codeQLQueryHistory.viewCsvResults",
+        "title": "View Results (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarifResults",
+        "title": "View Results (SARIF)"
       },
       {
         "command": "codeQLQueryHistory.viewDil",
@@ -430,6 +528,21 @@
           "when": "view == codeQLQueryHistory",
           "group": "navigation"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
         {
           "command": "codeQLAstViewer.clear",
           "when": "view == codeQLAstViewer",
@@ -493,7 +606,12 @@
           "when": "view == codeQLQueryHistory"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifResults",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
         },
@@ -526,7 +644,13 @@
         },
         {
           "command": "codeQL.runQueries",
-          "group": "9_qlCommands"
+          "group": "9_qlCommands",
+          "when": "resourceScheme != codeql-zip-archive"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .qlref"
         }
       ],
       "commandPalette": [
@@ -542,6 +666,10 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -619,7 +747,11 @@
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifResults",
           "when": "false"
         },
         {
@@ -634,6 +766,18 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "false"
+        },
         {
           "command": "codeQLAstViewer.gotoCode",
           "when": "false"
@@ -656,9 +800,17 @@
           "command": "codeQL.runQuery",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
         }
       ]
     },
@@ -690,7 +842,7 @@
     "viewsWelcome": [
       {
         "view": "codeQLAstViewer",
-        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a CodeQL database.\n[View AST](command:codeQL.viewAst)"
       },
       {
         "view": "codeQLQueryHistory",
@@ -698,7 +850,7 @@
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a Code QL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
       }
     ]
   },
@@ -708,7 +860,8 @@
     "watch:extension": "tsc --watch",
     "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
-    "integration": "node ./out/vscode-tests/run-integration-tests.js",
+    "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
+    "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
     "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
@@ -722,6 +875,7 @@
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.5",
     "node-fetch": "~2.6.0",
+    "path-browserify": "^1.0.1",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "semver": "~7.3.2",
@@ -729,6 +883,7 @@
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
+    "vscode-extension-telemetry": "^0.1.6",
     "vscode-jsonrpc": "^5.0.1",
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
@@ -740,14 +895,15 @@
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^9.0.3",
+    "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
+    "@types/gulp-replace": "0.0.31",
     "@types/gulp-sourcemaps": "0.0.32",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
-    "@types/mocha": "~8.0.3",
+    "@types/mocha": "^8.2.0",
     "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
@@ -760,12 +916,13 @@
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.43.0",
+    "@types/vscode": "^1.57.0",
     "@types/webpack": "^4.32.1",
     "@types/xml2js": "~0.4.4",
-    "@typescript-eslint/eslint-plugin": "~2.23.0",
-    "@typescript-eslint/parser": "~2.23.0",
+    "@typescript-eslint/eslint-plugin": "^4.26.0",
+    "@typescript-eslint/parser": "^4.26.0",
     "ansi-colors": "^4.1.1",
+    "applicationinsights": "^1.8.7",
     "chai": "^4.2.0",
     "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
@@ -773,12 +930,13 @@
     "eslint-plugin-react": "~7.19.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
+    "gulp-replace": "^1.0.0",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
     "husky": "~4.2.5",
     "jsonc-parser": "^2.3.0",
     "lint-staged": "~10.2.2",
-    "mocha": "~8.1.3",
+    "mocha": "^8.2.1",
     "mocha-sinon": "~2.1.0",
     "npm-run-all": "^4.1.5",
     "prettier": "~2.0.5",
@@ -787,15 +945,15 @@
     "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
-    "ts-loader": "^5.4.5",
+    "ts-loader": "^8.1.0",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "~3.8.3",
+    "typescript": "^4.3.2",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
     "vscode-test": "^1.4.0",
-    "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2"
+    "webpack": "^5.28.0",
+    "webpack-cli": "^4.6.0"
   },
   "husky": {
     "hooks": {

extensions/ql-vscode/snippets.json

@@ -0,0 +1,134 @@
+{
+  "Query Metadata": {
+    "prefix": "querymetadata",
+    "body": [
+      "/**",
+      " * @name $1",
+      " * @description $2",
+      " * @kind $3",
+      " * @id $4",
+      " * @tags $5",
+      " */"
+    ],
+    "description": "Metadata for a query"
+  },
+  "Class": {
+    "prefix": "class",
+    "body": ["class $1 extends $2 {", "\t$0", "}"],
+    "description": "A class"
+  },
+  "From/Where/Select": {
+    "prefix": "from",
+    "body": ["from $1", "where $2", "select $3"],
+    "description": "A from/where/select statement"
+  },
+  "Predicate": {
+    "prefix": "predicate",
+    "body": ["predicate $1($2) {", "\t$0", "}"],
+    "description": "A predicate"
+  },
+  "Dataflow Tracking Class": {
+    "prefix": "dataflowtracking",
+    "body": [
+      "class $1 extends DataFlow::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a dataflow tracking class"
+  },
+  "Taint Tracking Class": {
+    "prefix": "tainttracking",
+    "body": [
+      "class $1 extends TaintTracking::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a taint tracking class"
+  },
+  "Count": {
+    "prefix": "count",
+    "body": ["count($1 | $2 | $3)"],
+    "description": "A count aggregate"
+  },
+  "Max": {
+    "prefix": "max",
+    "body": ["max($1 | $2 | $3)"],
+    "description": "A max aggregate"
+  },
+  "Min": {
+    "prefix": "min",
+    "body": ["min($1 | $2 | $3)"],
+    "description": "A min aggregate"
+  },
+  "Average": {
+    "prefix": "avg",
+    "body": ["avg($1 | $2 | $3)"],
+    "description": "An average aggregate"
+  },
+  "Sum": {
+    "prefix": "sum",
+    "body": ["sum($1 | $2 | $3)"],
+    "description": "A sum aggregate"
+  },
+  "Concatenation": {
+    "prefix": "concat",
+    "body": ["concat($1 | $2 | $3)"],
+    "description": "A concatenation aggregate"
+  },
+  "Rank": {
+    "prefix": "rank",
+    "body": ["rank[$1]($2 | $3 | $4)"],
+    "description": "A rank aggregate"
+  },
+  "Strict Sum": {
+    "prefix": "strictsum",
+    "body": ["strictsum($1 | $2 | $3)"],
+    "description": "A strict sum aggregate"
+  },
+  "Strict Concatenation": {
+    "prefix": "strictconcat",
+    "body": ["strictconcat($1 | $2 | $3)"],
+    "description": "A strict concatenation aggregate"
+  },
+  "Strict Count": {
+    "prefix": "strictcount",
+    "body": ["strictcount($1 | $2 | $3)"],
+    "description": "A strict count aggregate"
+  },
+  "Unique": {
+    "prefix": "unique",
+    "body": ["unique($1 | $2 | $3)"],
+    "description": "A unique aggregate"
+  },
+  "Exists": {
+    "prefix": "exists",
+    "body": ["exists($1 | $2 | $3)"],
+    "description": "An exists quantifier"
+  },
+  "For All": {
+    "prefix": "forall",
+    "body": ["forall($1 | $2 | $3)"],
+    "description": "A for all quantifier"
+  },
+  "For All and Exists": {
+    "prefix": "forex",
+    "body": ["forex($1 | $2 | $3)"],
+    "description": "A for all and exists quantifier"
+  }
+}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -115,7 +115,7 @@ class InvalidSourceArchiveUriError extends Error {
 export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
   if (!uri.authority) {
     // Uri is malformed, but this is recoverable
-    logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
+    void logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
     return {
       pathWithinSourceArchive: '/',
       sourceArchiveZipPath: uri.path
@@ -141,7 +141,7 @@ function ensureFile(map: DirectoryHierarchyMap, file: string) {
   const dirname = path.dirname(file);
   if (dirname === '.') {
     const error = `Ill-formed path ${file} in zip archive (expected absolute path)`;
-    logger.log(error);
+    void logger.log(error);
     throw new Error(error);
   }
   ensureDir(map, dirname);

extensions/ql-vscode/src/astViewer.ts

@@ -18,8 +18,9 @@ import { DatabaseItem } from './databases';
 import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
 import { showLocation } from './interface-utils';
 import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
-import { commandRunner } from './helpers';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { commandRunner } from './commandRunner';
+import { DisposableObject } from './pure/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
 
 export interface AstItem {
   id: BqrsId;
@@ -40,7 +41,7 @@ class AstViewerDataProvider extends DisposableObject implements TreeDataProvider
   public db: DatabaseItem | undefined;
 
   private _onDidChangeTreeData =
-    new EventEmitter<AstItem | undefined>();
+    this.push(new EventEmitter<AstItem | undefined>());
   readonly onDidChangeTreeData: Event<AstItem | undefined> =
     this._onDidChangeTreeData.event;
 
@@ -55,7 +56,7 @@ class AstViewerDataProvider extends DisposableObject implements TreeDataProvider
   }
 
   refresh(): void {
-    this._onDidChangeTreeData.fire();
+    this._onDidChangeTreeData.fire(undefined);
   }
   getChildren(item?: AstItem): ProviderResult<AstItem[]> {
     const children = item ? item.children : this.roots;
@@ -129,8 +130,13 @@ export class AstViewer extends DisposableObject {
     this.treeDataProvider.db = db;
     this.treeDataProvider.refresh();
     this.treeView.message = `AST for ${path.basename(fileName)}`;
-    this.treeView.reveal(roots[0], { focus: false });
     this.currentFile = fileName;
+    // Handle error on reveal. This could happen if
+    // the tree view is disposed during the reveal.
+    this.treeView.reveal(roots[0], { focus: false })?.then(
+      () => { /**/ },
+      err => showAndLogErrorMessage(err)
+    );
   }
 
   private updateTreeSelection(e: TextEditorSelectionChangeEvent) {
@@ -178,7 +184,12 @@ export class AstViewer extends DisposableObject {
 
       const targetItem = findBest(range, this.treeDataProvider.roots);
       if (targetItem) {
-        this.treeView.reveal(targetItem);
+        // Handle error on reveal. This could happen if
+        // the tree view is disposed during the reveal.
+        this.treeView.reveal(targetItem)?.then(
+          () => { /**/ },
+          err => showAndLogErrorMessage(err)
+        );
       }
     }
   }

extensions/ql-vscode/src/cli-version.ts

@@ -18,7 +18,7 @@ export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): P
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.
-    logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
+    void logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
     return undefined;
   }
 }

extensions/ql-vscode/src/cli.ts

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/camelcase */
 import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
@@ -24,6 +23,11 @@ import { CompilationMessage } from './pure/messages';
  */
 const SARIF_FORMAT = 'sarifv2.1.0';
 
+/**
+ * The string used to specify CSV format.
+ */
+const CSV_FORMAT = 'csv';
+
 /**
  * Flags to pass to all cli commands.
  */
@@ -50,6 +54,7 @@ export interface DbInfo {
   sourceArchiveRoot: string;
   datasetFolder: string;
   logsFolder: string;
+  languages: string[];
 }
 
 /**
@@ -58,6 +63,7 @@ export interface DbInfo {
 export interface UpgradesInfo {
   scripts: string[];
   finalDbscheme: string;
+  matchesTarget?: boolean;
 }
 
 /**
@@ -65,6 +71,11 @@ export interface UpgradesInfo {
  */
 export type QlpacksInfo = { [name: string]: string[] };
 
+/**
+ * The expected output of `codeql resolve qlref`.
+ */
+export type QlrefInfo = { resolvedPath: string };
+
 // `codeql bqrs interpret` requires both of these to be present or
 // both absent.
 export interface SourceInfo {
@@ -96,6 +107,7 @@ export interface TestCompleted {
   evaluationMs: number;
   expected: string;
   diff: string[] | undefined;
+  failureDescription?: string;
 }
 
 /**
@@ -118,10 +130,6 @@ interface BqrsDecodeOptions {
  */
 export class CodeQLCliServer implements Disposable {
 
-  /**
-   * CLI version where --kind=DIL was introduced
-   */
-  private static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
 
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
@@ -133,15 +141,22 @@ export class CodeQLCliServer implements Disposable {
   nullBuffer: Buffer;
 
   /** Version of current cli, lazily computed by the `getVersion()` method */
-  _version: SemVer | undefined;
+  private _version: SemVer | undefined;
 
   /** Path to current codeQL executable, or undefined if not running yet. */
   codeQlPath: string | undefined;
 
+  cliConstraints = new CliVersionConstraint(this);
+
+  /**
+   * When set to true, ignore some modal popups and assume user has clicked "yes".
+   */
+  public quiet = false;
+
   constructor(
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    private logger: Logger
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -149,6 +164,7 @@ export class CodeQLCliServer implements Disposable {
     if (this.distributionProvider.onDidChangeDistribution) {
       this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
+        this._version = undefined;
       });
     }
     if (this.cliConfig.onDidChangeConfiguration) {
@@ -166,15 +182,15 @@ export class CodeQLCliServer implements Disposable {
   killProcessIfRunning(): void {
     if (this.process) {
       // Tell the Java CLI server process to shut down.
-      this.logger.log('Sending shutdown request');
+      void this.logger.log('Sending shutdown request');
       try {
         this.process.stdin.write(JSON.stringify(['shutdown']), 'utf8');
         this.process.stdin.write(this.nullBuffer);
-        this.logger.log('Sent shutdown request');
+        void this.logger.log('Sent shutdown request');
       } catch (e) {
         // We are probably fine here, the process has already closed stdin.
-        this.logger.log(`Shutdown request failed: process stdin may have already closed. The error was ${e}`);
-        this.logger.log('Stopping the process anyway.');
+        void this.logger.log(`Shutdown request failed: process stdin may have already closed. The error was ${e}`);
+        void this.logger.log('Stopping the process anyway.');
       }
       // Close the stdin and stdout streams.
       // This is important on Windows where the child process may not die cleanly.
@@ -254,9 +270,9 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(' ');
-      this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
-        await new Promise((resolve, reject) => {
+        await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
           process.stdout.addListener('data', (newData: Buffer) => {
             stdoutBuffers.push(newData);
@@ -281,7 +297,7 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString('utf8', 0, fullBuffer.length - 1);
-        this.logger.log('CLI command succeeded.');
+        void this.logger.log('CLI command succeeded.');
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
@@ -294,7 +310,7 @@ export class CodeQLCliServer implements Disposable {
         newError.stack += (err.stack || '');
         throw newError;
       } finally {
-        this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
+        void this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
         // Remove the listeners we set up.
         process.stdout.removeAllListeners('data');
         process.stderr.removeAllListeners('data');
@@ -354,7 +370,7 @@ export class CodeQLCliServer implements Disposable {
       }
       if (logger !== undefined) {
         // The human-readable output goes to stderr.
-        logStream(child.stderr!, logger);
+        void logStream(child.stderr!, logger);
       }
 
       for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
@@ -435,12 +451,15 @@ export class CodeQLCliServer implements Disposable {
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
    * @param commandArgs The arguments to pass to the `codeql` command.
    * @param description Description of the action being run, to be shown in log and error messages.
+   * @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
    * @param progressReporter Used to output progress messages, e.g. to the status bar.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
-  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, progressReporter?: ProgressReporter): Promise<OutputType> {
-    // Add format argument first, in case commandArgs contains positional parameters.
-    const args = ['--format', 'json'].concat(commandArgs);
+  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, addFormat = true, progressReporter?: ProgressReporter): Promise<OutputType> {
+    let args: string[] = [];
+    if (addFormat) // Add format argument first, in case commandArgs contains positional parameters.
+      args = args.concat(['--format', 'json']);
+    args = args.concat(commandArgs);
     const result = await this.runCodeQlCliCommand(command, args, description, progressReporter);
     try {
       return JSON.parse(result) as OutputType;
@@ -479,6 +498,18 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  public async resolveQlref(qlref: string): Promise<QlrefInfo> {
+    const subcommandArgs = [
+      qlref
+    ];
+    return await this.runJsonCodeQlCliCommand<QlrefInfo>(
+      ['resolve', 'qlref'],
+      subcommandArgs,
+      'Resolving qlref',
+      false
+    );
+  }
+
   /**
    * Runs QL tests.
    * @param testPaths Full paths of the tests to run.
@@ -489,12 +520,12 @@ export class CodeQLCliServer implements Disposable {
     testPaths: string[], workspaces: string[], options: TestRunOptions
   ): AsyncGenerator<TestCompleted, void, unknown> {
 
-    const subcommandArgs = [
+    const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
       '--additional-packs', workspaces.join(path.delimiter),
       '--threads',
       this.cliConfig.numberTestThreads.toString(),
       ...testPaths
-    ];
+    ]);
 
     for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
       subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
@@ -523,7 +554,7 @@ export class CodeQLCliServer implements Disposable {
     if (queryMemoryMb !== undefined) {
       args.push('--ram', queryMemoryMb.toString());
     }
-    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', progressReporter);
+    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', true, progressReporter);
   }
   /**
    * Gets the headers (and optionally pagination info) of a bqrs.
@@ -562,32 +593,48 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
-      `-t=kind=${metadata.kind}`,
-      `-t=id=${metadata.id}`,
       '--output', interpretedResultsPath,
-      '--format', SARIF_FORMAT,
+      '--format', format,
+      // Forward all of the query metadata.
+      ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`)
+    ];
+    if (format == SARIF_FORMAT) {
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      '--no-group-results',
-    ];
+      args.push('--no-group-results');
+    }
     if (sourceInfo !== undefined) {
       args.push(
         '--source-archive', sourceInfo.sourceArchive,
         '--source-location-prefix', sourceInfo.sourceLocationPrefix
       );
     }
+
+    args.push(
+      '--threads',
+      this.cliConfig.numberThreads.toString(),
+    );
+
     args.push(resultsPath);
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
+  }
+
+  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
 
     let output: string;
     try {
       output = await fs.readFile(interpretedResultsPath, 'utf8');
-    } catch (err) {
-      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
+    } catch (e) {
+      const rawMessage = e.stderr || e.message;
+      const errorMessage = rawMessage.startsWith('Cannot create a string')
+        ? `SARIF too large. ${rawMessage}`
+        : rawMessage;
+      throw new Error(`Reading output of interpretation failed: ${errorMessage}`);
     }
     try {
       return JSON.parse(output) as sarif.Log;
@@ -596,6 +643,9 @@ export class CodeQLCliServer implements Disposable {
     }
   }
 
+  async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
+    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+  }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
     const sortDirectionStrings = sortDirections.map(direction => {
@@ -635,12 +685,19 @@ export class CodeQLCliServer implements Disposable {
    * Gets information necessary for upgrading a database.
    * @param dbScheme the path to the dbscheme of the database to be upgraded.
    * @param searchPath A list of directories to search for upgrade scripts.
+   * @param allowDowngradesIfPossible Whether we should try and include downgrades of we can.
+   * @param targetDbScheme The dbscheme to try to upgrade to.
    * @returns A list of database upgrade script directories
    */
-  resolveUpgrades(dbScheme: string, searchPath: string[]): Promise<UpgradesInfo> {
+  async resolveUpgrades(dbScheme: string, searchPath: string[], allowDowngradesIfPossible: boolean, targetDbScheme?: string): Promise<UpgradesInfo> {
     const args = ['--additional-packs', searchPath.join(path.delimiter), '--dbscheme', dbScheme];
-
-    return this.runJsonCodeQlCliCommand<UpgradesInfo>(
+    if (targetDbScheme) {
+      args.push('--target-dbscheme', targetDbScheme);
+      if (allowDowngradesIfPossible && await this.cliConstraints.supportsDowngrades()) {
+        args.push('--allow-downgrades');
+      }
+    }
+    return await this.runJsonCodeQlCliCommand<UpgradesInfo>(
       ['resolve', 'upgrades'],
       args,
       'Resolving database upgrade scripts',
@@ -689,7 +746,7 @@ export class CodeQLCliServer implements Disposable {
   }
 
   async generateDil(qloFile: string, outFile: string): Promise<void> {
-    const extraArgs = (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0
+    const extraArgs = await this.cliConstraints.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
       : ['-o', outFile, qloFile];
     await this.runCodeQlCliCommand(
@@ -699,7 +756,7 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  private async getVersion() {
+  public async getVersion() {
     if (!this._version) {
       this._version = await this.refreshVersion();
     }
@@ -755,7 +812,7 @@ export function spawnServer(
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Starting ${name}` });
   }
-  logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`);
+  void logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`);
   const child = child_process.spawn(base, args);
   if (!child || !child.pid) {
     throw new Error(`Failed to start ${name} using command ${base} ${argsString}.`);
@@ -771,7 +828,7 @@ export function spawnServer(
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Started ${name}` });
   }
-  logger.log(`${name} started on PID: ${child.pid}`);
+  void logger.log(`${name} started on PID: ${child.pid}`);
   return child;
 }
 
@@ -800,10 +857,10 @@ export async function runCodeQlCliCommand(
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
     }
-    logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
+    void logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
     const result = await promisify(child_process.execFile)(codeQlPath, args);
-    logger.log(result.stderr);
-    logger.log('CLI command succeeded.');
+    void logger.log(result.stderr);
+    void logger.log('CLI command succeeded.');
     return result.stdout;
   } catch (err) {
     throw new Error(`${description} failed: ${err.stderr || err}`);
@@ -839,6 +896,20 @@ class SplitBuffer {
     this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
   }
 
+  /**
+   * A version of startsWith that isn't overriden by a broken version of ms-python.
+   *
+   * The definition comes from
+   * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
+   * which is CC0/public domain
+   *
+   * See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
+   */
+  private static startsWith(s: string, searchString: string, position: number): boolean {
+    const pos = position > 0 ? position | 0 : 0;
+    return s.substring(pos, pos + searchString.length) === searchString;
+  }
+
   /**
    * Extract the next full line from the buffer, if one is available.
    * @returns The text of the next available full line (without the separator), or `undefined` if no
@@ -847,7 +918,7 @@ class SplitBuffer {
   public getNextLine(): string | undefined {
     while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
       for (const separator of this.separators) {
-        if (this.buffer.startsWith(separator, this.searchIndex)) {
+        if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
           const line = this.buffer.substr(0, this.searchIndex);
           this.buffer = this.buffer.substr(this.searchIndex + separator.length);
           this.searchIndex = 0;
@@ -904,7 +975,8 @@ const lineEndings = ['\r\n', '\r', '\n'];
  */
 async function logStream(stream: Readable, logger: Logger): Promise<void> {
   for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
-    logger.log(line);
+    // Await the result of log here in order to ensure the logs are written in the correct order.
+    await logger.log(line);
   }
 }
 
@@ -920,3 +992,60 @@ export function shouldDebugQueryServer() {
     && process.env.QUERY_SERVER_JAVA_DEBUG !== '0'
     && process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
 }
+
+export class CliVersionConstraint {
+
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  public static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
+
+  /**
+   * CLI version where languages are exposed during a `codeql resolve database` command.
+   */
+  public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
+
+  /**
+   * CLI version where `codeql resolve upgrades` supports
+   * the `--allow-downgrades` flag
+   */
+  public static CLI_VERSION_WITH_DOWNGRADES = new SemVer('2.4.4');
+
+  /**
+   * CLI version where the `codeql resolve qlref` command is available.
+   */
+  public static CLI_VERSION_WITH_RESOLVE_QLREF = new SemVer('2.5.1');
+
+  /**
+   * CLI version where database registration was introduced
+   */
+  public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
+
+  constructor(private readonly cli: CodeQLCliServer) {
+    /**/
+  }
+
+  private async isVersionAtLeast(v: SemVer) {
+    return (await this.cli.getVersion()).compare(v) >= 0;
+  }
+
+  public async supportsDecompileDil() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DECOMPILE_KIND_DIL);
+  }
+
+  public async supportsLanguageName() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
+  }
+
+  public async supportsDowngrades() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
+  }
+
+  public async supportsResolveQlref() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF);
+  }
+
+  async supportsDatabaseRegistration() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
+  }
+}

extensions/ql-vscode/src/commandRunner.ts

@@ -0,0 +1,241 @@
+import {
+  CancellationToken,
+  ProgressOptions,
+  window as Window,
+  commands,
+  Disposable,
+  ProgressLocation
+} from 'vscode';
+import { showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
+import { logger } from './logging';
+import { telemetryListener } from './telemetry';
+
+export class UserCancellationException extends Error {
+  /**
+   * @param message The error message
+   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
+   */
+  constructor(message?: string, public readonly silent = false) {
+    super(message);
+  }
+}
+
+export interface ProgressUpdate {
+  /**
+   * The current step
+   */
+  step: number;
+  /**
+   * The maximum step. This *should* be constant for a single job.
+   */
+  maxStep: number;
+  /**
+   * The current progress message
+   */
+  message: string;
+}
+
+export type ProgressCallback = (p: ProgressUpdate) => void;
+
+/**
+ * A task that handles command invocations from `commandRunner`
+ * and includes a progress monitor.
+ *
+ *
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `ProgressTask` instance.
+ *
+ * @param progress a progress handler function. Call this
+ * function with a `ProgressUpdate` instance in order to
+ * denote some progress being achieved on this task.
+ * @param token a cencellation token
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+export type ProgressTask<R> = (
+  progress: ProgressCallback,
+  token: CancellationToken,
+  ...args: any[]
+) => Thenable<R>;
+
+/**
+ * A task that handles command invocations from `commandRunner`.
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `NoProgressTask` instance.
+ *
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+type NoProgressTask = ((...args: any[]) => Promise<any>);
+
+/**
+ * This mediates between the kind of progress callbacks we want to
+ * write (where we *set* current progress position and give
+ * `maxSteps`) and the kind vscode progress api expects us to write
+ * (which increment progress by a certain amount out of 100%).
+ *
+ * Where possible, the `commandRunner` function below should be used
+ * instead of this function. The commandRunner is meant for wrapping
+ * top-level commands and provides error handling and other support
+ * automatically.
+ *
+ * Only use this function if you need a progress monitor and the
+ * control flow does not always come from a command (eg- during
+ * extension activation, or from an internal language server
+ * request).
+ */
+export function withProgress<R>(
+  options: ProgressOptions,
+  task: ProgressTask<R>,
+  ...args: any[]
+): Thenable<R> {
+  let progressAchieved = 0;
+  return Window.withProgress(options,
+    (progress, token) => {
+      return task(p => {
+        const { message, step, maxStep } = p;
+        const increment = 100 * (step - progressAchieved) / maxStep;
+        progressAchieved = step;
+        progress.report({ message, increment });
+      }, token, ...args);
+    });
+}
+
+/**
+ * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
+ *
+ * In this variant of the command runner, no progress monitor is used.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task.
+ */
+export function commandRunner(
+  commandId: string,
+  task: NoProgressTask,
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+
+    try {
+      return await task(...args);
+    } catch (e) {
+      error = e;
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          void logger.log(errorMessage);
+        } else {
+          void showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        void showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
+      }
+      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
+    }
+  });
+}
+
+/**
+ * A generic wrapper for command registration.  This wrapper adds uniform error handling,
+ * progress monitoring, and cancellation for commands.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task after the progress callback
+ * and cancellation token.
+ * @param progressOptions Progress options to be sent to the progress monitor.
+ */
+export function commandRunnerWithProgress<R>(
+  commandId: string,
+  task: ProgressTask<R>,
+  progressOptions: Partial<ProgressOptions>
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+    const progressOptionsWithDefaults = {
+      location: ProgressLocation.Notification,
+      ...progressOptions
+    };
+    try {
+      return await withProgress(progressOptionsWithDefaults, task, ...args);
+    } catch (e) {
+      error = e;
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          void logger.log(errorMessage);
+        } else {
+          void showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        void showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
+      }
+      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
+    }
+  });
+}
+
+/**
+ * Displays a progress monitor that indicates how much progess has been made
+ * reading from a stream.
+ *
+ * @param readable The stream to read progress from
+ * @param messagePrefix A prefix for displaying the message
+ * @param totalNumBytes Total number of bytes in this stream
+ * @param progress The progress callback used to set messages
+ */
+export function reportStreamProgress(
+  readable: NodeJS.ReadableStream,
+  messagePrefix: string,
+  totalNumBytes?: number,
+  progress?: ProgressCallback
+) {
+  if (progress && totalNumBytes) {
+    let numBytesDownloaded = 0;
+    const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+    const updateProgress = () => {
+      progress({
+        step: numBytesDownloaded,
+        maxStep: totalNumBytes,
+        message: `${messagePrefix} [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
+      });
+    };
+
+    // Display the progress straight away rather than waiting for the first chunk.
+    updateProgress();
+
+    readable.on('data', data => {
+      numBytesDownloaded += data.length;
+      updateProgress();
+    });
+  } else if (progress) {
+    progress({
+      step: 1,
+      maxStep: 2,
+      message: `${messagePrefix} (Size unknown)`,
+    });
+  }
+}

extensions/ql-vscode/src/compare/compare-interface.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from '../vscode-utils/disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import {
   WebviewPanel,
   ExtensionContext,
@@ -173,7 +173,7 @@ export class CompareInterfaceManager extends DisposableObject {
         break;
 
       case 'changeCompare':
-        this.changeTable(msg.newResultSetName);
+        await this.changeTable(msg.newResultSetName);
         break;
 
       case 'viewSourceFile':
@@ -267,11 +267,11 @@ export class CompareInterfaceManager extends DisposableObject {
     return resultsDiff(fromResults, toResults);
   }
 
-  private openQuery(kind: 'from' | 'to') {
+  private async openQuery(kind: 'from' | 'to') {
     const toOpen =
       kind === 'from' ? this.comparePair?.from : this.comparePair?.to;
     if (toOpen) {
-      this.showQueryResultsCallback(toOpen);
+      await this.showQueryResultsCallback(toOpen);
     }
   }
 }

extensions/ql-vscode/src/compare/view/Compare.tsx

@@ -21,7 +21,7 @@ const emptyComparison: SetComparisonsMessage = {
   message: 'Empty comparison'
 };
 
-export function Compare(_: {}): JSX.Element {
+export function Compare(_: Record<string, never>): JSX.Element {
   const [comparison, setComparison] = useState<SetComparisonsMessage>(
     emptyComparison
   );
@@ -38,7 +38,9 @@ export function Compare(_: {}): JSX.Element {
             setComparison(msg);
         }
       } else {
-        console.error(`Invalid event origin ${evt.origin}`);
+        // sanitize origin
+        const origin = evt.origin.replace(/\n|\r/g, '');
+        console.error(`Invalid event origin ${origin}`);
       }
     });
   });
@@ -64,8 +66,8 @@ export function Compare(_: {}): JSX.Element {
         {hasRows ? (
           <CompareTable comparison={comparison}></CompareTable>
         ) : (
-            <div className="vscode-codeql__compare-message">{message}</div>
-          )}
+          <div className="vscode-codeql__compare-message">{message}</div>
+        )}
       </>
     );
   } catch (err) {

extensions/ql-vscode/src/config.ts

@@ -1,10 +1,10 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
 
 /** Helper class to look up a labelled (and possibly nested) setting. */
-class Setting {
+export class Setting {
   name: string;
   parent?: Setting;
 
@@ -39,10 +39,19 @@ class Setting {
 
 const ROOT_SETTING = new Setting('codeQL');
 
-// Distribution configuration
+// Global configuration
+const TELEMETRY_SETTING = new Setting('telemetry', ROOT_SETTING);
+const AST_VIEWER_SETTING = new Setting('astViewer', ROOT_SETTING);
+const GLOBAL_TELEMETRY_SETTING = new Setting('telemetry');
 
+export const LOG_TELEMETRY = new Setting('logTelemetry', TELEMETRY_SETTING);
+export const ENABLE_TELEMETRY = new Setting('enableTelemetry', TELEMETRY_SETTING);
+
+export const GLOBAL_ENABLE_TELEMETRY = new Setting('enableTelemetry', GLOBAL_TELEMETRY_SETTING);
+
+// Distribution configuration
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
-const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
+export const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
 const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
@@ -52,7 +61,8 @@ const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
 
 export interface DistributionConfig {
-  customCodeQlPath?: string;
+  readonly customCodeQlPath?: string;
+  updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
   ownerName?: string;
@@ -64,26 +74,33 @@ export interface DistributionConfig {
 
 const RUNNING_QUERIES_SETTING = new Setting('runningQueries', ROOT_SETTING);
 const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES_SETTING);
+const SAVE_CACHE_SETTING = new Setting('saveCache', RUNNING_QUERIES_SETTING);
+const CACHE_SIZE_SETTING = new Setting('cacheSize', RUNNING_QUERIES_SETTING);
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
 const RUNNING_TESTS_SETTING = new Setting('runningTests', ROOT_SETTING);
 const RESULTS_DISPLAY_SETTING = new Setting('resultsDisplay', ROOT_SETTING);
 
+export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting('additionalTestArguments', RUNNING_TESTS_SETTING);
 export const NUMBER_OF_TEST_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_TESTS_SETTING);
 export const MAX_QUERIES = new Setting('maxQueries', RUNNING_QUERIES_SETTING);
 export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
 export const PAGE_SIZE = new Setting('pageSize', RESULTS_DISPLAY_SETTING);
+const CUSTOM_LOG_DIRECTORY_SETTING = new Setting('customLogDirectory', RUNNING_QUERIES_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
-const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
+const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING, DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING];
 
 export interface QueryServerConfig {
   codeQlPath: string;
   debug: boolean;
   numThreads: number;
+  saveCache: boolean;
+  cacheSize: number;
   queryMemoryMb?: number;
   timeoutSecs: number;
+  customLogDirectory?: string;
   onDidChangeConfiguration?: Event<void>;
 }
 
@@ -95,15 +112,17 @@ export interface QueryHistoryConfig {
   onDidChangeConfiguration: Event<void>;
 }
 
-const CLI_SETTINGS = [NUMBER_OF_TEST_THREADS_SETTING];
+const CLI_SETTINGS = [ADDITIONAL_TEST_ARGUMENTS_SETTING, NUMBER_OF_TEST_THREADS_SETTING, NUMBER_OF_THREADS_SETTING];
 
 export interface CliConfig {
+  additionalTestArguments: string[];
   numberTestThreads: number;
+  numberThreads: number;
   onDidChangeConfiguration?: Event<void>;
 }
 
 
-abstract class ConfigListener extends DisposableObject {
+export abstract class ConfigListener extends DisposableObject {
   protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());
 
   constructor() {
@@ -128,7 +147,7 @@ abstract class ConfigListener extends DisposableObject {
 
   protected abstract handleDidChangeConfiguration(e: ConfigurationChangeEvent): void;
   private updateConfiguration(): void {
-    this._onDidChangeConfiguration.fire();
+    this._onDidChangeConfiguration.fire(undefined);
   }
 
   public get onDidChangeConfiguration(): Event<void> {
@@ -149,6 +168,10 @@ export class DistributionConfigListener extends ConfigListener implements Distri
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
+  public async updateCustomCodeQlPath(newPath: string | undefined) {
+    await CUSTOM_CODEQL_PATH_SETTING.updateValue(newPath, ConfigurationTarget.Global);
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(DISTRIBUTION_CHANGE_SETTINGS, e);
   }
@@ -166,7 +189,7 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
       config.push(distributionManager.onDidChangeDistribution(async () => {
         const codeQlPath = await distributionManager.getCodeQlPathWithoutVersionCheck();
         config._codeQlPath = codeQlPath!;
-        config._onDidChangeConfiguration.fire();
+        config._onDidChangeConfiguration.fire(undefined);
       }));
     }
     return config;
@@ -176,10 +199,22 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     return this._codeQlPath;
   }
 
+  public get customLogDirectory(): string | undefined {
+    return CUSTOM_LOG_DIRECTORY_SETTING.getValue<string>() || undefined;
+  }
+
   public get numThreads(): number {
     return NUMBER_OF_THREADS_SETTING.getValue<number>();
   }
 
+  public get saveCache(): boolean {
+    return SAVE_CACHE_SETTING.getValue<boolean>();
+  }
+
+  public get cacheSize(): number {
+    return CACHE_SIZE_SETTING.getValue<number | null>() || 0;
+  }
+
   /** Gets the configured query timeout, in seconds. This looks up the setting at the time of access. */
   public get timeoutSecs(): number {
     return TIMEOUT_SETTING.getValue<number | null>() || 0;
@@ -191,7 +226,7 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
       return undefined;
     }
     if (memory == 0 || typeof (memory) !== 'number') {
-      logger.log(`Ignoring value '${memory}' for setting ${MEMORY_SETTING.qualifiedName}`);
+      void logger.log(`Ignoring value '${memory}' for setting ${MEMORY_SETTING.qualifiedName}`);
       return undefined;
     }
     return memory;
@@ -217,11 +252,18 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
 }
 
 export class CliConfigListener extends ConfigListener implements CliConfig {
+  public get additionalTestArguments(): string[] {
+    return ADDITIONAL_TEST_ARGUMENTS_SETTING.getValue();
+  }
 
   public get numberTestThreads(): number {
     return NUMBER_OF_TEST_THREADS_SETTING.getValue();
   }
 
+  public get numberThreads(): number {
+    return NUMBER_OF_THREADS_SETTING.getValue<number>();
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
   }
@@ -235,3 +277,17 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
  * want to enable experimental features, they can add them directly in
  * their vscode settings json file.
  */
+
+/**
+ * Enables canary features of this extension. Recommended for all internal users.
+ */
+export const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);
+
+export function isCanary() {
+  return !!CANARY_FEATURES.getValue<boolean>();
+}
+
+/**
+ * Avoids caching in the AST viewer if the user is also a canary user.
+ */
+export const NO_CACHE_AST_VIEWER = new Setting('disableCache', AST_VIEWER_SETTING);

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -8,7 +8,7 @@ import fileRangeFromURI from './fileRangeFromURI';
 import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { QueryWithResults, compileAndRunQueryAgainstDatabase } from '../run-queries';
-import { ProgressCallback } from '../helpers';
+import { ProgressCallback } from '../commandRunner';
 import { KeyType } from './keyType';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
 
@@ -52,9 +52,6 @@ export async function getLocationsForUriString(
   }
 
   const qlpack = await qlpackOfDatabase(cli, db);
-  if (qlpack === undefined) {
-    throw new Error('Can\'t infer qlpack from database source archive');
-  }
   const templates = createTemplates(uri.pathWithinSourceArchive);
 
   const links: FullLocationLink[] = [];

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -12,12 +12,13 @@ import {
 import { CodeQLCliServer } from '../cli';
 import { DatabaseItem } from '../databases';
 
-export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string | undefined> {
-  if (db.contents === undefined)
-    return undefined;
+export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string> {
+  if (db.contents === undefined) {
+    throw new Error('Database is invalid and cannot infer QLPack.');
+  }
   const datasetPath = db.contents.datasetUri.fsPath;
-  const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath);
-  return qlpack;
+  const dbscheme = await helpers.getPrimaryDbscheme(datasetPath);
+  return await helpers.getQlPackForDbscheme(cli, dbscheme);
 }
 
 
@@ -36,7 +37,7 @@ export async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyTy
 
   const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
   if (queries.length === 0) {
-    helpers.showAndLogErrorMessage(
+    void helpers.showAndLogErrorMessage(
       `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
       Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
       for this language.`

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -1,9 +1,21 @@
-import * as vscode from 'vscode';
+import {
+  CancellationToken,
+  DefinitionProvider,
+  Location,
+  LocationLink,
+  Position,
+  ProgressLocation,
+  ReferenceContext,
+  ReferenceProvider,
+  TextDocument,
+  Uri
+} from 'vscode';
 
 import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
-import { CachedOperation, ProgressCallback, withProgress } from '../helpers';
+import { CachedOperation } from '../helpers';
+import { ProgressCallback, withProgress } from '../commandRunner';
 import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';
@@ -13,6 +25,7 @@ import {
 } from './keyType';
 import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
 
 /**
  * Run templated CodeQL queries to find definitions and references in
@@ -21,20 +34,20 @@ import { qlpackOfDatabase, resolveQueries } from './queryResolver';
  * or from a selected identifier.
  */
 
-export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
-  private cache: CachedOperation<vscode.LocationLink[]>;
+export class TemplateQueryDefinitionProvider implements DefinitionProvider {
+  private cache: CachedOperation<LocationLink[]>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
   ) {
-    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+    this.cache = new CachedOperation<LocationLink[]>(this.getDefinitions.bind(this));
   }
 
-  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+  async provideDefinition(document: TextDocument, position: Position, _token: CancellationToken): Promise<LocationLink[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.LocationLink[] = [];
+    const locLinks: LocationLink[] = [];
     for (const link of fileLinks) {
       if (link.originSelectionRange!.contains(position)) {
         locLinks.push(link);
@@ -43,9 +56,9 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
     return locLinks;
   }
 
-  private async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+  private async getDefinitions(uriString: string): Promise<LocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding definitions'
     }, async (progress, token) => {
@@ -63,7 +76,7 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
   }
 }
 
-export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+export class TemplateQueryReferenceProvider implements ReferenceProvider {
   private cache: CachedOperation<FullLocationLink[]>;
 
   constructor(
@@ -75,13 +88,13 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
   }
 
   async provideReferences(
-    document: vscode.TextDocument,
-    position: vscode.Position,
-    _context: vscode.ReferenceContext,
-    _token: vscode.CancellationToken
-  ): Promise<vscode.Location[]> {
+    document: TextDocument,
+    position: Position,
+    _context: ReferenceContext,
+    _token: CancellationToken
+  ): Promise<Location[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.Location[] = [];
+    const locLinks: Location[] = [];
     for (const link of fileLinks) {
       if (link.targetRange!.contains(position)) {
         locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
@@ -92,7 +105,7 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 
   private async getReferences(uriString: string): Promise<FullLocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding references'
     }, async (progress, token) => {
@@ -111,40 +124,47 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 }
 
 export class TemplatePrintAstProvider {
-  private cache: CachedOperation<QueryWithResults | undefined>;
+  private cache: CachedOperation<QueryWithResults>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
-
-    // Note: progress and token are only used if a cached value is not available
-    private progress: ProgressCallback,
-    private token: vscode.CancellationToken
   ) {
-    this.cache = new CachedOperation<QueryWithResults | undefined>(this.getAst.bind(this));
+    this.cache = new CachedOperation<QueryWithResults>(this.getAst.bind(this));
   }
 
-  async provideAst(document?: vscode.TextDocument): Promise<AstBuilder | undefined> {
+  async provideAst(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    document?: TextDocument
+  ): Promise<AstBuilder | undefined> {
     if (!document) {
-      return;
-    }
-    const queryResults = await this.cache.get(document.uri.toString());
-    if (!queryResults) {
-      return;
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
+    const queryResults = this.shouldCache()
+      ? await this.cache.get(document.uri.toString(), progress, token)
+      : await this.getAst(document.uri.toString(), progress, token);
 
     return new AstBuilder(
       queryResults, this.cli,
-      this.dbm.findDatabaseItem(vscode.Uri.parse(queryResults.database.databaseUri!, true))!,
+      this.dbm.findDatabaseItem(Uri.parse(queryResults.database.databaseUri!, true))!,
       document.fileName
     );
   }
 
-  private async getAst(uriString: string): Promise<QueryWithResults> {
-    const uri = vscode.Uri.parse(uriString, true);
+  private shouldCache() {
+    return !(isCanary() && NO_CACHE_AST_VIEWER.getValue<boolean>());
+  }
+
+  private async getAst(
+    uriString: string,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<QueryWithResults> {
+    const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
-      throw new Error('AST Viewing is only available for databases with zipped source archives.');
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
 
     const zippedArchive = decodeSourceArchiveUri(uri);
@@ -156,9 +176,6 @@ export class TemplatePrintAstProvider {
     }
 
     const qlpack = await qlpackOfDatabase(this.cli, db);
-    if (!qlpack) {
-      throw new Error('Can\'t infer qlpack from database source archive');
-    }
     const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintAstQuery);
     if (queries.length > 1) {
       throw new Error('Found multiple Print AST queries. Can\'t continue');
@@ -183,9 +200,9 @@ export class TemplatePrintAstProvider {
       this.qs,
       db,
       false,
-      vscode.Uri.file(query),
-      this.progress,
-      this.token,
+      Uri.file(query),
+      progress,
+      token,
       templates
     );
   }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -12,22 +12,26 @@ import * as path from 'path';
 
 import { DatabaseManager, DatabaseItem } from './databases';
 import {
-  ProgressCallback,
   showAndLogInformationMessage,
 } from './helpers';
+import {
+  reportStreamProgress,
+  ProgressCallback,
+} from './commandRunner';
 import { logger } from './logging';
+import { tmpDir } from './run-queries';
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
  *
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportInternetDatabase(
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  _: CancellationToken,
+  token: CancellationToken,
 ): Promise<DatabaseItem | undefined> {
   const databaseUrl = await window.showInputBox({
     prompt: 'Enter URL of zipfile of database to download',
@@ -40,14 +44,15 @@ export async function promptImportInternetDatabase(
 
   const item = await databaseArchiveFetcher(
     databaseUrl,
-    databasesManager,
+    databaseManager,
     storagePath,
-    progress
+    progress,
+    token
   );
 
   if (item) {
-    commands.executeCommand('codeQLDatabases.focus');
-    showAndLogInformationMessage('Database downloaded and imported successfully.');
+    await commands.executeCommand('codeQLDatabases.focus');
+    void showAndLogInformationMessage('Database downloaded and imported successfully.');
   }
   return item;
 
@@ -58,18 +63,18 @@ export async function promptImportInternetDatabase(
  * User enters a project url and then the user is asked which language
  * to download (if there is more than one)
  *
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportLgtmDatabase(
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  _: CancellationToken
+  token: CancellationToken
 ): Promise<DatabaseItem | undefined> {
   const lgtmUrl = await window.showInputBox({
     prompt:
-      'Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)',
+      'Enter the project slug or URL on LGTM (e.g., g/github/codeql or https://lgtm.com/projects/g/github/codeql)',
   });
   if (!lgtmUrl) {
     return;
@@ -80,13 +85,14 @@ export async function promptImportLgtmDatabase(
     if (databaseUrl) {
       const item = await databaseArchiveFetcher(
         databaseUrl,
-        databasesManager,
+        databaseManager,
         storagePath,
-        progress
+        progress,
+        token
       );
       if (item) {
-        commands.executeCommand('codeQLDatabases.focus');
-        showAndLogInformationMessage('Database downloaded and imported successfully.');
+        await commands.executeCommand('codeQLDatabases.focus');
+        void showAndLogInformationMessage('Database downloaded and imported successfully.');
       }
       return item;
     }
@@ -100,26 +106,27 @@ export async function promptImportLgtmDatabase(
  * Imports a database from a local archive.
  *
  * @param databaseUrl the file url of the archive to import
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function importArchiveDatabase(
   databaseUrl: string,
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  _: CancellationToken,
+  token: CancellationToken,
 ): Promise<DatabaseItem | undefined> {
   try {
     const item = await databaseArchiveFetcher(
       databaseUrl,
-      databasesManager,
+      databaseManager,
       storagePath,
-      progress
+      progress,
+      token
     );
     if (item) {
-      commands.executeCommand('codeQLDatabases.focus');
-      showAndLogInformationMessage('Database unzipped and imported successfully.');
+      await commands.executeCommand('codeQLDatabases.focus');
+      void showAndLogInformationMessage('Database unzipped and imported successfully.');
     }
     return item;
   } catch (e) {
@@ -137,17 +144,19 @@ export async function importArchiveDatabase(
  * or in the local filesystem.
  *
  * @param databaseUrl URL from which to grab the database
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
- * @param progressCallback optional callback to send progress messages to
+ * @param progress callback to send progress messages to
+ * @param token cancellation token
  */
 async function databaseArchiveFetcher(
   databaseUrl: string,
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
-  progressCallback?: ProgressCallback
+  progress: ProgressCallback,
+  token: CancellationToken
 ): Promise<DatabaseItem> {
-  progressCallback?.({
+  progress({
     message: 'Getting database',
     step: 1,
     maxStep: 4,
@@ -159,12 +168,12 @@ async function databaseArchiveFetcher(
   const unzipPath = await getStorageFolder(storagePath, databaseUrl);
 
   if (isFile(databaseUrl)) {
-    await readAndUnzip(databaseUrl, unzipPath);
+    await readAndUnzip(databaseUrl, unzipPath, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, progressCallback);
+    await fetchAndUnzip(databaseUrl, unzipPath, progress);
   }
 
-  progressCallback?.({
+  progress({
     message: 'Opening database',
     step: 3,
     maxStep: 4,
@@ -177,15 +186,15 @@ async function databaseArchiveFetcher(
     'codeql-database.yml'
   );
   if (dbPath) {
-    progressCallback?.({
+    progress({
       message: 'Validating and fixing source location',
       step: 4,
       maxStep: 4,
     });
     await ensureZippedSourceLocation(dbPath);
 
-    const item = await databasesManager.openDatabase(Uri.file(dbPath));
-    databasesManager.setCurrentDatabaseItem(item);
+    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath));
+    await databaseManager.setCurrentDatabaseItem(item);
     return item;
   } else {
     throw new Error('Database not found in archive.');
@@ -232,48 +241,67 @@ function validateHttpsUrl(databaseUrl: string) {
   }
 }
 
-async function readAndUnzip(databaseUrl: string, unzipPath: string) {
-  const databaseFile = Uri.parse(databaseUrl).fsPath;
-  const directory = await unzipper.Open.file(databaseFile);
+async function readAndUnzip(
+  zipUrl: string,
+  unzipPath: string,
+  progress?: ProgressCallback
+) {
+  // TODO: Providing progress as the file is unzipped is currently blocked
+  // on https://github.com/ZJONSSON/node-unzipper/issues/222
+  const zipFile = Uri.parse(zipUrl).fsPath;
+  progress?.({
+    maxStep: 10,
+    step: 9,
+    message: `Unzipping into ${path.basename(unzipPath)}`
+  });
+  // Must get the zip central directory since streaming the
+  // zip contents may not have correct local file headers.
+  // Instead, we can only rely on the central directory.
+  const directory = await unzipper.Open.file(zipFile);
   await directory.extract({ path: unzipPath });
 }
 
 async function fetchAndUnzip(
   databaseUrl: string,
   unzipPath: string,
-  progressCallback?: ProgressCallback
+  progress?: ProgressCallback
 ) {
-  const response = await fetch(databaseUrl);
+  // Although it is possible to download and stream directly to an unzipped directory,
+  // we need to avoid this for two reasons. The central directory is located at the
+  // end of the zip file. It is the source of truth of the content locations. Individual
+  // file headers may be incorrect. Additionally, saving to file first will reduce memory
+  // pressure compared with unzipping while downloading the archive.
 
-  await checkForFailingResponse(response);
+  const archivePath = path.join(tmpDir.name, `archive-${Date.now()}.zip`);
 
-  const unzipStream = unzipper.Extract({
-    path: unzipPath,
-  });
-
-  progressCallback?.({
+  progress?.({
     maxStep: 3,
-    message: 'Unzipping database',
-    step: 2,
-  });
-  await new Promise((resolve, reject) => {
-    const handler = (err: Error) => {
-      if (err.message.startsWith('invalid signature')) {
-        reject(new Error('Not a valid archive.'));
-      } else {
-        reject(err);
-      }
-    };
-    response.body.on('error', handler);
-    unzipStream.on('error', handler);
-    unzipStream.on('close', resolve);
-    response.body.pipe(unzipStream);
+    message: 'Downloading database',
+    step: 1,
   });
+
+  const response = await checkForFailingResponse(await fetch(databaseUrl));
+  const archiveFileStream = fs.createWriteStream(archivePath);
+
+  const contentLength = response.headers.get('content-length');
+  const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
+  reportStreamProgress(response.body, 'Downloading database', totalNumBytes, progress);
+
+  await new Promise((resolve, reject) =>
+    response.body.pipe(archiveFileStream)
+      .on('finish', resolve)
+      .on('error', reject)
+  );
+
+  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, progress);
+
+  // remove archivePath eagerly since these archives can be large.
+  await fs.remove(archivePath);
 }
 
-async function checkForFailingResponse(response: Response): Promise<void | never> {
+async function checkForFailingResponse(response: Response): Promise<Response | never> {
   if (response.ok) {
-    return;
+    return response;
   }
 
   // An error downloading the database. Attempt to extract the resaon behind it.
@@ -324,13 +352,14 @@ export async function findDirWithFile(
 
 /**
  * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
- * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * There are several possibilities for the provider: in addition to GitHub.com (g),
  * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
  *
- * After the {provider}/{org}/{name} path components, there may be the components
- * related to sub pages.
+ * This function accepts any url that matches the pattern above. It also accepts the
+ * raw project slug, e.g., `g/myorg/myproject`
  *
- * This function accepts any url that matches the patter above
+ * After the `{provider}/{org}/{name}` path components, there may be the components
+ * related to sub pages.
  *
  * @param lgtmUrl The URL to the lgtm project
  *
@@ -342,6 +371,10 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
     return false;
   }
 
+  if (convertRawLgtmSlug(lgtmUrl)) {
+    return true;
+  }
+
   try {
     const uri = Uri.parse(lgtmUrl, true);
     if (uri.scheme !== 'https') {
@@ -359,9 +392,23 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
   }
 }
 
+function convertRawLgtmSlug(maybeSlug: string): string | undefined {
+  if (!maybeSlug) {
+    return;
+  }
+  const segments = maybeSlug.split('/');
+  const providers = ['g', 'gl', 'b', 'git'];
+  if (segments.length === 3 && providers.includes(segments[0])) {
+    return `https://lgtm.com/projects/${maybeSlug}`;
+  }
+  return;
+}
+
 // exported for testing
 export async function convertToDatabaseUrl(lgtmUrl: string) {
   try {
+    lgtmUrl = convertRawLgtmSlug(lgtmUrl) || lgtmUrl;
+
     const uri = Uri.parse(lgtmUrl, true);
     const paths = ['api', 'v1.0'].concat(
       uri.path.split('/').filter((segment) => segment)
@@ -386,7 +433,7 @@ export async function convertToDatabaseUrl(lgtmUrl: string) {
       language,
     ].join('/')}`;
   } catch (e) {
-    logger.log(`Error: ${e.message}`);
+    void logger.log(`Error: ${e.message}`);
     throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
   }
 }

extensions/ql-vscode/src/databases-ui.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import {
   Event,
   EventEmitter,
@@ -12,26 +12,25 @@ import {
 } from 'vscode';
 import * as fs from 'fs-extra';
 
-import * as cli from './cli';
 import {
   DatabaseChangedEvent,
   DatabaseItem,
   DatabaseManager,
-  getUpgradesDirectories,
-  isLikelyDatabaseRoot,
-  isLikelyDbLanguageFolder,
 } from './databases';
 import {
   commandRunner,
   commandRunnerWithProgress,
-  getOnDiskWorkspaceFolders,
   ProgressCallback,
+} from './commandRunner';
+import {
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder,
   showAndLogErrorMessage
 } from './helpers';
 import { logger } from './logging';
 import { clearCacheInDatabase } from './run-queries';
 import * as qsClient from './queryserver-client';
-import { upgradeDatabase } from './upgrades';
+import { upgradeDatabaseExplicit } from './upgrades';
 import {
   importArchiveDatabase,
   promptImportInternetDatabase,
@@ -81,7 +80,7 @@ class DatabaseTreeDataProvider extends DisposableObject
   implements TreeDataProvider<DatabaseItem> {
   private _sortOrder = SortOrder.NameAsc;
 
-  private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
+  private readonly _onDidChangeTreeData = this.push(new EventEmitter<DatabaseItem | undefined>());
   private currentDatabaseItem: DatabaseItem | undefined;
 
   constructor(
@@ -109,7 +108,7 @@ class DatabaseTreeDataProvider extends DisposableObject
   }
 
   private handleDidChangeDatabaseItem = (event: DatabaseChangedEvent): void => {
-    // Note that events from the databse manager are instances of DatabaseChangedEvent
+    // Note that events from the database manager are instances of DatabaseChangedEvent
     // and events fired by the UI are instances of DatabaseItem
 
     // When event.item is undefined, then the entire tree is refreshed.
@@ -143,6 +142,7 @@ class DatabaseTreeDataProvider extends DisposableObject
       );
     }
     item.tooltip = element.databaseUri.fsPath;
+    item.description = element.language;
     return item;
   }
 
@@ -179,7 +179,7 @@ class DatabaseTreeDataProvider extends DisposableObject
 
   public set sortOrder(newSortOrder: SortOrder) {
     this._sortOrder = newSortOrder;
-    this._onDidChangeTreeData.fire();
+    this._onDidChangeTreeData.fire(undefined);
   }
 }
 
@@ -215,7 +215,6 @@ export class DatabaseUI extends DisposableObject {
   private treeDataProvider: DatabaseTreeDataProvider;
 
   public constructor(
-    private cliserver: cli.CodeQLCliServer,
     private databaseManager: DatabaseManager,
     private readonly queryServer: qsClient.QueryServerClient | undefined,
     private readonly storagePath: string,
@@ -235,7 +234,7 @@ export class DatabaseUI extends DisposableObject {
   }
 
   init() {
-    logger.log('Registering database panel commands.');
+    void logger.log('Registering database panel commands.');
     this.push(
       commandRunnerWithProgress(
         'codeQL.setCurrentDatabase',
@@ -296,7 +295,7 @@ export class DatabaseUI extends DisposableObject {
         'codeQLDatabases.chooseDatabaseLgtm',
         this.handleChooseDatabaseLgtm,
         {
-          title: 'Adding database from LGTM',
+          title: 'Adding database from LGTM. Choose a language from the dropdown, if requested.',
         })
     );
     this.push(
@@ -318,9 +317,13 @@ export class DatabaseUI extends DisposableObject {
       )
     );
     this.push(
-      commandRunner(
+      commandRunnerWithProgress(
         'codeQLDatabases.removeDatabase',
-        this.handleRemoveDatabase
+        this.handleRemoveDatabase,
+        {
+          title: 'Removing database',
+          cancellable: false
+        }
       )
     );
     this.push(
@@ -366,14 +369,24 @@ export class DatabaseUI extends DisposableObject {
     try {
       return await this.chooseAndSetDatabase(true, progress, token);
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
       return undefined;
     }
   };
 
   handleRemoveOrphanedDatabases = async (): Promise<void> => {
-    logger.log('Removing orphaned databases from workspace storage.');
-    let dbDirs =
+    void logger.log('Removing orphaned databases from workspace storage.');
+    let dbDirs = undefined;
+
+    if (
+      !(await fs.pathExists(this.storagePath)) ||
+      !(await fs.stat(this.storagePath)).isDirectory()
+    ) {
+      void logger.log('Missing or invalid storage directory. Not trying to remove orphaned databases.');
+      return;
+    }
+
+    dbDirs =
       // read directory
       (await fs.readdir(this.storagePath, { withFileTypes: true }))
         // remove non-directories
@@ -390,7 +403,7 @@ export class DatabaseUI extends DisposableObject {
     dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
 
     if (!dbDirs.length) {
-      logger.log('No orphaned databases found.');
+      void logger.log('No orphaned databases found.');
       return;
     }
 
@@ -399,8 +412,8 @@ export class DatabaseUI extends DisposableObject {
     await Promise.all(
       dbDirs.map(async dbDir => {
         try {
-          logger.log(`Deleting orphaned database '${dbDir}'.`);
-          await fs.rmdir(dbDir, { recursive: true } as any);  // typings doesn't recognize the options argument
+          void logger.log(`Deleting orphaned database '${dbDir}'.`);
+          await fs.remove(dbDir);
         } catch (e) {
           failures.push(`${path.basename(dbDir)}`);
         }
@@ -409,10 +422,9 @@ export class DatabaseUI extends DisposableObject {
 
     if (failures.length) {
       const dirname = path.dirname(failures[0]);
-      showAndLogErrorMessage(
-        `Failed to delete unused databases:\n  ${
-        failures.join('\n  ')
-        }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.`
+      void showAndLogErrorMessage(
+        `Failed to delete unused databases (${failures.join(', ')
+        }).\nTo delete unused databases, please remove them manually from the storage folder ${dirname}.`
       );
     }
   };
@@ -425,7 +437,7 @@ export class DatabaseUI extends DisposableObject {
     try {
       return await this.chooseAndSetDatabase(false, progress, token);
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
       return undefined;
     }
   };
@@ -523,25 +535,10 @@ export class DatabaseUI extends DisposableObject {
     }
 
     // Search for upgrade scripts in any workspace folders available
-    const searchPath: string[] = getOnDiskWorkspaceFolders();
 
-    const upgradeInfo = await this.cliserver.resolveUpgrades(
-      databaseItem.contents.dbSchemeUri.fsPath,
-      searchPath
-    );
-
-    const { scripts, finalDbscheme } = upgradeInfo;
-
-    if (finalDbscheme === undefined) {
-      throw new Error('Could not determine target dbscheme to upgrade to.');
-    }
-    const targetDbSchemeUri = Uri.file(finalDbscheme);
-
-    await upgradeDatabase(
+    await upgradeDatabaseExplicit(
       this.queryServer,
       databaseItem,
-      targetDbSchemeUri,
-      getUpgradesDirectories(scripts),
       progress,
       token
     );
@@ -580,28 +577,29 @@ export class DatabaseUI extends DisposableObject {
           token
         );
       } else {
-        await this.setCurrentDatabase(uri);
+        await this.setCurrentDatabase(progress, token, uri);
       }
     } catch (e) {
       // rethrow and let this be handled by default error handling.
       throw new Error(
-        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${
-        e.message
+        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${e.message
         }`
       );
     }
   };
 
   private handleRemoveDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
     databaseItem: DatabaseItem,
     multiSelect: DatabaseItem[] | undefined
   ): Promise<void> => {
     if (multiSelect?.length) {
-      multiSelect.forEach((dbItem) =>
-        this.databaseManager.removeDatabaseItem(dbItem)
-      );
+      await Promise.all(multiSelect.map((dbItem) =>
+        this.databaseManager.removeDatabaseItem(progress, token, dbItem)
+      ));
     } else {
-      this.databaseManager.removeDatabaseItem(databaseItem);
+      await this.databaseManager.removeDatabaseItem(progress, token, databaseItem);
     }
   };
 
@@ -617,7 +615,7 @@ export class DatabaseUI extends DisposableObject {
     });
 
     if (newName) {
-      this.databaseManager.renameDatabaseItem(databaseItem, newName);
+      await this.databaseManager.renameDatabaseItem(databaseItem, newName);
     }
   };
 
@@ -651,11 +649,13 @@ export class DatabaseUI extends DisposableObject {
   }
 
   private async setCurrentDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken,
     uri: Uri
   ): Promise<DatabaseItem | undefined> {
     let databaseItem = this.databaseManager.findDatabaseItem(uri);
     if (databaseItem === undefined) {
-      databaseItem = await this.databaseManager.openDatabase(uri);
+      databaseItem = await this.databaseManager.openDatabase(progress, token, uri);
     }
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
 
@@ -680,7 +680,7 @@ export class DatabaseUI extends DisposableObject {
     if (byFolder) {
       const fixedUri = await this.fixDbUri(uri);
       // we are selecting a database folder
-      return await this.setCurrentDatabase(fixedUri);
+      return await this.setCurrentDatabase(progress, token, fixedUri);
     } else {
       // we are selecting a database archive. Must unzip into a workspace-controlled area
       // before importing.
@@ -701,7 +701,7 @@ export class DatabaseUI extends DisposableObject {
    * 2. If the selected URI is a directory matching db-*, choose the containing directory
    * 3. choose the current directory
    *
-   * @param uri a URI that is a datbase folder or inside it
+   * @param uri a URI that is a database folder or inside it
    *
    * @return the actual database folder found by using the heuristics above.
    */

extensions/ql-vscode/src/databases.ts

@@ -4,11 +4,21 @@ import * as path from 'path';
 import * as vscode from 'vscode';
 import * as cli from './cli';
 import { ExtensionContext } from 'vscode';
-import { showAndLogErrorMessage, showAndLogWarningMessage, showAndLogInformationMessage } from './helpers';
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  showAndLogInformationMessage,
+  isLikelyDatabaseRoot
+} from './helpers';
+import {
+  ProgressCallback,
+  withProgress
+} from './commandRunner';
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { QueryServerConfig } from './config';
+import { DisposableObject } from './pure/disposable-object';
 import { Logger, logger } from './logging';
+import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
+import { QueryServerClient } from './queryserver-client';
 
 /**
  * databases.ts
@@ -36,11 +46,13 @@ export interface DatabaseOptions {
   displayName?: string;
   ignoreSourceArchive?: boolean;
   dateAdded?: number | undefined;
+  language?: string;
 }
 
-interface FullDatabaseOptions extends DatabaseOptions {
+export interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
   dateAdded: number | undefined;
+  language: string | undefined;
 }
 
 interface PersistedDatabaseItem {
@@ -103,7 +115,7 @@ async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
 
   const dbAbsolutePath = path.join(parentDirectory, dbRelativePaths[0]);
   if (dbRelativePaths.length > 1) {
-    showAndLogWarningMessage(`Found multiple dataset directories in database, using '${dbAbsolutePath}'.`);
+    void showAndLogWarningMessage(`Found multiple dataset directories in database, using '${dbAbsolutePath}'.`);
   }
 
   return vscode.Uri.file(dbAbsolutePath);
@@ -126,7 +138,7 @@ async function findSourceArchive(
     }
   }
   if (!silent) {
-    showAndLogInformationMessage(
+    void showAndLogInformationMessage(
       `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`
     );
   }
@@ -193,6 +205,9 @@ export interface DatabaseItem {
   readonly databaseUri: vscode.Uri;
   /** The name of the database to be displayed in the UI */
   name: string;
+
+  /** The primary language of the database or empty string if unknown */
+  readonly language: string;
   /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
   readonly sourceArchive: vscode.Uri | undefined;
   /**
@@ -249,6 +264,16 @@ export interface DatabaseItem {
    * Holds if `uri` belongs to this database's source archive.
    */
   belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
+
+  /**
+   * Whether the database may be affected by test execution for the given path.
+   */
+  isAffectedByTest(testPath: string): Promise<boolean>;
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  getPersistedState(): PersistedDatabaseItem;
 }
 
 export enum DatabaseEventKind {
@@ -427,6 +452,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return dbInfo.datasetFolder;
   }
 
+  public get language() {
+    return this.options.language || '';
+  }
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive.
    */
@@ -446,6 +475,27 @@ export class DatabaseItemImpl implements DatabaseItem {
     return uri.scheme === zipArchiveScheme &&
       decodeSourceArchiveUri(uri).sourceArchiveZipPath === this.sourceArchive.fsPath;
   }
+
+  public async isAffectedByTest(testPath: string): Promise<boolean> {
+    const databasePath = this.databaseUri.fsPath;
+    if (!databasePath.endsWith('.testproj')) {
+      return false;
+    }
+    try {
+      const stats = await fs.stat(testPath);
+      if (stats.isDirectory()) {
+        return !path.relative(testPath, databasePath).startsWith('..');
+      } else {
+        // database for /one/two/three/test.ql is at /one/two/three/three.testproj
+        const testdir = path.dirname(testPath);
+        const testdirbase = path.basename(testdir);
+        return databasePath == path.join(testdir, testdirbase + '.testproj');
+      }
+    } catch {
+      // No information available for test path - assume database is unaffected.
+      return false;
+    }
+  }
 }
 
 /**
@@ -456,7 +506,7 @@ export class DatabaseItemImpl implements DatabaseItem {
 function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | undefined> {
   return new Promise((res, _rej) => {
     const timeout = setTimeout(() => {
-      logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
+      void logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
       res(undefined);
       dispose();
     }, timeoutMs);
@@ -479,42 +529,64 @@ export class DatabaseManager extends DisposableObject {
   private readonly _onDidChangeCurrentDatabaseItem = this.push(new vscode.EventEmitter<DatabaseChangedEvent>());
   readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event;
 
-  private readonly _databaseItems: DatabaseItemImpl[] = [];
+  private readonly _databaseItems: DatabaseItem[] = [];
   private _currentDatabaseItem: DatabaseItem | undefined = undefined;
 
   constructor(
-    private ctx: ExtensionContext,
-    public config: QueryServerConfig,
+    private readonly ctx: ExtensionContext,
+    private readonly qs: QueryServerClient,
+    private readonly cli: cli.CodeQLCliServer,
     public logger: Logger
   ) {
     super();
 
-    this.loadPersistedState();  // Let this run async.
+    qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
+
+    // Let this run async.
+    void this.loadPersistedState();
   }
 
   public async openDatabase(
-    uri: vscode.Uri, options?: DatabaseOptions
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    uri: vscode.Uri,
   ): Promise<DatabaseItem> {
-
     const contents = await resolveDatabaseContents(uri);
-    const realOptions = options || {};
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj';
     const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ?
-        realOptions.ignoreSourceArchive : isQLTestDatabase,
-      displayName: realOptions.displayName,
-      dateAdded: realOptions.dateAdded || Date.now()
+      ignoreSourceArchive: isQLTestDatabase,
+      // displayName is only set if a user explicitly renames a database
+      displayName: undefined,
+      dateAdded: Date.now(),
+      language: await this.getPrimaryLanguage(uri.fsPath)
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => {
       this._onDidChangeDatabaseItem.fire(event);
     });
-    await this.addDatabaseItem(databaseItem);
+
+    await this.addDatabaseItem(progress, token, databaseItem);
     await this.addDatabaseSourceArchiveFolder(databaseItem);
 
     return databaseItem;
   }
 
+  private async reregisterDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken
+  ) {
+    let completed = 0;
+    await Promise.all(this._databaseItems.map(async (databaseItem) => {
+      await this.registerDatabase(progress, token, databaseItem);
+      completed++;
+      progress({
+        maxStep: this._databaseItems.length,
+        step: completed,
+        message: 'Re-registering databases'
+      });
+    }));
+  }
+
   private async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
     // The folder may already be in workspace state from a previous
     // session. If not, add it.
@@ -533,15 +605,15 @@ export class DatabaseManager extends DisposableObject {
       const end = (vscode.workspace.workspaceFolders || []).length;
       const uri = item.getSourceArchiveExplorerUri();
       if (uri === undefined) {
-        logger.log(`Couldn't obtain file explorer uri for ${item.name}`);
+        void logger.log(`Couldn't obtain file explorer uri for ${item.name}`);
       }
       else {
-        logger.log(`Adding workspace folder for ${item.name} source archive at index ${end}`);
+        void logger.log(`Adding workspace folder for ${item.name} source archive at index ${end}`);
         if ((vscode.workspace.workspaceFolders || []).length < 2) {
           // Adding this workspace folder makes the workspace
           // multi-root, which may surprise the user. Let them know
           // we're doing this.
-          vscode.window.showInformationMessage(`Adding workspace folder for source archive of database ${item.name}.`);
+          void vscode.window.showInformationMessage(`Adding workspace folder for source archive of database ${item.name}.`);
         }
         vscode.workspace.updateWorkspaceFolders(end, 0, {
           name: `[${item.name} source archive]`,
@@ -555,12 +627,15 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async createDatabaseItemFromPersistedState(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
     state: PersistedDatabaseItem
   ): Promise<DatabaseItem> {
 
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
     let dateAdded = undefined;
+    let language = undefined;
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -571,43 +646,69 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.dateAdded === 'number') {
         dateAdded = state.options.dateAdded;
       }
+      language = state.options.language;
     }
+
+    const dbBaseUri = vscode.Uri.parse(state.uri, true);
+    if (language === undefined) {
+      // we haven't been successful yet at getting the language. try again
+      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
+    }
+
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive,
       displayName,
-      dateAdded
+      dateAdded,
+      language
     };
-    const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri, true), undefined, fullOptions,
+    const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions,
       (event) => {
         this._onDidChangeDatabaseItem.fire(event);
       });
-    await this.addDatabaseItem(item);
 
+    await this.addDatabaseItem(progress, token, item);
     return item;
   }
 
   private async loadPersistedState(): Promise<void> {
-    const currentDatabaseUri = this.ctx.workspaceState.get<string>(CURRENT_DB);
-    const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(DB_LIST, []);
-
-    try {
-      for (const database of databases) {
-        const databaseItem = await this.createDatabaseItemFromPersistedState(database);
+    return withProgress({
+      location: vscode.ProgressLocation.Notification
+    },
+      async (progress, token) => {
+        const currentDatabaseUri = this.ctx.workspaceState.get<string>(CURRENT_DB);
+        const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(DB_LIST, []);
+        let step = 0;
+        progress({
+          maxStep: databases.length,
+          message: 'Loading persisted databases',
+          step
+        });
         try {
-          await databaseItem.refresh();
-          if (currentDatabaseUri === database.uri) {
-            this.setCurrentDatabaseItem(databaseItem, true);
+          for (const database of databases) {
+            progress({
+              maxStep: databases.length,
+              message: `Loading ${database.options?.displayName || 'databases'}`,
+              step: ++step
+            });
+
+            const databaseItem = await this.createDatabaseItemFromPersistedState(progress, token, database);
+            try {
+              await databaseItem.refresh();
+              await this.registerDatabase(progress, token, databaseItem);
+              if (currentDatabaseUri === database.uri) {
+                await this.setCurrentDatabaseItem(databaseItem, true);
+              }
+            }
+            catch (e) {
+              // When loading from persisted state, leave invalid databases in the list. They will be
+              // marked as invalid, and cannot be set as the current database.
+            }
           }
+        } catch (e) {
+          // database list had an unexpected type - nothing to be done?
+          void showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
         }
-        catch (e) {
-          // When loading from persisted state, leave invalid databases in the list. They will be
-          // marked as invalid, and cannot be set as the current database.
-        }
-      }
-    } catch (e) {
-      // database list had an unexpected type - nothing to be done?
-      showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
-    }
+      });
   }
 
   public get databaseItems(): readonly DatabaseItem[] {
@@ -618,8 +719,10 @@ export class DatabaseManager extends DisposableObject {
     return this._currentDatabaseItem;
   }
 
-  public async setCurrentDatabaseItem(item: DatabaseItem | undefined,
-    skipRefresh = false): Promise<void> {
+  public async setCurrentDatabaseItem(
+    item: DatabaseItem | undefined,
+    skipRefresh = false
+  ): Promise<void> {
 
     if (!skipRefresh && (item !== undefined)) {
       await item.refresh();  // Will throw on invalid database.
@@ -627,6 +730,7 @@ export class DatabaseManager extends DisposableObject {
     if (this._currentDatabaseItem !== item) {
       this._currentDatabaseItem = item;
       this.updatePersistedCurrentDatabaseItem();
+
       this._onDidChangeCurrentDatabaseItem.fire({
         item,
         kind: DatabaseEventKind.Change
@@ -653,9 +757,20 @@ export class DatabaseManager extends DisposableObject {
     return this._databaseItems.find(item => item.sourceArchive && item.sourceArchive.toString(true) === uriString);
   }
 
-  private async addDatabaseItem(item: DatabaseItemImpl) {
+  private async addDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem
+  ) {
     this._databaseItems.push(item);
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
+
+    // Add this database item to the allow-list
+    // Database items reconstituted from persisted state
+    // will not have their contents yet.
+    if (item.contents?.datasetUri) {
+      await this.registerDatabase(progress, token, item);
+    }
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
       item: undefined,
@@ -665,7 +780,7 @@ export class DatabaseManager extends DisposableObject {
 
   public async renameDatabaseItem(item: DatabaseItem, newName: string) {
     item.name = newName;
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
     this._onDidChangeDatabaseItem.fire({
       // pass undefined so that the entire tree is rebuilt in order to re-sort
       item: undefined,
@@ -673,30 +788,40 @@ export class DatabaseManager extends DisposableObject {
     });
   }
 
-  public removeDatabaseItem(item: DatabaseItem) {
-    if (this._currentDatabaseItem == item)
+  public async removeDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem
+  ) {
+    if (this._currentDatabaseItem == item) {
       this._currentDatabaseItem = undefined;
+    }
     const index = this.databaseItems.findIndex(searchItem => searchItem === item);
     if (index >= 0) {
       this._databaseItems.splice(index, 1);
     }
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
 
     // Delete folder from workspace, if it is still there
-    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(folder => item.belongsToSourceArchiveExplorerUri(folder.uri));
-    if (index >= 0) {
-      logger.log(`Removing workspace folder at index ${folderIndex}`);
+    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
+      folder => item.belongsToSourceArchiveExplorerUri(folder.uri)
+    );
+    if (folderIndex >= 0) {
+      void logger.log(`Removing workspace folder at index ${folderIndex}`);
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
     // Delete folder from file system only if it is controlled by the extension
     if (this.isExtensionControlledLocation(item.databaseUri)) {
-      logger.log('Deleting database from filesystem.');
-      fs.remove(item.databaseUri.path).then(
-        () => logger.log(`Deleted '${item.databaseUri.path}'`),
-        e => logger.log(`Failed to delete '${item.databaseUri.path}'. Reason: ${e.message}`));
+      void logger.log('Deleting database from filesystem.');
+      fs.remove(item.databaseUri.fsPath).then(
+        () => void logger.log(`Deleted '${item.databaseUri.fsPath}'`),
+        e => void logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
     }
 
+    // Remove this database item from the allow-list
+    await this.deregisterDatabase(progress, token, item);
+
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
       item: undefined,
@@ -704,18 +829,63 @@ export class DatabaseManager extends DisposableObject {
     });
   }
 
+  private async deregisterDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
+    }
+  }
+
+  private async registerDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
+    }
+  }
+
   private updatePersistedCurrentDatabaseItem(): void {
-    this.ctx.workspaceState.update(CURRENT_DB, this._currentDatabaseItem ?
+    void this.ctx.workspaceState.update(CURRENT_DB, this._currentDatabaseItem ?
       this._currentDatabaseItem.databaseUri.toString(true) : undefined);
   }
 
-  private updatePersistedDatabaseList(): void {
-    this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
+  private async updatePersistedDatabaseList(): Promise<void> {
+    await this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
   }
 
   private isExtensionControlledLocation(uri: vscode.Uri) {
     const storagePath = this.ctx.storagePath || this.ctx.globalStoragePath;
-    return uri.path.startsWith(storagePath);
+    // the uri.fsPath function on windows returns a lowercase drive letter,
+    // but storagePath will have an uppercase drive letter. Be sure to compare
+    // URIs to URIs only
+    if (storagePath) {
+      return uri.fsPath.startsWith(vscode.Uri.file(storagePath).fsPath);
+    }
+    return false;
+  }
+
+  private async getPrimaryLanguage(dbPath: string) {
+    if (!(await this.cli.cliConstraints.supportsLanguageName())) {
+      // return undefined so that we recalculate on restart until the cli is at a version that
+      // supports this feature. This recalculation is cheap since we avoid calling into the cli
+      // unless we know it can return the langauges property.
+      return undefined;
+    }
+    const dbInfo = await this.cli.resolveDatabase(dbPath);
+    return dbInfo.languages?.[0] || '';
   }
 }
 
@@ -728,23 +898,3 @@ export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] {
   const uniqueParentDirs = new Set(parentDirs);
   return Array.from(uniqueParentDirs).map(filePath => vscode.Uri.file(filePath));
 }
-
-
-// TODO: Get the list of supported languages from a list that will be auto-updated.
-
-export async function isLikelyDatabaseRoot(fsPath: string) {
-  const [a, b, c] = (await Promise.all([
-    // databases can have either .dbinfo or codeql-database.yml.
-    fs.pathExists(path.join(fsPath, '.dbinfo')),
-    fs.pathExists(path.join(fsPath, 'codeql-database.yml')),
-
-    // they *must* have a db-language folder
-    (await fs.readdir(fsPath)).some(isLikelyDbLanguageFolder)
-  ]));
-
-  return (a || b) && c;
-}
-
-export function isLikelyDbLanguageFolder(dbPath: string) {
-  return !!path.basename(dbPath).startsWith('db-');
-}

extensions/ql-vscode/src/discovery.ts

@@ -1,5 +1,5 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { showAndLogErrorMessage } from './helpers';
+import { DisposableObject } from './pure/disposable-object';
+import { logger } from './logging';
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
@@ -59,23 +59,23 @@ export abstract class Discovery<T> extends DisposableObject {
         this.discoveryInProgress = false;
         this.update(results);
       }
-    });
+    })
 
-    discoveryPromise.catch(err => {
-      showAndLogErrorMessage(`${this.name} failed. Reason: ${err.message}`);
-    });
+      .catch(err => {
+        void logger.log(`${this.name} failed. Reason: ${err.message}`);
+      })
 
-    discoveryPromise.finally(() => {
-      if (this.retry) {
-        // Another refresh request came in while we were still running a previous discovery
-        // operation. Since the discovery results we just computed are now stale, we'll launch
-        // another discovery operation instead of updating.
-        // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-        // initial discovery operation failed.
-        this.retry = false;
-        this.launchDiscovery();
-      }
-    });
+      .finally(() => {
+        if (this.retry) {
+          // Another refresh request came in while we were still running a previous discovery
+          // operation. Since the discovery results we just computed are now stale, we'll launch
+          // another discovery operation instead of updating.
+          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+          // initial discovery operation failed.
+          this.retry = false;
+          this.launchDiscovery();
+        }
+      });
   }
 
   /**

extensions/ql-vscode/src/distribution.ts

@@ -7,10 +7,15 @@ import * as unzipper from 'unzipper';
 import * as url from 'url';
 import { ExtensionContext, Event } from 'vscode';
 import { DistributionConfig } from './config';
-import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from './helpers';
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage
+} from './helpers';
 import { logger } from './logging';
-import * as helpers from './helpers';
 import { getCodeQlCliVersion } from './cli-version';
+import { ProgressCallback, reportStreamProgress } from './commandRunner';
 
 /**
  * distribution.ts
@@ -49,16 +54,36 @@ export interface DistributionProvider {
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
-    this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
+
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  public static getRequiredAssetName(): string {
+    switch (os.platform()) {
+      case 'linux':
+        return 'codeql-linux64.zip';
+      case 'darwin':
+        return 'codeql-osx64.zip';
+      case 'win32':
+        return 'codeql-win64.zip';
+      default:
+        return 'codeql.zip';
+    }
+  }
+
+  constructor(
+    public readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    extensionContext: ExtensionContext
+  ) {
     this._onDidChangeDistribution = config.onDidChangeConfiguration;
-    this._updateCheckRateLimiter = new InvocationRateLimiter(
+    this.extensionSpecificDistributionManager =
+      new ExtensionSpecificDistributionManager(config, versionRange, extensionContext);
+    this.updateCheckRateLimiter = new InvocationRateLimiter(
       extensionContext,
       'extensionSpecificDistributionUpdateCheck',
-      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+      () => this.extensionSpecificDistributionManager.checkForUpdatesToDistribution()
     );
-    this._versionRange = versionRange;
   }
 
   /**
@@ -95,9 +120,9 @@ export class DistributionManager implements DistributionProvider {
      * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
      * includePrerelease config option is set.
      */
-    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this.config.includePrerelease;
 
-    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+    if (!semver.satisfies(version, this.versionRange, { includePrerelease })) {
       return {
         distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
@@ -126,9 +151,9 @@ export class DistributionManager implements DistributionProvider {
    */
   async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath) {
-      if (!await fs.pathExists(this._config.customCodeQlPath)) {
-        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
+    if (this.config.customCodeQlPath) {
+      if (!await fs.pathExists(this.config.customCodeQlPath)) {
+        void showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
           'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
           'that a CodeQL executable exists at the specified path or remove the setting.');
         return undefined;
@@ -137,18 +162,18 @@ export class DistributionManager implements DistributionProvider {
       // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
       if (
         deprecatedCodeQlLauncherName() &&
-        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        this.config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
         await this.hasNewLauncherName()
       ) {
         warnDeprecatedLauncher();
       }
       return {
-        codeQlPath: this._config.customCodeQlPath,
+        codeQlPath: this.config.customCodeQlPath,
         kind: DistributionKind.CustomPathConfig
       };
     }
 
-    const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionSpecificCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
       return {
         codeQlPath: extensionSpecificCodeQlPath,
@@ -166,7 +191,7 @@ export class DistributionManager implements DistributionProvider {
           };
         }
       }
-      logger.log('INFO: Could not find CodeQL on path.');
+      void logger.log('INFO: Could not find CodeQL on path.');
     }
 
     return undefined;
@@ -181,12 +206,12 @@ export class DistributionManager implements DistributionProvider {
   public async checkForUpdatesToExtensionManagedDistribution(
     minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
     const distribution = await this.getDistributionWithoutVersionCheck();
-    const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionManagedCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
       return createInvalidLocationResult();
     }
-    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    const updateCheckResult = await this.updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
     switch (updateCheckResult.kind) {
       case InvocationRateLimiterResultKind.Invoked:
         return updateCheckResult.result;
@@ -200,9 +225,11 @@ export class DistributionManager implements DistributionProvider {
    *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
-  public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
-    return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
+  public installExtensionManagedDistributionRelease(
+    release: Release,
+    progressCallback?: ProgressCallback
+  ): Promise<void> {
+    return this.extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
   }
 
   public get onDidChangeDistribution(): Event<void> | undefined {
@@ -215,27 +242,27 @@ export class DistributionManager implements DistributionProvider {
    * installation. False otherwise.
    */
   private async hasNewLauncherName(): Promise<boolean> {
-    if (!this._config.customCodeQlPath) {
+    if (!this.config.customCodeQlPath) {
       // not managed externally
       return false;
     }
-    const dir = path.dirname(this._config.customCodeQlPath);
+    const dir = path.dirname(this.config.customCodeQlPath);
     const newLaunderPath = path.join(dir, codeQlLauncherName());
     return await fs.pathExists(newLaunderPath);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
-  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
+  private readonly extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionRange: semver.Range;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
-    this._extensionContext = extensionContext;
-    this._config = config;
-    this._versionRange = versionRange;
+  constructor(
+    private readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    private readonly extensionContext: ExtensionContext
+  ) {
+    /**/
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@@ -249,7 +276,7 @@ class ExtensionSpecificDistributionManager {
       try {
         await this.removeDistribution();
       } catch (e) {
-        logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
+        void logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
           `${this.getDistributionStoragePath()} but encountered an error: ${e}.`);
       }
     }
@@ -283,29 +310,29 @@ class ExtensionSpecificDistributionManager {
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async installDistributionRelease(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     await this.downloadDistribution(release, progressCallback);
     // Store the installed release within the global extension state.
-    this.storeInstalledRelease(release);
+    await this.storeInstalledRelease(release);
   }
 
   private async downloadDistribution(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     try {
       await this.removeDistribution();
     } catch (e) {
-      logger.log(`Tried to clean up old version of CLI at ${this.getDistributionStoragePath()} ` +
+      void logger.log(`Tried to clean up old version of CLI at ${this.getDistributionStoragePath()} ` +
         `but encountered an error: ${e}.`);
     }
 
     // Filter assets to the unique one that we require.
-    const requiredAssetName = this.getRequiredAssetName();
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
     const assets = release.assets.filter(asset => asset.name === requiredAssetName);
     if (assets.length === 0) {
       throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
     }
     if (assets.length > 1) {
-      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+      void logger.log('WARNING: chose a release with more than one asset to install, found ' +
         assets.map(asset => asset.name).join(', '));
     }
 
@@ -317,27 +344,8 @@ class ExtensionSpecificDistributionManager {
       const archiveFile = fs.createWriteStream(archivePath);
 
       const contentLength = assetStream.headers.get('content-length');
-      let numBytesDownloaded = 0;
-
-      if (progressCallback && contentLength !== null) {
-        const totalNumBytes = parseInt(contentLength, 10);
-        const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-        const updateProgress = (): void => {
-          progressCallback({
-            step: numBytesDownloaded,
-            maxStep: totalNumBytes,
-            message: `Downloading CodeQL CLI… [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
-          });
-        };
-
-        // Display the progress straight away rather than waiting for the first chunk.
-        updateProgress();
-
-        assetStream.body.on('data', data => {
-          numBytesDownloaded += data.length;
-          updateProgress();
-        });
-      }
+      const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
+      reportStreamProgress(assetStream.body, `Downloading CodeQL CLI ${release.name}…`, totalNumBytes, progressCallback);
 
       await new Promise((resolve, reject) =>
         assetStream.body.pipe(archiveFile)
@@ -347,7 +355,7 @@ class ExtensionSpecificDistributionManager {
 
       await this.bumpDistributionFolderIndex();
 
-      logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
+      void logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
       await extractZipArchive(archivePath, this.getDistributionStoragePath());
     } finally {
       await fs.remove(tmpDirectory);
@@ -360,37 +368,27 @@ class ExtensionSpecificDistributionManager {
    * This should not be called for a distribution that is currently in use, as remove may fail.
    */
   private async removeDistribution(): Promise<void> {
-    this.storeInstalledRelease(undefined);
+    await this.storeInstalledRelease(undefined);
     if (await fs.pathExists(this.getDistributionStoragePath())) {
       await fs.remove(this.getDistributionStoragePath());
     }
   }
 
-  /**
-   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
-   */
-  private getRequiredAssetName(): string {
-    if (os.platform() === 'linux') return 'codeql-linux64.zip';
-    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
-    if (os.platform() === 'win32') return 'codeql-win64.zip';
-    return 'codeql.zip';
-  }
-
   private async getLatestRelease(): Promise<Release> {
-    const requiredAssetName = this.getRequiredAssetName();
-    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
+    void logger.log(`Searching for latest release including ${requiredAssetName}.`);
     return this.createReleasesApiConsumer().getLatestRelease(
-      this._versionRange,
-      this._config.includePrerelease,
+      this.versionRange,
+      this.config.includePrerelease,
       release => {
         const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
         if (matchingAssets.length === 0) {
           // For example, this could be a release with no platform-specific assets.
-          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          void logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
           return false;
         }
         if (matchingAssets.length > 1) {
-          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          void logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
           return false;
         }
         return true;
@@ -399,23 +397,23 @@ class ExtensionSpecificDistributionManager {
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
-    const ownerName = this._config.ownerName ? this._config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
-    const repositoryName = this._config.repositoryName ? this._config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
-    return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
+    const ownerName = this.config.ownerName ? this.config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
+    const repositoryName = this.config.repositoryName ? this.config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
+    return new ReleasesApiConsumer(ownerName, repositoryName, this.config.personalAccessToken);
   }
 
   private async bumpDistributionFolderIndex(): Promise<void> {
-    const index = this._extensionContext.globalState.get(
+    const index = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    await this._extensionContext.globalState.update(
+    await this.extensionContext.globalState.update(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
   }
 
   private getDistributionStoragePath(): string {
     // Use an empty string for the initial distribution for backwards compatibility.
-    const distributionFolderIndex = this._extensionContext.globalState.get(
+    const distributionFolderIndex = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || '';
-    return path.join(this._extensionContext.globalStoragePath,
+    return path.join(this.extensionContext.globalStoragePath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName + distributionFolderIndex);
   }
 
@@ -425,17 +423,13 @@ class ExtensionSpecificDistributionManager {
   }
 
   private getInstalledRelease(): Release | undefined {
-    return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
+    return this.extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
   }
 
   private async storeInstalledRelease(release: Release | undefined): Promise<void> {
-    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+    await this.extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionContext: ExtensionContext;
-  private readonly _versionRange: semver.Range;
-
   private static readonly _currentDistributionFolderBaseName = 'distribution';
   private static readonly _currentDistributionFolderIndexStateKey = 'distributionFolderIndex';
   private static readonly _installedReleaseStateKey = 'distributionRelease';
@@ -576,7 +570,7 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-function codeQlLauncherName(): string {
+export function codeQlLauncherName(): string {
   return (os.platform() === 'win32') ? 'codeql.exe' : 'codeql';
 }
 
@@ -713,14 +707,14 @@ export async function getExecutableFromDirectory(directory: string, warnWhenNotF
     return alternateExpectedLauncherPath;
   }
   if (warnWhenNotFound) {
-    logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+    void logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
       'Will try PATH.');
   }
   return undefined;
 }
 
 function warnDeprecatedLauncher() {
-  helpers.showAndLogWarningMessage(
+  void showAndLogWarningMessage(
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
     `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
   );

extensions/ql-vscode/src/extension.ts

@@ -13,12 +13,13 @@ import {
   window
 } from 'vscode';
 import { LanguageClient } from 'vscode-languageclient';
+import * as os from 'os';
 import * as path from 'path';
 import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
 
 import { AstViewer } from './astViewer';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { CodeQLCliServer } from './cli';
+import { CodeQLCliServer, CliVersionConstraint } from './cli';
 import {
   CliConfigListener,
   DistributionConfigListener,
@@ -59,6 +60,15 @@ import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
 import { CompareInterfaceManager } from './compare/compare-interface';
 import { gatherQlFiles } from './pure/files';
+import { initializeTelemetry } from './telemetry';
+import {
+  commandRunner,
+  commandRunnerWithProgress,
+  ProgressCallback,
+  withProgress,
+  ProgressUpdate
+} from './commandRunner';
+import { CodeQlStatusBarHandler } from './status-bar';
 
 /**
  * extension.ts
@@ -87,6 +97,9 @@ const errorStubs: Disposable[] = [];
  */
 let isInstallingOrUpdatingDistribution = false;
 
+const extensionId = 'GitHub.vscode-codeql';
+const extension = extensions.getExtension(extensionId);
+
 /**
  * If the user tries to execute vscode commands after extension activation is failed, give
  * a sensible error message.
@@ -97,8 +110,6 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
-  const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
-  const extension = extensions.getExtension(extensionId);
   if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
   }
@@ -108,26 +119,54 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 
   stubbedCommands.forEach(command => {
     if (excludedCommands.indexOf(command) === -1) {
-      errorStubs.push(helpers.commandRunner(command, stubGenerator(command)));
+      errorStubs.push(commandRunner(command, stubGenerator(command)));
     }
   });
 }
 
-export async function activate(ctx: ExtensionContext): Promise<void> {
-  logger.log('Starting CodeQL extension');
+/**
+ * The publicly available interface for this extension. This is to
+ * be used in our tests.
+ */
+export interface CodeQLExtensionInterface {
+  readonly ctx: ExtensionContext;
+  readonly cliServer: CodeQLCliServer;
+  readonly qs: qsClient.QueryServerClient;
+  readonly distributionManager: DistributionManager;
+  readonly databaseManager: DatabaseManager;
+  readonly databaseUI: DatabaseUI;
+  readonly dispose: () => void;
+}
 
-  initializeLogging(ctx);
-  languageSupport.install();
+/**
+ * Returns the CodeQLExtensionInterface, or an empty object if the interface is not
+ * available after activation is complete. This will happen if there is no cli
+ * installed when the extension starts. Downloading and installing the cli
+ * will happen at a later time.
+ *
+ * @param ctx The extension context
+ *
+ * @returns CodeQLExtensionInterface
+ */
+export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | Record<string, never>> {
+  void logger.log(`Starting ${extensionId} extension`);
+  if (extension === undefined) {
+    throw new Error(`Can't find extension ${extensionId}`);
+  }
 
   const distributionConfigListener = new DistributionConfigListener();
+  await initializeLogging(ctx);
+  await initializeTelemetry(extension, ctx);
+  languageSupport.install();
+
   ctx.subscriptions.push(distributionConfigListener);
   const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
+  const distributionManager = new DistributionManager(distributionConfigListener, codeQlVersionRange, ctx);
 
   const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
   registerErrorStubs([checkForUpdatesCommand], command => (async () => {
-    helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
+    void helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
   }));
 
   interface DistributionUpdateConfig {
@@ -139,7 +178,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
   async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
     const minSecondsSinceLastUpdateCheck = config.isUserInitiated ? 0 : 86400;
     const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
-      helpers.showAndLogInformationMessage : async (message: string) => logger.log(message);
+      helpers.showAndLogInformationMessage : async (message: string) => void logger.log(message);
     const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
 
     // We do want to auto update if there is no distribution at all
@@ -147,7 +186,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
     switch (result.kind) {
       case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
-        logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
+        void logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
           `${minSecondsSinceLastUpdateCheck} seconds.`);
         break;
       case DistributionUpdateCheckResultKind.AlreadyUpToDate:
@@ -170,11 +209,11 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
             location: ProgressLocation.Notification,
           };
 
-          await helpers.withProgress(progressOptions, progress =>
+          await withProgress(progressOptions, progress =>
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
-          helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
+          void helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
         }
         break;
       default:
@@ -206,12 +245,12 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
         codeQlInstalled ? 'check for updates to' : 'install') + ' CodeQL CLI';
 
       if (e instanceof GithubRateLimitedError) {
-        alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
+        void alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
           `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString(env.language)}.`);
       } else if (e instanceof GithubApiError) {
-        alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
+        void alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
       }
-      alertFunction(`Unable to ${taskDescription}. ` + e);
+      void alertFunction(`Unable to ${taskDescription}. ` + e);
     } finally {
       isInstallingOrUpdatingDistribution = false;
     }
@@ -221,7 +260,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
+        void logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
       case FindDistributionResultKind.IncompatibleDistribution: {
         const fixGuidanceMessage = (() => {
@@ -236,16 +275,20 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
           }
         })();
 
-        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
-          'is incompatible with this extension. ' + fixGuidanceMessage);
+        void helpers.showAndLogWarningMessage(
+          `The current version of the CodeQL CLI (${result.version.raw}) ` +
+          `is incompatible with this extension. ${fixGuidanceMessage}`
+        );
         break;
       }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
-        helpers.showAndLogWarningMessage('Compatibility with the configured CodeQL CLI could not be determined. ' +
-          'You may experience problems using the extension.');
+        void helpers.showAndLogWarningMessage(
+          'Compatibility with the configured CodeQL CLI could not be determined. ' +
+          'You may experience problems using the extension.'
+        );
         break;
       case FindDistributionResultKind.NoDistribution:
-        helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
+        void helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
         break;
       default:
         assertNever(result);
@@ -253,22 +296,30 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
+  async function installOrUpdateThenTryActivate(
+    config: DistributionUpdateConfig
+  ): Promise<CodeQLExtensionInterface | Record<string, never>> {
+
     await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
-
+    let extensionInterface: CodeQLExtensionInterface | Record<string, never> = {};
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(
+        ctx,
+        distributionManager,
+        distributionConfigListener
+      );
+
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = 'Install CodeQL CLI';
-        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+        const chosenAction = await void helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
           items: [installActionName]
         });
         if (chosenAction === installActionName) {
-          installOrUpdateThenTryActivate({
+          await installOrUpdateThenTryActivate({
             isUserInitiated: true,
             shouldDisplayMessageWhenNoUpdates: false,
             allowAutoUpdating: true
@@ -276,6 +327,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
         }
       });
     }
+    return extensionInterface;
   }
 
   ctx.subscriptions.push(distributionConfigListener.onDidChangeConfiguration(() => installOrUpdateThenTryActivate({
@@ -283,13 +335,13 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     shouldDisplayMessageWhenNoUpdates: false,
     allowAutoUpdating: true
   })));
-  ctx.subscriptions.push(helpers.commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
     isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: true,
     allowAutoUpdating: true
   })));
 
-  await installOrUpdateThenTryActivate({
+  return await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
 
@@ -301,20 +353,21 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
-  distributionManager: DistributionManager
-): Promise<void> {
+  distributionManager: DistributionManager,
+  distributionConfigListener: DistributionConfigListener
+): Promise<CodeQLExtensionInterface> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
   errorStubs.forEach((stub) => stub.dispose());
 
-  logger.log('Initializing configuration listener...');
+  void logger.log('Initializing configuration listener...');
   const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(
     distributionManager
   );
   ctx.subscriptions.push(qlConfigurationListener);
 
-  logger.log('Initializing CodeQL cli server...');
+  void logger.log('Initializing CodeQL cli server...');
   const cliServer = new CodeQLCliServer(
     distributionManager,
     new CliConfigListener(),
@@ -322,12 +375,16 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cliServer);
 
-  logger.log('Initializing query server client.');
+  const statusBar = new CodeQlStatusBarHandler(cliServer, distributionConfigListener);
+  ctx.subscriptions.push(statusBar);
+
+  void logger.log('Initializing query server client.');
   const qs = new qsClient.QueryServerClient(
     qlConfigurationListener,
     cliServer,
     {
       logger: queryServerLogger,
+      contextStoragePath: getContextStoragePath(ctx),
     },
     (task) =>
       Window.withProgress(
@@ -338,12 +395,11 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
-  logger.log('Initializing database manager.');
-  const dbm = new DatabaseManager(ctx, qlConfigurationListener, logger);
+  void logger.log('Initializing database manager.');
+  const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
   ctx.subscriptions.push(dbm);
-  logger.log('Initializing database panel.');
+  void logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(
-    cliServer,
     dbm,
     qs,
     getContextStoragePath(ctx),
@@ -352,8 +408,9 @@ async function activateWithInstalledDistribution(
   databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
-  logger.log('Initializing query history manager.');
+  void logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  ctx.subscriptions.push(queryHistoryConfigurationListener);
   const showResults = async (item: CompletedQuery) =>
     showResultsForCompletedQuery(item, WebviewReveal.Forced);
 
@@ -366,11 +423,11 @@ async function activateWithInstalledDistribution(
       showResultsForComparison(from, to),
   );
   ctx.subscriptions.push(qhm);
-  logger.log('Initializing results panel interface.');
+  void logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
 
-  logger.log('Initializing compare panel interface.');
+  void logger.log('Initializing compare panel interface.');
   const cmpm = new CompareInterfaceManager(
     ctx,
     dbm,
@@ -380,7 +437,7 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cmpm);
 
-  logger.log('Initializing source archive filesystem provider.');
+  void logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
   async function showResultsForComparison(
@@ -390,7 +447,7 @@ async function activateWithInstalledDistribution(
     try {
       await cmpm.showResults(from, to);
     } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+      void helpers.showAndLogErrorMessage(e.message);
     }
   }
 
@@ -404,7 +461,7 @@ async function activateWithInstalledDistribution(
   async function compileAndRunQuery(
     quickEval: boolean,
     selectedQuery: Uri | undefined,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<void> {
     if (qs !== undefined) {
@@ -421,18 +478,35 @@ async function activateWithInstalledDistribution(
         progress,
         token
       );
-      const item = qhm.addQuery(info);
+      const item = qhm.buildCompletedQuery(info);
       await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      // The call to showResults potentially creates SARIF file;
-      // Update the tree item context value to allow viewing that
-      // SARIF file from context menu.
-      await qhm.updateTreeItemContextValue(item);
+      // Note we must update the query history view after showing results as the
+      // display and sorting might depend on the number of results
+      await qhm.addCompletedQuery(item);
+    }
+  }
+
+  async function openReferencedFile(
+    selectedQuery: Uri
+  ): Promise<void> {
+    if (qs !== undefined) {
+      if (await cliServer.cliConstraints.supportsResolveQlref()) {
+        const resolved = await cliServer.resolveQlref(selectedQuery.path);
+        const uri = Uri.file(resolved.resolvedPath);
+        await window.showTextDocument(uri, { preview: false });
+      } else {
+        void helpers.showAndLogErrorMessage(
+          'Jumping from a .qlref file to the .ql file it references is not '
+          + 'supported with the CLI version you are running.\n'
+          + `Please upgrade your CLI to version ${CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF
+          } or later to use this feature.`);
+      }
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
-  logger.log('Initializing CodeQL language server.');
+  void logger.log('Initializing CodeQL language server.');
   const client = new LanguageClient(
     'CodeQL Language Server',
     () => spawnIdeServer(qlConfigurationListener),
@@ -450,25 +524,25 @@ async function activateWithInstalledDistribution(
     true
   );
 
-  logger.log('Initializing QLTest interface.');
+  void logger.log('Initializing QLTest interface.');
   const testExplorerExtension = extensions.getExtension<TestHub>(
     testExplorerExtensionId
   );
   if (testExplorerExtension) {
     const testHub = testExplorerExtension.exports;
-    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
+    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer, dbm);
     ctx.subscriptions.push(testAdapterFactory);
 
     const testUIService = new TestUIService(testHub);
     ctx.subscriptions.push(testUIService);
   }
 
-  logger.log('Registering top-level command palette commands.');
+  void logger.log('Registering top-level command palette commands.');
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.runQuery',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
       ) => await compileAndRunQuery(false, uri, progress, token),
@@ -479,10 +553,10 @@ async function activateWithInstalledDistribution(
     )
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.runQueries',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         _: Uri | undefined,
         multi: Uri[]
@@ -507,7 +581,7 @@ async function activateWithInstalledDistribution(
 
         // Use a wrapped progress so that messages appear with the queries remaining in it.
         let queriesRemaining = queryUris.length;
-        function wrappedProgress(update: helpers.ProgressUpdate) {
+        function wrappedProgress(update: ProgressUpdate) {
           const message = queriesRemaining > 1
             ? `${queriesRemaining} remaining. ${update.message}`
             : update.message;
@@ -543,10 +617,10 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.quickEval',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
       ) => await compileAndRunQuery(true, uri, progress, token),
@@ -556,8 +630,8 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.quickQuery', async (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.quickQuery', async (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       displayQuickQuery(ctx, cliServer, databaseUI, progress, token),
@@ -566,44 +640,58 @@ async function activateWithInstalledDistribution(
       }
     )
   );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.openReferencedFile',
+      openReferencedFile
+    )
+  );
 
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.restartQueryServer', async () => {
-      await qs.restartQueryServer();
-      helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
+    commandRunnerWithProgress('codeQL.restartQueryServer', async (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) => {
+      await qs.restartQueryServer(progress, token);
+      void helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
       });
+    }, {
+      title: 'Restarting Query Server'
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.chooseDatabaseFolder', (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseFolder(progress, token), {
+      title: 'Choose a Database from a Folder'
     })
   );
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.chooseDatabaseFolder', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseArchive', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
-      databaseUI.handleChooseDatabaseFolder(progress, token)
-    )
+      databaseUI.handleChooseDatabaseArchive(progress, token), {
+      title: 'Choose a Database from an Archive'
+    })
   );
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.chooseDatabaseArchive', (
-      progress: helpers.ProgressCallback,
-      token: CancellationToken
-    ) =>
-      databaseUI.handleChooseDatabaseArchive(progress, token)
-    )
-  );
-  ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseLgtm(progress, token),
       {
-        title: 'Adding database from LGTM',
+        title: 'Adding database from LGTM. Choose a language from the dropdown, if requested.',
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseInternet(progress, token),
@@ -613,11 +701,23 @@ async function activateWithInstalledDistribution(
       })
   );
 
-  logger.log('Starting language server.');
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openDocumentation', async () =>
+      env.openExternal(Uri.parse('https://codeql.github.com/docs/'))));
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.copyVersion', async () => {
+      const text = `CodeQL extension version: ${extension?.packageJSON.version} \nCodeQL CLI version: ${await cliServer.getVersion()} \nPlatform: ${os.platform()} ${os.arch()}`;
+      await env.clipboard.writeText(text);
+      void helpers.showAndLogInformationMessage(text);
+    }));
+
+
+  void logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
   // Jump-to-definition and find-references
-  logger.log('Registering jump-to-definition handlers.');
+  void logger.log('Registering jump-to-definition handlers.');
   languages.registerDefinitionProvider(
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
     new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
@@ -629,13 +729,18 @@ async function activateWithInstalledDistribution(
   );
 
   const astViewer = new AstViewer();
+  const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm);
+
   ctx.subscriptions.push(astViewer);
-  ctx.subscriptions.push(helpers.commandRunnerWithProgress('codeQL.viewAst', async (
-    progress: helpers.ProgressCallback,
+  ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
+    progress: ProgressCallback,
     token: CancellationToken
   ) => {
-    const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)
-      .provideAst(window.activeTextEditor?.document);
+    const ast = await templateProvider.provideAst(
+      progress,
+      token,
+      window.activeTextEditor?.document,
+    );
     if (ast) {
       astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
     }
@@ -644,20 +749,31 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
-  commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
+  await commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
 
-  logger.log('Successfully finished extension initialization.');
+  void logger.log('Successfully finished extension initialization.');
+
+  return {
+    ctx,
+    cliServer,
+    qs,
+    distributionManager,
+    databaseManager: dbm,
+    databaseUI,
+    dispose: () => {
+      ctx.subscriptions.forEach(d => d.dispose());
+    }
+  };
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {
   return ctx.storagePath || ctx.globalStoragePath;
 }
 
-function initializeLogging(ctx: ExtensionContext): void {
+async function initializeLogging(ctx: ExtensionContext): Promise<void> {
   const storagePath = getContextStoragePath(ctx);
-  logger.init(storagePath);
-  queryServerLogger.init(storagePath);
-  ideServerLogger.init(storagePath);
+  await logger.setLogStoragePath(storagePath, false);
+  await ideServerLogger.setLogStoragePath(storagePath, false);
   ctx.subscriptions.push(logger);
   ctx.subscriptions.push(queryServerLogger);
   ctx.subscriptions.push(ideServerLogger);

extensions/ql-vscode/src/helpers.ts

@@ -3,192 +3,39 @@ import * as glob from 'glob-promise';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
 import {
-  CancellationToken,
   ExtensionContext,
-  ProgressOptions,
+  Uri,
   window as Window,
   workspace,
-  commands,
-  Disposable,
-  ProgressLocation
+  env
 } from 'vscode';
 import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
 
-export class UserCancellationException extends Error {
-  /**
-   * @param message The error message
-   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
-   */
-  constructor(message?: string, public readonly silent = false) {
-    super(message);
-  }
-}
-
-export interface ProgressUpdate {
-  /**
-   * The current step
-   */
-  step: number;
-  /**
-   * The maximum step. This *should* be constant for a single job.
-   */
-  maxStep: number;
-  /**
-   * The current progress message
-   */
-  message: string;
-}
-
-export type ProgressCallback = (p: ProgressUpdate) => void;
-
-/**
- * A task that handles command invocations from `commandRunner`
- * and includes a progress monitor.
- *
- *
- * Arguments passed to the command handler are passed along,
- * untouched to this `ProgressTask` instance.
- *
- * @param progress a progress handler function. Call this
- * function with a `ProgressUpdate` instance in order to
- * denote some progress being achieved on this task.
- * @param token a cencellation token
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-export type ProgressTask<R> = (
-  progress: ProgressCallback,
-  token: CancellationToken,
-  ...args: any[]
-) => Thenable<R>;
-
-/**
- * A task that handles command invocations from `commandRunner`.
- * Arguments passed to the command handler are passed along,
- * untouched to this `NoProgressTask` instance.
- *
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-type NoProgressTask = ((...args: any[]) => Promise<any>);
-
-/**
- * This mediates between the kind of progress callbacks we want to
- * write (where we *set* current progress position and give
- * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%).
- *
- * Where possible, the `commandRunner` function below should be used
- * instead of this function. The commandRunner is meant for wrapping
- * top-level commands and provides error handling and other support
- * automatically.
- *
- * Only use this function if you need a progress monitor and the
- * control flow does not always come from a command (eg- during
- * extension activation, or from an internal language server
- * request).
- */
-export function withProgress<R>(
-  options: ProgressOptions,
-  task: ProgressTask<R>,
-  ...args: any[]
-): Thenable<R> {
-  let progressAchieved = 0;
-  return Window.withProgress(options,
-    (progress, token) => {
-      return task(p => {
-        const { message, step, maxStep } = p;
-        const increment = 100 * (step - progressAchieved) / maxStep;
-        progressAchieved = step;
-        progress.report({ message, increment });
-      }, token, ...args);
-    });
-}
-
-/**
- * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
- *
- * In this variant of the command runner, no progress monitor is used.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task.
- */
-export function commandRunner(
-  commandId: string,
-  task: NoProgressTask,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    try {
-      await task(...args);
-    } catch (e) {
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          logger.log(e.message);
-        } else {
-          showAndLogWarningMessage(e.message);
-        }
-      } else {
-        showAndLogErrorMessage(e.message || e);
-      }
-    }
-  });
-}
-
-/**
- * A generic wrapper for command registration.  This wrapper adds uniform error handling,
- * progress monitoring, and cancellation for commands.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task after the progress callback
- * and cancellation token.
- * @param progressOptions Progress options to be sent to the progress monitor.
- */
-export function commandRunnerWithProgress<R>(
-  commandId: string,
-  task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const progressOptionsWithDefaults = {
-      location: ProgressLocation.Notification,
-      ...progressOptions
-    };
-    try {
-      await withProgress(progressOptionsWithDefaults, task, ...args);
-    } catch (e) {
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          logger.log(e.message);
-        } else {
-          showAndLogWarningMessage(e.message);
-        }
-      } else {
-        showAndLogErrorMessage(e.message || e);
-      }
-    }
-  });
-}
-
 /**
  * Show an error message and log it to the console
  *
  * @param message The message to show.
  * @param options.outputLogger The output logger that will receive the message
  * @param options.items A set of items that will be rendered as actions in the message.
+ * @param options.fullMessage An alternate message that is added to the log, but not displayed
+ *                           in the popup. This is useful for adding extra detail to the logs
+ *                           that would be too noisy for the popup.
  *
  * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
 export async function showAndLogErrorMessage(message: string, {
   outputLogger = logger,
-  items = [] as string[]
+  items = [] as string[],
+  fullMessage = undefined as (string | undefined)
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
+  return internalShowAndLog(dropLinesExceptInitial(message), items, outputLogger, Window.showErrorMessage, fullMessage);
 }
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join('\n');
+}
+
 /**
  * Show a warning message and log it to the console
  *
@@ -222,10 +69,15 @@ export async function showAndLogInformationMessage(message: string, {
 
 type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
 
-async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
-  fn: ShowMessageFn): Promise<string | undefined> {
+async function internalShowAndLog(
+  message: string,
+  items: string[],
+  outputLogger = logger,
+  fn: ShowMessageFn,
+  fullMessage?: string
+): Promise<string | undefined> {
   const label = 'Show Log';
-  outputLogger.log(message);
+  void outputLogger.log(fullMessage || message);
   const result = await fn(message, label, ...items);
   if (result === label) {
     outputLogger.show();
@@ -235,17 +87,61 @@ async function internalShowAndLog(message: string, items: string[], outputLogger
 
 /**
  * Opens a modal dialog for the user to make a yes/no choice.
- * @param message The message to show.
  *
- * @return `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
  */
-export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
+export async function showBinaryChoiceDialog(message: string, modal = true): Promise<boolean | undefined> {
   const yesItem = { title: 'Yes', isCloseAffordance: false };
   const noItem = { title: 'No', isCloseAffordance: true };
-  const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
+  const chosenItem = await Window.showInformationMessage(message, { modal }, yesItem, noItem);
+  if (!chosenItem) {
+    return undefined;
+  }
   return chosenItem?.title === yesItem.title;
 }
 
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(message: string, url: string): Promise<boolean | undefined> {
+  const urlItem = { title: 'More Information', isCloseAffordance: false };
+  const yesItem = { title: 'Yes', isCloseAffordance: false };
+  const noItem = { title: 'No', isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await Window.showInformationMessage(message, { modal: true }, urlItem, yesItem, noItem);
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
 /**
  * Show an information message with a customisable action.
  * @param message The message to show.
@@ -358,22 +254,16 @@ function createRateLimitedResult(): RateLimitedResult {
   };
 }
 
-
-export type DatasetFolderInfo = {
-  dbscheme: string;
-  qlpack: string;
-}
-
 export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
   const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
   const packs: { packDir: string | undefined; packName: string }[] =
     Object.entries(qlpacks).map(([packName, dirs]) => {
       if (dirs.length < 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
+        void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
         return { packName, packDir: undefined };
       }
       if (dirs.length > 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
+        void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
       }
       return {
         packName,
@@ -391,7 +281,7 @@ export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemeP
   throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
 }
 
-export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise<DatasetFolderInfo> {
+export async function getPrimaryDbscheme(datasetFolder: string): Promise<string> {
   const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'));
 
   if (dbschemes.length < 1) {
@@ -400,31 +290,30 @@ export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFo
 
   dbschemes.sort();
   const dbscheme = dbschemes[0];
-  if (dbschemes.length > 1) {
-    Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
-  }
 
-  const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
-  return { dbscheme, qlpack };
+  if (dbschemes.length > 1) {
+    void Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
+  }
+  return dbscheme;
 }
 
 /**
  * A cached mapping from strings to value of type U.
  */
 export class CachedOperation<U> {
-  private readonly operation: (t: string) => Promise<U>;
+  private readonly operation: (t: string, ...args: any[]) => Promise<U>;
   private readonly cached: Map<string, U>;
   private readonly lru: string[];
   private readonly inProgressCallbacks: Map<string, [(u: U) => void, (reason?: any) => void][]>;
 
-  constructor(operation: (t: string) => Promise<U>, private cacheSize = 100) {
+  constructor(operation: (t: string, ...args: any[]) => Promise<U>, private cacheSize = 100) {
     this.operation = operation;
     this.lru = [];
     this.inProgressCallbacks = new Map<string, [(u: U) => void, (reason?: any) => void][]>();
     this.cached = new Map<string, U>();
   }
 
-  async get(t: string): Promise<U> {
+  async get(t: string, ...args: any[]): Promise<U> {
     // Try and retrieve from the cache
     const fromCache = this.cached.get(t);
     if (fromCache !== undefined) {
@@ -445,7 +334,7 @@ export class CachedOperation<U> {
     const callbacks: [(u: U) => void, (reason?: any) => void][] = [];
     this.inProgressCallbacks.set(t, callbacks);
     try {
-      const result = await this.operation(t);
+      const result = await this.operation(t, ...args);
       callbacks.forEach(f => f[0](result));
       this.inProgressCallbacks.delete(t);
       if (this.lru.length > this.cacheSize) {
@@ -464,3 +353,74 @@ export class CachedOperation<U> {
     }
   }
 }
+
+
+
+/**
+ * The following functions al heuristically determine metadata about databases.
+ */
+
+/**
+ * Note that this heuristic is only being used for backwards compatibility with
+ * CLI versions before the langauge name was introduced to dbInfo. Features
+ * that do not require backwards compatibility should call
+ * `cli.CodeQLCliServer.resolveDatabase` and use the first entry in the
+ * `languages` property.
+ *
+ * @see cli.CliVersionConstraint.supportsLanguageName
+ * @see cli.CodeQLCliServer.resolveDatabase
+ */
+const dbSchemeToLanguage = {
+  'semmlecode.javascript.dbscheme': 'javascript',
+  'semmlecode.cpp.dbscheme': 'cpp',
+  'semmlecode.dbscheme': 'java',
+  'semmlecode.python.dbscheme': 'python',
+  'semmlecode.csharp.dbscheme': 'csharp',
+  'go.dbscheme': 'go'
+};
+
+/**
+ * Returns the initial contents for an empty query, based on the language of the selected
+ * databse.
+ *
+ * First try to use the given language name. If that doesn't exist, try to infer it based on
+ * dbscheme. Otherwise return no import statement.
+ *
+ * @param language the database language or empty string if unknown
+ * @param dbscheme path to the dbscheme file
+ *
+ * @returns an import and empty select statement appropriate for the selected language
+ */
+export function getInitialQueryContents(language: string, dbscheme: string) {
+  if (!language) {
+    const dbschemeBase = path.basename(dbscheme) as keyof typeof dbSchemeToLanguage;
+    language = dbSchemeToLanguage[dbschemeBase];
+  }
+
+  return language
+    ? `import ${language}\n\nselect ""`
+    : 'select ""';
+}
+
+/**
+ * Heuristically determines if the directory passed in corresponds
+ * to a database root.
+ *
+ * @param maybeRoot
+ */
+export async function isLikelyDatabaseRoot(maybeRoot: string) {
+  const [a, b, c] = (await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    fs.pathExists(path.join(maybeRoot, '.dbinfo')),
+    fs.pathExists(path.join(maybeRoot, 'codeql-database.yml')),
+
+    // they *must* have a db-{language} folder
+    glob('db-*/', { cwd: maybeRoot })
+  ]));
+
+  return !!((a || b) && c);
+}
+
+export function isLikelyDbLanguageFolder(dbPath: string) {
+  return !!path.basename(dbPath).startsWith('db-');
+}

extensions/ql-vscode/src/interface-utils.ts

@@ -224,14 +224,14 @@ export async function jumpToLocation(
     } catch (e) {
       if (e instanceof Error) {
         if (e.message.match(/File not found/)) {
-          Window.showErrorMessage(
+          void Window.showErrorMessage(
             'Original file of this result is not in the database\'s source archive.'
           );
         } else {
-          logger.log(`Unable to handleMsgFromView: ${e.message}`);
+          void logger.log(`Unable to handleMsgFromView: ${e.message}`);
         }
       } else {
-        logger.log(`Unable to handleMsgFromView: ${e}`);
+        void logger.log(`Unable to handleMsgFromView: ${e}`);
       }
     }
   }

extensions/ql-vscode/src/interface.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as Sarif from 'sarif';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as vscode from 'vscode';
 import {
   Diagnostic,
@@ -30,8 +30,8 @@ import {
   RawResultsSortState,
 } from './pure/interface-types';
 import { Logger } from './logging';
-import { commandRunner } from './helpers';
 import * as messages from './pure/messages';
+import { commandRunner } from './commandRunner';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
@@ -119,7 +119,7 @@ export class InterfaceManager extends DisposableObject {
         this.handleSelectionChange.bind(this)
       )
     );
-    logger.log('Registering path-step navigation commands.');
+    void logger.log('Registering path-step navigation commands.');
     this.push(
       commandRunner(
         'codeQLQueryResults.nextPathStep',
@@ -137,16 +137,22 @@ export class InterfaceManager extends DisposableObject {
       this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
         if (kind === DatabaseEventKind.Remove) {
           this._diagnosticCollection.clear();
-          this.postMessage({
-            t: 'untoggleShowProblems'
-          });
+          if (this.isShowingPanel()) {
+            void this.postMessage({
+              t: 'untoggleShowProblems'
+            });
+          }
         }
       })
     );
   }
 
   async navigatePathStep(direction: number): Promise<void> {
-    this.postMessage({ t: 'navigatePath', direction });
+    await this.postMessage({ t: 'navigatePath', direction });
+  }
+
+  private isShowingPanel() {
+    return !!this._panel;
   }
 
   // Returns the webview panel, creating it if it doesn't already
@@ -168,6 +174,7 @@ export class InterfaceManager extends DisposableObject {
           ]
         }
       ));
+
       this._panel.onDidDispose(
         () => {
           this._panel = undefined;
@@ -200,14 +207,14 @@ export class InterfaceManager extends DisposableObject {
     sortState: InterpretedResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         'Failed to sort results since evaluation info was unknown.'
       );
       return;
     }
     // Notify the webview that it should expect new results.
     await this.postMessage({ t: 'resultsUpdating' });
-    this._displayedQuery.updateInterpretedSortState(sortState);
+    await this._displayedQuery.updateInterpretedSortState(sortState);
     await this.showResults(this._displayedQuery, WebviewReveal.NotForced, true);
   }
 
@@ -216,7 +223,7 @@ export class InterfaceManager extends DisposableObject {
     sortState: RawResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         'Failed to sort results since evaluation info was unknown.'
       );
       return;
@@ -236,58 +243,67 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
-    switch (msg.t) {
-      case 'viewSourceFile': {
-        await jumpToLocation(msg, this.databaseManager, this.logger);
-        break;
-      }
-      case 'toggleDiagnostics': {
-        if (msg.visible) {
-          const databaseItem = this.databaseManager.findDatabaseItem(
-            Uri.parse(msg.databaseUri)
-          );
-          if (databaseItem !== undefined) {
-            await this.showResultsAsDiagnostics(
-              msg.origResultsPaths,
-              msg.metadata,
-              databaseItem
+    try {
+      switch (msg.t) {
+        case 'viewSourceFile': {
+          await jumpToLocation(msg, this.databaseManager, this.logger);
+          break;
+        }
+        case 'toggleDiagnostics': {
+          if (msg.visible) {
+            const databaseItem = this.databaseManager.findDatabaseItem(
+              Uri.parse(msg.databaseUri)
+            );
+            if (databaseItem !== undefined) {
+              await this.showResultsAsDiagnostics(
+                msg.origResultsPaths,
+                msg.metadata,
+                databaseItem
+              );
+            }
+          } else {
+            // TODO: Only clear diagnostics on the same database.
+            this._diagnosticCollection.clear();
+          }
+          break;
+        }
+        case 'resultViewLoaded':
+          this._panelLoaded = true;
+          this._panelLoadedCallBacks.forEach((cb) => cb());
+          this._panelLoadedCallBacks = [];
+          break;
+        case 'changeSort':
+          await this.changeRawSortState(msg.resultSetName, msg.sortState);
+          break;
+        case 'changeInterpretedSort':
+          await this.changeInterpretedSortState(msg.sortState);
+          break;
+        case 'changePage':
+          if (msg.selectedTable === ALERTS_TABLE_NAME) {
+            await this.showPageOfInterpretedResults(msg.pageNumber);
+          }
+          else {
+            await this.showPageOfRawResults(
+              msg.selectedTable,
+              msg.pageNumber,
+              // When we are in an unsorted state, we guarantee that
+              // sortedResultsInfo doesn't have an entry for the current
+              // result set. Use this to determine whether or not we use
+              // the sorted bqrs file.
+              this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
             );
           }
-        } else {
-          // TODO: Only clear diagnostics on the same database.
-          this._diagnosticCollection.clear();
-        }
-        break;
+          break;
+        case 'openFile':
+          await this.openFile(msg.filePath);
+          break;
+        default:
+          assertNever(msg);
       }
-      case 'resultViewLoaded':
-        this._panelLoaded = true;
-        this._panelLoadedCallBacks.forEach((cb) => cb());
-        this._panelLoadedCallBacks = [];
-        break;
-      case 'changeSort':
-        await this.changeRawSortState(msg.resultSetName, msg.sortState);
-        break;
-      case 'changeInterpretedSort':
-        await this.changeInterpretedSortState(msg.sortState);
-        break;
-      case 'changePage':
-        if (msg.selectedTable === ALERTS_TABLE_NAME) {
-          await this.showPageOfInterpretedResults(msg.pageNumber);
-        }
-        else {
-          await this.showPageOfRawResults(
-            msg.selectedTable,
-            msg.pageNumber,
-            // When we are in an unsorted state, we guarantee that
-            // sortedResultsInfo doesn't have an entry for the current
-            // result set. Use this to determine whether or not we use
-            // the sorted bqrs file.
-            this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
-          );
-        }
-        break;
-      default:
-        assertNever(msg);
+    } catch (e) {
+      void showAndLogErrorMessage(e.message, {
+        fullMessage: e.stack
+      });
     }
   }
 
@@ -356,7 +372,7 @@ export class InterfaceManager extends DisposableObject {
       );
       // Address this click asynchronously so we still update the
       // query history immediately.
-      resultPromise.then((result) => {
+      void resultPromise.then((result) => {
         if (result === showButton) {
           panel.reveal();
         }
@@ -391,6 +407,7 @@ export class InterfaceManager extends DisposableObject {
       }
     );
     const resultSet = transformBqrsResultSet(schema, chunk);
+    results.setResultCount(interpretationPage?.numTotalResults || resultSet.schema.rows);
     const parsedResultSets: ParsedResultSets = {
       pageNumber: 0,
       pageSize,
@@ -413,6 +430,8 @@ export class InterfaceManager extends DisposableObject {
       database: results.database,
       shouldKeepOldResultsWhileRendering,
       metadata: results.query.metadata,
+      queryName: results.toString(),
+      queryPath: results.query.program.queryPath
     });
   }
 
@@ -444,6 +463,8 @@ export class InterfaceManager extends DisposableObject {
       resultSetNames,
       pageSize: PAGE_SIZE.getValue(),
       numPages: numInterpretedPages(this._interpretation),
+      queryName: this._displayedQuery.toString(),
+      queryPath: this._displayedQuery.query.program.queryPath
     });
   }
 
@@ -456,6 +477,11 @@ export class InterfaceManager extends DisposableObject {
     return schemas['result-sets'];
   }
 
+  public async openFile(filePath: string) {
+    const textDocument = await vscode.workspace.openTextDocument(filePath);
+    await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+  }
+
   /**
    * Show a page of raw results from the chosen table.
    */
@@ -517,6 +543,8 @@ export class InterfaceManager extends DisposableObject {
       database: results.database,
       shouldKeepOldResultsWhileRendering: false,
       metadata: results.query.metadata,
+      queryName: results.toString(),
+      queryPath: results.query.program.queryPath
     });
   }
 
@@ -526,23 +554,26 @@ export class InterfaceManager extends DisposableObject {
     sourceInfo: cli.SourceInfo | undefined,
     sourceLocationPrefix: string,
     sortState: InterpretedResultsSortState | undefined
-  ): Promise<Interpretation> {
+  ): Promise<Interpretation | undefined> {
+    if (!resultsPaths) {
+      void this.logger.log('No results path. Cannot display interpreted results.');
+      return undefined;
+    }
+
     const sarif = await interpretResults(
       this.cliServer,
       metadata,
       resultsPaths,
       sourceInfo
     );
+
     sarif.runs.forEach(run => {
-      if (run.results !== undefined)
+      if (run.results !== undefined) {
         sortInterpretedResults(run.results, sortState);
+      }
     });
 
-    const numTotalResults = (() => {
-      if (sarif.runs.length === 0) return 0;
-      if (sarif.runs[0].results === undefined) return 0;
-      return sarif.runs[0].results.length;
-    })();
+    const numTotalResults = sarif.runs[0]?.results?.length || 0;
 
     const interpretation: Interpretation = {
       sarif,
@@ -572,7 +603,7 @@ export class InterfaceManager extends DisposableObject {
       throw new Error('Tried to get interpreted results before interpretation finished');
     }
     if (this._interpretation.sarif.runs.length !== 1) {
-      this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
+      void this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
     }
     const interp = this._interpretation;
     return {
@@ -611,8 +642,8 @@ export class InterfaceManager extends DisposableObject {
       } catch (e) {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
-        this.logger.log(
-          `Exception during results interpretation: ${e.message}. Will show raw results instead.`
+        void showAndLogErrorMessage(
+          `Showing raw results instead of interpreted ones due to an error. ${e.message}`
         );
       }
     }
@@ -644,11 +675,15 @@ export class InterfaceManager extends DisposableObject {
       undefined
     );
 
+    if (!interpretation) {
+      return;
+    }
+
     try {
       await this.showProblemResultsAsDiagnostics(interpretation, database);
     } catch (e) {
       const msg = e instanceof Error ? e.message : e.toString();
-      this.logger.log(
+      void this.logger.log(
         `Exception while computing problem results as diagnostics: ${msg}`
       );
       this._diagnosticCollection.clear();
@@ -662,7 +697,7 @@ export class InterfaceManager extends DisposableObject {
     const { sarif, sourceLocationPrefix } = interpretation;
 
     if (!sarif.runs || !sarif.runs[0].results) {
-      this.logger.log(
+      void this.logger.log(
         'Didn\'t find a run in the sarif results. Error processing sarif?'
       );
       return;
@@ -673,11 +708,11 @@ export class InterfaceManager extends DisposableObject {
     for (const result of sarif.runs[0].results) {
       const message = result.message.text;
       if (message === undefined) {
-        this.logger.log('Sarif had result without plaintext message');
+        void this.logger.log('Sarif had result without plaintext message');
         continue;
       }
       if (!result.locations) {
-        this.logger.log('Sarif had result without location');
+        void this.logger.log('Sarif had result without location');
         continue;
       }
 
@@ -690,7 +725,7 @@ export class InterfaceManager extends DisposableObject {
       }
       const resultLocation = tryResolveLocation(sarifLoc, databaseItem);
       if (!resultLocation) {
-        this.logger.log('Sarif location was not resolvable ' + sarifLoc);
+        void this.logger.log('Sarif location was not resolvable ' + sarifLoc);
         continue;
       }
       const parsedMessage = parseSarifPlainTextMessage(message);

extensions/ql-vscode/src/logging.ts

@@ -1,5 +1,5 @@
 import { window as Window, OutputChannel, Progress, Disposable } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -28,9 +28,16 @@ export interface Logger {
   removeAdditionalLogLocation(location: string | undefined): void;
 
   /**
-   * The base location location where all side log files are stored.
+   * The base location where all side log files are stored.
    */
   getBaseLocation(): string | undefined;
+
+  /**
+   * Sets the location where logs are stored.
+   * @param storagePath The path where logs are stored.
+   * @param isCustomLogDirectory Whether the logs are stored in a custom, user-specified directory.
+   */
+  setLogStoragePath(storagePath: string, isCustomLogDirectory: boolean): Promise<void>;
 }
 
 export type ProgressReporter = Progress<{ message: string }>;
@@ -40,18 +47,24 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   public readonly outputChannel: OutputChannel;
   private readonly additionalLocations = new Map<string, AdditionalLogLocation>();
   private additionalLogLocationPath: string | undefined;
+  isCustomLogDirectory: boolean;
 
   constructor(private title: string) {
     super();
     this.outputChannel = Window.createOutputChannel(title);
     this.push(this.outputChannel);
+    this.isCustomLogDirectory = false;
   }
 
-  init(storagePath: string): void {
+  async setLogStoragePath(storagePath: string, isCustomLogDirectory: boolean): Promise<void> {
     this.additionalLogLocationPath = path.join(storagePath, this.title);
 
-    // clear out any old state from previous runs
-    fs.remove(this.additionalLogLocationPath);
+    this.isCustomLogDirectory = isCustomLogDirectory;
+
+    if (!this.isCustomLogDirectory) {
+      // clear out any old state from previous runs
+      await fs.remove(this.additionalLogLocationPath);
+    }
   }
 
   /**
@@ -80,7 +93,7 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
         this.outputChannel.appendLine(separator);
         this.outputChannel.appendLine(msg);
         this.outputChannel.appendLine(separator);
-        additional = new AdditionalLogLocation(logPath);
+        additional = new AdditionalLogLocation(logPath, !this.isCustomLogDirectory);
         this.additionalLocations.set(logPath, additional);
         this.track(additional);
       }
@@ -112,7 +125,7 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
 }
 
 class AdditionalLogLocation extends Disposable {
-  constructor(private location: string) {
+  constructor(private location: string, private shouldDeleteLogs: boolean) {
     super(() => { /**/ });
   }
 
@@ -128,7 +141,9 @@ class AdditionalLogLocation extends Disposable {
   }
 
   async dispose(): Promise<void> {
-    await fs.remove(this.location);
+    if (this.shouldDeleteLogs) {
+      await fs.remove(this.location);
+    }
   }
 }
 

extensions/ql-vscode/src/pure/disposable-object.ts

@@ -1,4 +1,11 @@
-import { Disposable } from 'vscode';
+
+// Avoid explicitly referencing Disposable type in vscode.
+// This file cannot have dependencies on the vscode API.
+interface Disposable {
+  dispose(): any;
+}
+
+export type DisposeHandler = (disposable: Disposable) => void;
 
 /**
  * Base class to make it easier to implement a `Disposable` that owns other disposable object.
@@ -40,21 +47,39 @@ export abstract class DisposableObject implements Disposable {
    * @param obj The object to stop tracking.
    */
   protected disposeAndStopTracking(obj: Disposable): void {
-    if (obj !== undefined) {
-      this.tracked!.delete(obj);
+    if (obj && this.tracked) {
+      this.tracked.delete(obj);
       obj.dispose();
     }
   }
 
-  public dispose() {
+  /**
+   * Dispose this object and all contained objects
+   *
+   * @param disposeHandler An optional dispose handler that gets
+   *      passed each element to dispose. The dispose handler
+   *      can choose how (and if) to dispose the object. The
+   *      primary usage is for tests that should not dispose
+   *      all items of a disposable.
+   */
+  public dispose(disposeHandler?: DisposeHandler) {
     if (this.tracked !== undefined) {
       for (const trackedObject of this.tracked.values()) {
-        trackedObject.dispose();
+        if (disposeHandler) {
+          disposeHandler(trackedObject);
+        } else {
+          trackedObject.dispose();
+        }
       }
       this.tracked = undefined;
     }
     while (this.disposables.length > 0) {
-      this.disposables.pop()!.dispose();
+      const disposable = this.disposables.pop()!;
+      if (disposeHandler) {
+        disposeHandler(disposable);
+      } else {
+        disposable.dispose();
+      }
     }
   }
 }

extensions/ql-vscode/src/pure/helpers-pure.ts

@@ -2,7 +2,7 @@
  * helpers-pure.ts
  * ------------
  *
- * Helper functions that don't depend on vscode and therefore can be used by the front-end and pure unit tests.
+ * Helper functions that don't depend on vscode or the CLI and therefore can be used by the front-end and pure unit tests.
  */
 
 /**

extensions/ql-vscode/src/pure/interface-types.ts

@@ -34,6 +34,7 @@ export interface QueryMetadata {
   description?: string;
   id?: string;
   kind?: string;
+  scored?: string;
 }
 
 export interface PreviousExecution {
@@ -88,6 +89,8 @@ export interface SetStateMsg {
   interpretation: undefined | Interpretation;
   database: DatabaseInfo;
   metadata?: QueryMetadata;
+  queryName: string;
+  queryPath: string;
   /**
    * Whether to keep displaying the old results while rendering the new results.
    *
@@ -116,6 +119,8 @@ export interface ShowInterpretedPageMsg {
   numPages: number;
   pageSize: number;
   resultSetNames: string[];
+  queryName: string;
+  queryPath: string;
 }
 
 /** Advance to the next or previous path no in the path viewer */
@@ -153,7 +158,8 @@ export type FromResultsViewMsg =
   | ChangeRawResultsSortMsg
   | ChangeInterpretedResultsSortMsg
   | ResultViewLoaded
-  | ChangePage;
+  | ChangePage
+  | OpenFileMsg;
 
 /**
  * Message from the results view to open a database source
@@ -165,6 +171,14 @@ export interface ViewSourceFileMsg {
   databaseUri: string;
 }
 
+/**
+ * Message from the results view to open a file in an editor.
+ */
+export interface OpenFileMsg {
+  t: 'openFile';
+  /* Full path to the file to open. */
+  filePath: string;
+}
 
 /**
  * Message from the results view to toggle the display of

extensions/ql-vscode/src/pure/messages.ts

@@ -150,6 +150,11 @@ export interface CompilationOptions {
    * Whether to disable toString values in the results.
    */
   noComputeToString: boolean;
+  /**
+   * Whether to ensure that elements that do not have a displayString
+   * get reported anyway. Useful for universal compilation options.
+   */
+  computeDefaultStrings: boolean;
 }
 
 /**
@@ -257,7 +262,7 @@ export interface CompilationTarget {
   /**
    * Compile as a normal query
    */
-  query?: {};
+  query?: Record<string, never>;
   /**
    * Compile as a quick evaluation
    */
@@ -380,8 +385,8 @@ export namespace ResultColumnKind {
    */
   export const BOOLEAN = 3;
   /**
- * A column of type `date`
- */
+   * A column of type `date`
+   */
   export const DATE = 4;
   /**
    * A column of a non-primitive type
@@ -401,6 +406,25 @@ export interface CompileUpgradeParams {
    * A directory to store parts of the compiled upgrade
    */
   upgradeTempDir: string;
+  /**
+   * Enable single file upgrades, set to true to allow
+   * using single file upgrades.
+   */
+  singleFileUpgrades: true;
+}
+
+/**
+ * Parameters for compiling an upgrade.
+ */
+export interface CompileUpgradeSequenceParams {
+  /**
+   * The sequence of upgrades to compile
+   */
+  upgradePaths: string[];
+  /**
+   * A directory to store parts of the compiled upgrade
+   */
+  upgradeTempDir: string;
 }
 
 /**
@@ -450,6 +474,19 @@ export interface CompileUpgradeResult {
    */
   error?: string;
 }
+
+export interface CompileUpgradeSequenceResult {
+  /**
+   * The compiled upgrades as a single file.
+   */
+  compiledUpgrade?: string;
+  /**
+   * Any errors that occurred when checking the scripts.
+   */
+  error?: string;
+}
+
+
 /**
  * A description of a upgrade process
  */
@@ -487,10 +524,13 @@ export interface UpgradeDescription {
   newSha: string;
 }
 
+
+export type CompiledUpgrades = MultiFileCompiledUpgrades | SingleFileCompiledUpgrades
+
 /**
- * A compiled upgrade.
+ * The parts shared by all compiled upgrades
  */
-export interface CompiledUpgrades {
+interface CompiledUpgradesBase {
   /**
   * The initial sha of the dbscheme to upgrade from
   */
@@ -499,14 +539,46 @@ export interface CompiledUpgrades {
    * The path to the new dataset statistics
    */
   newStatsPath: string;
+  /**
+   * The sha of the target dataset.
+   */
+  targetSha: string;
+}
+
+
+/**
+ * A compiled upgrade.
+ * The upgrade is spread among multiple files.
+ */
+interface MultiFileCompiledUpgrades extends CompiledUpgradesBase {
+  /**
+   * The path to the new dataset dbscheme
+   */
+  newDbscheme: string;
   /**
    * The steps in the upgrade path
    */
   scripts: CompiledUpgradeScript[];
   /**
-   * The sha of the target dataset.
+   * Will never exist in an old result
    */
-  targetSha: string;
+  compiledUpgradeFile?: never;
+}
+
+
+/**
+ * A compiled upgrade.
+ * The upgrade is in a single file.
+ */
+export interface SingleFileCompiledUpgrades extends CompiledUpgradesBase {
+  /**
+   * The steps in the upgrade path
+   */
+  descriptions: UpgradeDescription[];
+  /**
+   * A path to a file containing the upgrade
+   */
+  compiledUpgradeFile: string;
 }
 
 /**
@@ -651,6 +723,10 @@ export interface QueryToRun {
    * A uri pointing to the qlo to run.
    */
   qlo: string;
+  /**
+   * A uri pointing to the compiled upgrade file.
+   */
+  compiledUpgrade?: string;
   /**
    * The path where we should save this queries results
    */
@@ -750,7 +826,7 @@ export interface ResultSet {
 /**
  * The type returned when the evaluation is complete
  */
-export type EvaluationComplete = {};
+export type EvaluationComplete = Record<string, never>;
 
 /**
  * The result of a single query
@@ -837,7 +913,6 @@ export interface RunUpgradeParams {
   toRun: CompiledUpgrades;
 }
 
-
 /**
  * The result of running an upgrade
  */
@@ -857,6 +932,21 @@ export interface RunUpgradeResult {
   finalSha: string;
 }
 
+export interface RegisterDatabasesParams {
+  databases: Dataset[];
+}
+
+export interface DeregisterDatabasesParams {
+  databases: Dataset[];
+}
+
+export type RegisterDatabasesResult = {
+  registeredDatabases: Dataset[];
+};
+
+export type DeregisterDatabasesResult = {
+  registeredDatabases: Dataset[];
+};
 
 /**
  * Type for any action that could have progress messages.
@@ -913,7 +1003,10 @@ export const checkUpgrade = new rpc.RequestType<WithProgressId<UpgradeParams>, C
  * Compile an upgrade script to upgrade a dataset.
  */
 export const compileUpgrade = new rpc.RequestType<WithProgressId<CompileUpgradeParams>, CompileUpgradeResult, void, void>('compilation/compileUpgrade');
-
+/**
+ * Compile an upgrade script to upgrade a dataset.
+ */
+export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<CompileUpgradeSequenceParams>, CompileUpgradeSequenceResult, void, void>('compilation/compileUpgradeSequence');
 
 /**
  * Clear the cache of a dataset
@@ -934,6 +1027,20 @@ export const runQueries = new rpc.RequestType<WithProgressId<EvaluateQueriesPara
  */
 export const runUpgrade = new rpc.RequestType<WithProgressId<RunUpgradeParams>, RunUpgradeResult, void, void>('evaluation/runUpgrade');
 
+export const registerDatabases = new rpc.RequestType<
+  WithProgressId<RegisterDatabasesParams>,
+  RegisterDatabasesResult,
+  void,
+  void
+>('evaluation/registerDatabases');
+
+export const deregisterDatabases = new rpc.RequestType<
+  WithProgressId<DeregisterDatabasesParams>,
+  DeregisterDatabasesResult,
+  void,
+  void
+>('evaluation/deregisterDatabases');
+
 /**
  * Request returned to the client to notify completion of a query.
  * The full runQueries job is completed when all queries are acknowledged.

extensions/ql-vscode/src/qltest-discovery.ts

@@ -3,6 +3,7 @@ import { Discovery } from './discovery';
 import { EventEmitter, Event, Uri, RelativePattern, WorkspaceFolder, env } from 'vscode';
 import { MultiFileSystemWatcher } from './vscode-utils/multi-file-system-watcher';
 import { CodeQLCliServer } from './cli';
+import * as fs from 'fs-extra';
 
 /**
  * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`.
@@ -166,10 +167,12 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
   protected update(results: QLTestDiscoveryResults): void {
     this._testDirectory = results.testDirectory;
 
-    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
     this.watcher.clear();
+    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
     this.watcher.addWatch(new RelativePattern(results.watchPath, '**/*.{ql,qlref}'));
-    this._onDidChangeTests.fire();
+    // need to explicitly watch for changes to directories themselves.
+    this.watcher.addWatch(new RelativePattern(results.watchPath, '**/'));
+    this._onDidChangeTests.fire(undefined);
   }
 
   /**
@@ -180,19 +183,21 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
   private async discoverTests(): Promise<QLTestDirectory> {
     const fullPath = this.workspaceFolder.uri.fsPath;
     const name = this.workspaceFolder.name;
-    const resolvedTests = (await this.cliServer.resolveTests(fullPath))
-      .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
-
     const rootDirectory = new QLTestDirectory(fullPath, name);
-    for (const testPath of resolvedTests) {
-      const relativePath = path.normalize(path.relative(fullPath, testPath));
-      const dirName = path.dirname(relativePath);
-      const parentDirectory = rootDirectory.createDirectory(dirName);
-      parentDirectory.addChild(new QLTestFile(testPath, path.basename(testPath)));
+
+    // Don't try discovery on workspace folders that don't exist on the filesystem
+    if ((await fs.pathExists(fullPath))) {
+      const resolvedTests = (await this.cliServer.resolveTests(fullPath))
+        .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
+      for (const testPath of resolvedTests) {
+        const relativePath = path.normalize(path.relative(fullPath, testPath));
+        const dirName = path.dirname(relativePath);
+        const parentDirectory = rootDirectory.createDirectory(dirName);
+        parentDirectory.addChild(new QLTestFile(testPath, path.basename(testPath)));
+      }
+
+      rootDirectory.finish();
     }
-
-    rootDirectory.finish();
-
     return rootDirectory;
   }
 

extensions/ql-vscode/src/query-history.ts

@@ -1,14 +1,21 @@
 import * as path from 'path';
 import * as vscode from 'vscode';
-import { window as Window } from 'vscode';
+import { window as Window, env } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
-import * as helpers from './helpers';
+import {
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+  showAndLogWarningMessage,
+  showBinaryChoiceDialog
+} from './helpers';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
+import { commandRunner } from './commandRunner';
+import { assertNever } from './pure/helpers-pure';
 
 /**
  * query-history.ts
@@ -52,23 +59,23 @@ const SHOW_QUERY_TEXT_QUICK_EVAL_MSG = `\
  */
 const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
 
-interface QueryHistoryDataProvider extends vscode.TreeDataProvider<CompletedQuery> {
-  updateTreeItemContextValue(element: CompletedQuery): Promise<void>;
+enum SortOrder {
+  NameAsc = 'NameAsc',
+  NameDesc = 'NameDesc',
+  DateAsc = 'DateAsc',
+  DateDesc = 'DateDesc',
+  CountAsc = 'CountAsc',
+  CountDesc = 'CountDesc',
 }
 
 /**
  * Tree data provider for the query history view.
  */
-class HistoryTreeDataProvider implements QueryHistoryDataProvider {
-  /**
-   * XXX: This idiom for how to get a `.fire()`-able event emitter was
-   * cargo culted from another vscode extension. It seems rather
-   * involved and I hope there's something better that can be done
-   * instead.
-   */
-  private _onDidChangeTreeData: vscode.EventEmitter<
-    CompletedQuery | undefined
-  > = new vscode.EventEmitter<CompletedQuery | undefined>();
+export class HistoryTreeDataProvider extends DisposableObject {
+  private _sortOrder = SortOrder.DateAsc;
+
+  private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
+
   readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
     ._onDidChangeTreeData.event;
 
@@ -82,49 +89,58 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
   private current: CompletedQuery | undefined;
 
   constructor(extensionPath: string) {
+    super();
     this.failedIconPath = path.join(
       extensionPath,
       FAILED_QUERY_HISTORY_ITEM_ICON
     );
   }
 
-  async updateTreeItemContextValue(element: CompletedQuery): Promise<void> {
-    // Mark this query history item according to whether it has a
-    // SARIF file so that we can make context menu items conditionally
-    // available.
-    const hasResults = await element.query.hasInterpretedResults();
-    element.treeItem!.contextValue = hasResults
-      ? 'interpretedResultsItem'
-      : 'rawResultsItem';
-    this.refresh();
-  }
-
   async getTreeItem(element: CompletedQuery): Promise<vscode.TreeItem> {
-    if (element.treeItem !== undefined)
-      return element.treeItem;
+    const treeItem = new vscode.TreeItem(element.toString());
 
-    const it = new vscode.TreeItem(element.toString());
-
-    it.command = {
+    treeItem.command = {
       title: 'Query History Item',
       command: 'codeQLQueryHistory.itemClicked',
       arguments: [element],
     };
 
-    element.treeItem = it;
-    this.updateTreeItemContextValue(element);
+    // Mark this query history item according to whether it has a
+    // SARIF file so that we can make context menu items conditionally
+    // available.
+    const hasResults = await element.query.hasInterpretedResults();
+    treeItem.contextValue = hasResults
+      ? 'interpretedResultsItem'
+      : 'rawResultsItem';
 
     if (!element.didRunSuccessfully) {
-      it.iconPath = this.failedIconPath;
+      treeItem.iconPath = this.failedIconPath;
     }
 
-    return it;
+    return treeItem;
   }
 
   getChildren(
     element?: CompletedQuery
   ): vscode.ProviderResult<CompletedQuery[]> {
-    return element ? [] : this.history;
+    return element ? [] : this.history.sort((q1, q2) => {
+      switch (this.sortOrder) {
+        case SortOrder.NameAsc:
+          return q1.toString().localeCompare(q2.toString(), env.language);
+        case SortOrder.NameDesc:
+          return q2.toString().localeCompare(q1.toString(), env.language);
+        case SortOrder.DateAsc:
+          return q1.date.getTime() - q2.date.getTime();
+        case SortOrder.DateDesc:
+          return q2.date.getTime() - q1.date.getTime();
+        case SortOrder.CountAsc:
+          return q1.resultCount - q2.resultCount;
+        case SortOrder.CountDesc:
+          return q2.resultCount - q1.resultCount;
+        default:
+          assertNever(this.sortOrder);
+      }
+    });
   }
 
   getParent(_element: CompletedQuery): vscode.ProviderResult<CompletedQuery> {
@@ -135,7 +151,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
     return this.current;
   }
 
-  push(item: CompletedQuery): void {
+  pushQuery(item: CompletedQuery): void {
     this.current = item;
     this.history.push(item);
     this.refresh();
@@ -163,13 +179,22 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
     return this.history;
   }
 
-  refresh() {
-    this._onDidChangeTreeData.fire(undefined);
+  refresh(completedQuery?: CompletedQuery) {
+    this._onDidChangeTreeData.fire(completedQuery);
   }
 
   find(queryId: number): CompletedQuery | undefined {
     return this.allHistory.find((query) => query.query.queryID === queryId);
   }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire(undefined);
+  }
 }
 
 /**
@@ -178,6 +203,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
  */
 const DOUBLE_CLICK_TIME = 500;
 
+const NO_QUERY_SELECTED = 'No query selected. Select a query history item you have already run and try again.';
 export class QueryHistoryManager extends DisposableObject {
   treeDataProvider: HistoryTreeDataProvider;
   treeView: vscode.TreeView<CompletedQuery>;
@@ -204,6 +230,7 @@ export class QueryHistoryManager extends DisposableObject {
       canSelectMany: true,
     });
     this.push(this.treeView);
+    this.push(treeDataProvider);
 
     // Lazily update the tree view selection due to limitations of TreeView API (see
     // `updateTreeViewSelectionIfVisible` doc for details)
@@ -222,57 +249,81 @@ export class QueryHistoryManager extends DisposableObject {
       })
     );
 
-    logger.log('Registering query history panel commands.');
+    void logger.log('Registering query history panel commands.');
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.openQuery',
         this.handleOpenQuery.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.removeHistoryItem',
         this.handleRemoveHistoryItem.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
+        'codeQLQueryHistory.sortByName',
+        this.handleSortByName.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByDate',
+        this.handleSortByDate.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByCount',
+        this.handleSortByCount.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
         'codeQLQueryHistory.setLabel',
         this.handleSetLabel.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.compareWith',
         this.handleCompareWith.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.showQueryLog',
         this.handleShowQueryLog.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.showQueryText',
         this.handleShowQueryText.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
-        'codeQLQueryHistory.viewSarif',
-        this.handleViewSarif.bind(this)
+      commandRunner(
+        'codeQLQueryHistory.viewCsvResults',
+        this.handleViewCsvResults.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
+        'codeQLQueryHistory.viewSarifResults',
+        this.handleViewSarifResults.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
         'codeQLQueryHistory.viewDil',
         this.handleViewDil.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.itemClicked',
         async (item: CompletedQuery) => {
           return this.handleItemClicked(item, [item]);
@@ -315,6 +366,10 @@ export class QueryHistoryManager extends DisposableObject {
       return;
     }
 
+    if (!finalSingleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
     const textDocument = await vscode.workspace.openTextDocument(
       vscode.Uri.file(finalSingleItem.query.program.queryPath)
     );
@@ -347,11 +402,35 @@ export class QueryHistoryManager extends DisposableObject {
     });
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
-      this.treeView.reveal(current);
+      await this.treeView.reveal(current);
       await this.invokeCallbackOn(current);
     }
   }
 
+  async handleSortByName() {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  }
+
+  async handleSortByDate() {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAsc;
+    }
+  }
+
+  async handleSortByCount() {
+    if (this.treeDataProvider.sortOrder === SortOrder.CountAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.CountDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.CountAsc;
+    }
+  }
+
   async handleSetLabel(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
@@ -367,11 +446,14 @@ export class QueryHistoryManager extends DisposableObject {
     });
     // undefined response means the user cancelled the dialog; don't change anything
     if (response !== undefined) {
-      if (response === '')
-        // Interpret empty string response as 'go back to using default'
-        singleItem.options.label = undefined;
-      else singleItem.options.label = response;
-      this.treeDataProvider.refresh();
+      // Interpret empty string response as 'go back to using default'
+      singleItem.options.label = response === '' ? undefined : response;
+      if (this.treeDataProvider.sortOrder === SortOrder.NameAsc ||
+        this.treeDataProvider.sortOrder === SortOrder.NameDesc) {
+        this.treeDataProvider.refresh();
+      } else {
+        this.treeDataProvider.refresh(singleItem);
+      }
     }
   }
 
@@ -388,10 +470,10 @@ export class QueryHistoryManager extends DisposableObject {
       const to = await this.findOtherQueryToCompare(from, multiSelect);
 
       if (from && to) {
-        this.doCompareCallback(from, to);
+        await this.doCompareCallback(from, to);
       }
     } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
     }
   }
 
@@ -403,6 +485,11 @@ export class QueryHistoryManager extends DisposableObject {
     if (!this.assertSingleQuery(finalMultiSelect)) {
       return;
     }
+
+    if (!finalSingleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
     this.treeDataProvider.setCurrentItem(finalSingleItem);
 
     const now = new Date();
@@ -433,7 +520,7 @@ export class QueryHistoryManager extends DisposableObject {
     if (singleItem.logFileLocation) {
       await this.tryOpenExternalFile(singleItem.logFileLocation);
     } else {
-      helpers.showAndLogWarningMessage('No log file available');
+      void showAndLogWarningMessage('No log file available');
     }
   }
 
@@ -445,6 +532,10 @@ export class QueryHistoryManager extends DisposableObject {
       return;
     }
 
+    if (!singleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
     const queryName = singleItem.queryName.endsWith('.ql')
       ? singleItem.queryName
       : singleItem.queryName + '.ql';
@@ -459,7 +550,7 @@ export class QueryHistoryManager extends DisposableObject {
     await vscode.window.showTextDocument(doc, { preview: false });
   }
 
-  async handleViewSarif(
+  async handleViewSarifResults(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
   ) {
@@ -474,12 +565,25 @@ export class QueryHistoryManager extends DisposableObject {
       );
     } else {
       const label = singleItem.getLabel();
-      helpers.showAndLogInformationMessage(
+      void showAndLogInformationMessage(
         `Query ${label} has no interpreted results.`
       );
     }
   }
 
+  async handleViewCsvResults(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    await this.tryOpenExternalFile(
+      await singleItem.query.ensureCsvProduced(this.qs)
+    );
+  }
+
   async handleViewDil(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[],
@@ -511,13 +615,16 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
-  addQuery(info: QueryWithResults): CompletedQuery {
+  buildCompletedQuery(info: QueryWithResults): CompletedQuery {
     const item = new CompletedQuery(info, this.queryHistoryConfigListener);
-    this.treeDataProvider.push(item);
-    this.updateTreeViewSelectionIfVisible();
     return item;
   }
 
+  addCompletedQuery(item: CompletedQuery) {
+    this.treeDataProvider.pushQuery(item);
+    this.updateTreeViewSelectionIfVisible();
+  }
+
   find(queryId: number): CompletedQuery | undefined {
     return this.treeDataProvider.find(queryId);
   }
@@ -537,7 +644,7 @@ export class QueryHistoryManager extends DisposableObject {
         // We must fire the onDidChangeTreeData event to ensure the current element can be selected
         // using `reveal` if the tree view was not visible when the current element was added.
         this.treeDataProvider.refresh();
-        this.treeView.reveal(current);
+        void this.treeView.reveal(current);
       }
     }
   }
@@ -553,7 +660,7 @@ export class QueryHistoryManager extends DisposableObject {
         ) ||
         e.message.includes('too large to open')
       ) {
-        const res = await helpers.showBinaryChoiceDialog(
+        const res = await showBinaryChoiceDialog(
           `VS Code does not allow extensions to open files >50MB. This file
 exceeds that limit. Do you want to open it outside of VS Code?
 
@@ -564,13 +671,13 @@ the file in the file explorer and dragging it into the workspace.`
           try {
             await vscode.commands.executeCommand('revealFileInOS', uri);
           } catch (e) {
-            helpers.showAndLogErrorMessage(e.message);
+            void showAndLogErrorMessage(e.message);
           }
         }
       } else {
-        helpers.showAndLogErrorMessage(`Could not open file ${fileLocation}`);
-        logger.log(e.message);
-        logger.log(e.stack);
+        void showAndLogErrorMessage(`Could not open file ${fileLocation}`);
+        void logger.log(e.message);
+        void logger.log(e.stack);
       }
     }
   }
@@ -622,7 +729,7 @@ the file in the file explorer and dragging it into the workspace.`
 
   private assertSingleQuery(multiSelect: CompletedQuery[] = [], message = 'Please select a single query.') {
     if (multiSelect.length > 1) {
-      helpers.showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         message
       );
       return false;
@@ -686,7 +793,7 @@ the file in the file explorer and dragging it into the workspace.`
     };
   }
 
-  async updateTreeItemContextValue(element: CompletedQuery): Promise<void> {
-    this.treeDataProvider.updateTreeItemContextValue(element);
+  async refreshTreeView(completedQuery: CompletedQuery): Promise<void> {
+    this.treeDataProvider.refresh(completedQuery);
   }
 }

extensions/ql-vscode/src/query-results.ts

@@ -1,4 +1,4 @@
-import { env, TreeItem } from 'vscode';
+import { env } from 'vscode';
 
 import { QueryWithResults, tmpDir, QueryInfo } from './run-queries';
 import * as messages from './pure/messages';
@@ -11,13 +11,14 @@ import { QueryHistoryConfig } from './config';
 import { QueryHistoryItemOptions } from './query-history';
 
 export class CompletedQuery implements QueryWithResults {
+  readonly date: Date;
   readonly time: string;
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
   readonly database: DatabaseInfo;
   readonly logFileLocation?: string;
   options: QueryHistoryItemOptions;
-  treeItem?: TreeItem;
+  resultCount: number;
   dispose: () => void;
 
   /**
@@ -45,8 +46,14 @@ export class CompletedQuery implements QueryWithResults {
     this.options = evaluation.options;
     this.dispose = evaluation.dispose;
 
-    this.time = new Date().toLocaleString(env.language);
+    this.date = new Date();
+    this.time = this.date.toLocaleString(env.language);
     this.sortedResultsInfo = new Map();
+    this.resultCount = 0;
+  }
+
+  setResultCount(value: number) {
+    this.resultCount = value;
   }
 
   get databaseName(): string {
@@ -81,11 +88,12 @@ export class CompletedQuery implements QueryWithResults {
   }
 
   interpolate(template: string): string {
-    const { databaseName, queryName, time, statusString } = this;
+    const { databaseName, queryName, time, resultCount, statusString } = this;
     const replacements: { [k: string]: string } = {
       t: time,
       q: queryName,
       d: databaseName,
+      r: resultCount.toString(),
       s: statusString,
       '%': '%',
     };
@@ -171,17 +179,20 @@ export async function interpretResults(
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }
+  return await server.interpretBqrs(ensureMetadataIsComplete(metadata), resultsPath, interpretedResultsPath, sourceInfo);
+}
+
+export function ensureMetadataIsComplete(metadata: QueryMetadata | undefined) {
   if (metadata === undefined) {
     throw new Error('Can\'t interpret results without query metadata');
   }
-  let { kind, id } = metadata;
-  if (kind === undefined) {
+  if (metadata.kind === undefined) {
     throw new Error('Can\'t interpret results without query metadata including kind');
   }
-  if (id === undefined) {
+  if (metadata.id === undefined) {
     // Interpretation per se doesn't really require an id, but the
     // SARIF format does, so in the absence of one, we use a dummy id.
-    id = 'dummy-id';
+    metadata.id = 'dummy-id';
   }
-  return await server.interpretBqrs({ kind, id }, resultsPath, interpretedResultsPath, sourceInfo);
+  return metadata;
 }

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,16 +1,20 @@
 import * as cp from 'child_process';
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { Disposable } from 'vscode';
-import { CancellationToken, createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
+import { DisposableObject } from './pure/disposable-object';
+import { Disposable, CancellationToken, commands } from 'vscode';
+import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
 import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
 import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
 import * as messages from './pure/messages';
+import { ProgressCallback, ProgressTask } from './commandRunner';
+import * as fs from 'fs-extra';
+import * as helpers from './helpers';
 
 type ServerOpts = {
   logger: Logger;
+  contextStoragePath: string;
 }
 
 /** A running query server process and its associated message connection. */
@@ -26,7 +30,7 @@ class ServerProcess implements Disposable {
   }
 
   dispose(): void {
-    this.logger.log('Stopping query server...');
+    void this.logger.log('Stopping query server...');
     this.connection.dispose();
     this.child.stdin!.end();
     this.child.stderr!.destroy();
@@ -34,7 +38,7 @@ class ServerProcess implements Disposable {
 
     // On Windows, we usually have to terminate the process before closing its stdout.
     this.child.stdout!.destroy();
-    this.logger.log('Stopped query server.');
+    void this.logger.log('Stopped query server.');
   }
 }
 
@@ -47,22 +51,36 @@ type WithProgressReporting = (task: (progress: ProgressReporter, token: Cancella
  * to restart it (which disposes the existing process and starts a new one).
  */
 export class QueryServerClient extends DisposableObject {
+
   serverProcess?: ServerProcess;
   evaluationResultCallbacks: { [key: number]: (res: EvaluationResult) => void };
   progressCallbacks: { [key: number]: ((res: ProgressMessage) => void) | undefined };
   nextCallback: number;
   nextProgress: number;
   withProgressReporting: WithProgressReporting;
+
+  private readonly queryServerStartListeners = [] as ProgressTask<void>[];
+
+  // Can't use standard vscode EventEmitter here since they do not cause the calling
+  // function to fail if one of the event handlers fail. This is something that
+  // we need here.
+  readonly onDidStartQueryServer = (e: ProgressTask<void>) => {
+    this.queryServerStartListeners.push(e);
+  }
+
   public activeQueryName: string | undefined;
 
-  constructor(readonly config: QueryServerConfig, readonly cliServer: cli.CodeQLCliServer, readonly opts: ServerOpts, withProgressReporting: WithProgressReporting) {
+  constructor(
+    readonly config: QueryServerConfig,
+    readonly cliServer: cli.CodeQLCliServer,
+    readonly opts: ServerOpts,
+    withProgressReporting: WithProgressReporting
+  ) {
     super();
     // When the query server configuration changes, restart the query server.
     if (config.onDidChangeConfiguration !== undefined) {
-      this.push(config.onDidChangeConfiguration(async () => {
-        this.logger.log('Restarting query server due to configuration changes...');
-        await this.restartQueryServer();
-      }, this));
+      this.push(config.onDidChangeConfiguration(() =>
+        commands.executeCommand('codeQL.restartQueryServer')));
     }
     this.withProgressReporting = withProgressReporting;
     this.nextCallback = 0;
@@ -71,6 +89,26 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
+  async initLogger() {
+    let storagePath = this.opts.contextStoragePath;
+    let isCustomLogDirectory = false;
+    if (this.config.customLogDirectory) {
+      try {
+        if (!(await fs.pathExists(this.config.customLogDirectory))) {
+          await fs.mkdir(this.config.customLogDirectory);
+        }
+        void this.logger.log(`Saving query server logs to user-specified directory: ${this.config.customLogDirectory}.`);
+        storagePath = this.config.customLogDirectory;
+        isCustomLogDirectory = true;
+      } catch (e) {
+        void helpers.showAndLogErrorMessage(`${this.config.customLogDirectory} is not a valid directory. Logs will be stored in a temporary workspace directory instead.`);
+      }
+    }
+
+    await this.logger.setLogStoragePath(storagePath, isCustomLogDirectory);
+
+  }
+
   get logger(): Logger {
     return this.opts.logger;
   }
@@ -80,14 +118,24 @@ export class QueryServerClient extends DisposableObject {
     if (this.serverProcess !== undefined) {
       this.disposeAndStopTracking(this.serverProcess);
     } else {
-      this.logger.log('No server process to be stopped.');
+      void this.logger.log('No server process to be stopped.');
     }
   }
 
   /** Restarts the query server by disposing of the current server process and then starting a new one. */
-  async restartQueryServer(): Promise<void> {
+  async restartQueryServer(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<void> {
     this.stopQueryServer();
     await this.startQueryServer();
+
+    // Ensure we await all responses from event handlers so that
+    // errors can be properly reported to the user.
+    await Promise.all(this.queryServerStartListeners.map(handler => handler(
+      progress,
+      token
+    )));
   }
 
   showLog(): void {
@@ -102,8 +150,23 @@ export class QueryServerClient extends DisposableObject {
 
   /** Starts a new query server process, sending progress messages to the given reporter. */
   private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
+    await this.initLogger();
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
+
+    if (this.config.saveCache) {
+      args.push('--save-cache');
+    }
+
+    if (this.config.cacheSize > 0) {
+      args.push('--max-disk-cache');
+      args.push(this.config.cacheSize.toString());
+    }
+
+    if (await this.cliServer.cliConstraints.supportsDatabaseRegistration()) {
+      args.push('--require-db-registration');
+    }
+
     if (this.config.debug) {
       args.push('--debug', '--tuple-counting');
     }
@@ -129,9 +192,8 @@ export class QueryServerClient extends DisposableObject {
     const connection = createMessageConnection(child.stdout, child.stdin);
     connection.onRequest(completeQuery, res => {
       if (!(res.runId in this.evaluationResultCallbacks)) {
-        this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
-      }
-      else {
+        void this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
+      } else {
         const baseLocation = this.logger.getBaseLocation();
         if (baseLocation && this.activeQueryName) {
           res.logFileLocation = path.join(baseLocation, this.activeQueryName);
@@ -146,7 +208,7 @@ export class QueryServerClient extends DisposableObject {
         callback(res);
       }
     });
-    this.serverProcess = new ServerProcess(child, connection, this.opts.logger);
+    this.serverProcess = new ServerProcess(child, connection, this.logger);
     // Ensure the server process is disposed together with this client.
     this.track(this.serverProcess);
     connection.listen();

extensions/ql-vscode/src/quick-query.ts

@@ -1,48 +1,59 @@
 import * as fs from 'fs-extra';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
-import { CancellationToken, ExtensionContext, window as Window, workspace, Uri } from 'vscode';
+import {
+  CancellationToken,
+  ExtensionContext,
+  window as Window,
+  workspace,
+  Uri
+} from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import { CodeQLCliServer } from './cli';
 import { DatabaseUI } from './databases-ui';
-import * as helpers from './helpers';
-import { logger } from './logging';
+import {
+  getInitialQueryContents,
+  getPrimaryDbscheme,
+  getQlPackForDbscheme,
+  showBinaryChoiceDialog,
+} from './helpers';
+import {
+  ProgressCallback,
+  UserCancellationException
+} from './commandRunner';
 
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
 const QUICK_QUERY_WORKSPACE_FOLDER_NAME = 'Quick Queries';
+const QLPACK_FILE_HEADER = '# This is an automatically generated file.\n\n';
 
 export function isQuickQueryPath(queryPath: string): boolean {
   return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
 }
 
-/**
- * `getBaseText` heuristically returns an appropriate import statement
- * prelude based on the filename of the dbscheme file given. TODO: add
- * a 'default import' field to the qlpack itself, and use that.
- */
-function getBaseText(dbschemeBase: string) {
-  if (dbschemeBase == 'semmlecode.javascript.dbscheme') return 'import javascript\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.cpp.dbscheme') return 'import cpp\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.dbscheme') return 'import java\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.python.dbscheme') return 'import python\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.csharp.dbscheme') return 'import csharp\n\nselect ""';
-  if (dbschemeBase == 'go.dbscheme') return 'import go\n\nselect ""';
-  return 'select ""';
-}
-
-function getQuickQueriesDir(ctx: ExtensionContext): string {
+async function getQuickQueriesDir(ctx: ExtensionContext): Promise<string> {
   const storagePath = ctx.storagePath;
   if (storagePath === undefined) {
     throw new Error('Workspace storage path is undefined');
   }
   const queriesPath = path.join(storagePath, QUICK_QUERIES_DIR_NAME);
-  fs.ensureDir(queriesPath, { mode: 0o700 });
+  await fs.ensureDir(queriesPath, { mode: 0o700 });
   return queriesPath;
 }
 
+function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
+  workspace.updateWorkspaceFolders(
+    index,
+    len,
+    { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
+  );
+}
 
-
+function findExistingQuickQueryEditor() {
+  return Window.visibleTextEditors.find(editor =>
+    path.basename(editor.document.uri.fsPath) === QUICK_QUERY_QUERY_NAME
+  );
+}
 
 /**
  * Show a buffer the user can enter a simple query into.
@@ -51,30 +62,22 @@ export async function displayQuickQuery(
   ctx: ExtensionContext,
   cliServer: CodeQLCliServer,
   databaseUI: DatabaseUI,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken
 ) {
 
-  function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
-    workspace.updateWorkspaceFolders(
-      index,
-      len,
-      { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
-    );
-  }
-
   try {
-    const workspaceFolders = workspace.workspaceFolders || [];
-    const queriesDir = await getQuickQueriesDir(ctx);
-
     // If there is already a quick query open, don't clobber it, just
     // show it.
-    const existing = workspace.textDocuments.find(doc => path.basename(doc.uri.fsPath) === QUICK_QUERY_QUERY_NAME);
-    if (existing !== undefined) {
-      Window.showTextDocument(existing);
+    const existing = findExistingQuickQueryEditor();
+    if (existing) {
+      await Window.showTextDocument(existing.document);
       return;
     }
 
+    const workspaceFolders = workspace.workspaceFolders || [];
+    const queriesDir = await getQuickQueriesDir(ctx);
+
     // We need to have a multi-root workspace to make quick query work
     // at all. Changing the workspace from single-root to multi-root
     // causes a restart of the whole extension host environment, so we
@@ -85,7 +88,7 @@ export async function displayQuickQuery(
     // being undefined) just let the user know that they're in for a
     // restart.
     if (workspace.workspaceFile === undefined) {
-      const makeMultiRoot = await helpers.showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
+      const makeMultiRoot = await showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
       if (makeMultiRoot) {
         updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
       }
@@ -93,10 +96,11 @@ export async function displayQuickQuery(
     }
 
     const index = workspaceFolders.findIndex(folder => folder.name === QUICK_QUERY_WORKSPACE_FOLDER_NAME);
-    if (index === -1)
+    if (index === -1) {
       updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
-    else
+    } else {
       updateQuickQueryDir(queriesDir, index, 1);
+    }
 
     // We're going to infer which qlpack to use from the current database
     const dbItem = await databaseUI.getDatabaseItem(progress, token);
@@ -105,31 +109,40 @@ export async function displayQuickQuery(
     }
 
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
-    const { qlpack, dbscheme } = await helpers.resolveDatasetFolder(cliServer, datasetFolder);
-    const quickQueryQlpackYaml: any = {
-      name: 'quick-query',
-      version: '1.0.0',
-      libraryPathDependencies: [qlpack]
-    };
-
-    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const dbscheme = await getPrimaryDbscheme(datasetFolder);
+    const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
     const qlPackFile = path.join(queriesDir, 'qlpack.yml');
-    await fs.writeFile(qlFile, getBaseText(path.basename(dbscheme)), 'utf8');
-    await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8');
-    Window.showTextDocument(await workspace.openTextDocument(qlFile));
-  }
+    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const shouldRewrite = await checkShouldRewrite(qlPackFile, qlpack);
 
-  // TODO: clean up error handling for top-level commands like this
-  catch (e) {
-    if (e instanceof helpers.UserCancellationException) {
-      logger.log(e.message);
+    // Only rewrite the qlpack file if the database has changed
+    if (shouldRewrite) {
+      const quickQueryQlpackYaml: any = {
+        name: 'quick-query',
+        version: '1.0.0',
+        libraryPathDependencies: [qlpack]
+      };
+      await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.safeDump(quickQueryQlpackYaml), 'utf8');
     }
-    else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      logger.log(e.message);
+
+    if (shouldRewrite || !(await fs.pathExists(qlFile))) {
+      await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
     }
-    else if (e instanceof Error)
-      helpers.showAndLogErrorMessage(e.message);
-    else
+
+    await Window.showTextDocument(await workspace.openTextDocument(qlFile));
+  } catch (e) {
+    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      throw new UserCancellationException(e.message);
+    } else {
       throw e;
+    }
   }
 }
+
+async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
+  if (!(await fs.pathExists(qlPackFile))) {
+    return true;
+  }
+  const qlPackContents: any = yaml.safeLoad(await fs.readFile(qlPackFile, 'utf8'));
+  return qlPackContents.libraryPathDependencies?.[0] !== newDependency;
+}

extensions/ql-vscode/src/run-queries.ts

@@ -1,7 +1,7 @@
 import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import * as tmp from 'tmp';
+import * as tmp from 'tmp-promise';
 import {
   CancellationToken,
   ConfigurationTarget,
@@ -14,15 +14,17 @@ import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 
 import * as cli from './cli';
 import * as config from './config';
-import { DatabaseItem, getUpgradesDirectories } from './databases';
-import * as helpers from './helpers';
+import { DatabaseItem } from './databases';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { DatabaseInfo, QueryMetadata, ResultsPaths } from './pure/interface-types';
 import { logger } from './logging';
 import * as messages from './pure/messages';
 import { QueryHistoryItemOptions } from './query-history';
 import * as qsClient from './queryserver-client';
 import { isQuickQueryPath } from './quick-query';
-import { upgradeDatabase } from './upgrades';
+import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
+import { ensureMetadataIsComplete } from './query-results';
 
 /**
  * run-queries.ts
@@ -52,6 +54,7 @@ export class QueryInfo {
 
   readonly compiledQueryPath: string;
   readonly dilPath: string;
+  readonly csvPath: string;
   readonly resultsPaths: ResultsPaths;
   readonly dataset: Uri; // guarantee the existence of a well-defined dataset dir at this point
   readonly queryID: number;
@@ -67,6 +70,7 @@ export class QueryInfo {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
     this.dilPath = path.join(tmpDir.name, `results${this.queryID}.dil`);
+    this.csvPath = path.join(tmpDir.name, `results${this.queryID}.csv`);
     this.resultsPaths = {
       resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`),
       interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`)
@@ -79,7 +83,8 @@ export class QueryInfo {
 
   async run(
     qs: qsClient.QueryServerClient,
-    progress: helpers.ProgressCallback,
+    upgradeQlo: string | undefined,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.EvaluationResult> {
     let result: messages.EvaluationResult | null = null;
@@ -89,6 +94,7 @@ export class QueryInfo {
     const queryToRun: messages.QueryToRun = {
       resultsPath: this.resultsPaths.resultsPath,
       qlo: Uri.file(this.compiledQueryPath).toString(),
+      compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
       allowUnknownTemplates: true,
       templateValues: this.templates,
       id: callbackId,
@@ -121,7 +127,7 @@ export class QueryInfo {
 
   async compile(
     qs: qsClient.QueryServerClient,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.CompilationMessage[]> {
     let compiled: messages.CheckQueryResult | undefined;
@@ -138,6 +144,7 @@ export class QueryInfo {
           localChecking: false,
           noComputeGetUrl: false,
           noComputeToString: false,
+          computeDefaultStrings: true
         },
         extraOptions: {
           timeoutSecs: qs.config.timeoutSecs
@@ -149,7 +156,7 @@ export class QueryInfo {
 
       compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
     } finally {
-      qs.logger.log(' - - - COMPILATION DONE - - - ');
+      void qs.logger.log(' - - - COMPILATION DONE - - - ');
     }
     return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
   }
@@ -160,9 +167,17 @@ export class QueryInfo {
   async canHaveInterpretedResults(): Promise<boolean> {
     const hasMetadataFile = await this.dbItem.hasMetadataFile();
     if (!hasMetadataFile) {
-      logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
+      void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
     }
-    return hasMetadataFile;
+
+    const hasKind = !!this.metadata?.kind;
+    if (!hasKind) {
+      void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
+    }
+
+    const isTable = hasKind && this.metadata?.kind === 'table';
+
+    return hasMetadataFile && hasKind && !isTable;
   }
 
   /**
@@ -179,6 +194,13 @@ export class QueryInfo {
     return fs.pathExists(this.dilPath);
   }
 
+  /**
+   * Holds if this query already has CSV results produced
+   */
+  async hasCsv(): Promise<boolean> {
+    return fs.pathExists(this.csvPath);
+  }
+
   async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
     if (await this.hasDil()) {
       return this.dilPath;
@@ -194,8 +216,27 @@ export class QueryInfo {
     return this.dilPath;
   }
 
+  async ensureCsvProduced(qs: qsClient.QueryServerClient): Promise<string> {
+    if (await this.hasCsv()) {
+      return this.csvPath;
+    }
+
+    let sourceInfo;
+    if (this.dbItem.sourceArchive !== undefined) {
+      sourceInfo = {
+        sourceArchive: this.dbItem.sourceArchive.fsPath,
+        sourceLocationPrefix: await this.dbItem.getSourceLocationPrefix(
+          qs.cliServer
+        ),
+      };
+    }
+
+    await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
+    return this.csvPath;
+  }
 }
 
+
 export interface QueryWithResults {
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
@@ -208,7 +249,7 @@ export interface QueryWithResults {
 export async function clearCacheInDatabase(
   qs: qsClient.QueryServerClient,
   dbItem: DatabaseItem,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<messages.ClearCacheResult> {
   if (dbItem.contents === undefined) {
@@ -284,13 +325,13 @@ async function checkDbschemeCompatibility(
   cliServer: cli.CodeQLCliServer,
   qs: qsClient.QueryServerClient,
   query: QueryInfo,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<void> {
-  const searchPath = helpers.getOnDiskWorkspaceFolders();
+  const searchPath = getOnDiskWorkspaceFolders();
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
-    const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
+    const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
     const hash = async function(filename: string): Promise<string> {
       return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
     };
@@ -309,18 +350,15 @@ async function checkDbschemeCompatibility(
     const upgradableTo = await hash(finalDbscheme);
 
     if (upgradableTo != dbschemeOfLib) {
-      logger.log(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but database has scheme ${query.program.dbschemePath}, and no upgrade path found`);
-      throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace. Please try using a newer version of the query libraries.`);
+      reportNoUpgradePath(query);
     }
 
     if (upgradableTo == dbschemeOfLib &&
       dbschemeOfDb != dbschemeOfLib) {
       // Try to upgrade the database
-      await upgradeDatabase(
+      await upgradeDatabaseExplicit(
         qs,
         query.dbItem,
-        Uri.file(finalDbscheme),
-        getUpgradesDirectories(scripts),
         progress,
         token
       );
@@ -328,6 +366,42 @@ async function checkDbschemeCompatibility(
   }
 }
 
+function reportNoUpgradePath(query: QueryInfo) {
+  throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`);
+}
+
+/**
+ * Compile a non-destructive upgrade.
+ */
+async function compileNonDestructiveUpgrade(
+  qs: qsClient.QueryServerClient,
+  upgradeTemp: tmp.DirectoryResult,
+  query: QueryInfo,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<string> {
+  const searchPath = getOnDiskWorkspaceFolders();
+
+  if (!query.dbItem?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
+  }
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme);
+
+  if (!matchesTarget) {
+    reportNoUpgradePath(query);
+  }
+  const result = await compileDatabaseUpgradeSequence(qs, query.dbItem, scripts, upgradeTemp, progress, token);
+  if (result.compiledUpgrade === undefined) {
+    const error = result.error || '[no error message available]';
+    throw new Error(error);
+  }
+  // We can upgrade to the actual target
+  query.program.dbschemePath = query.queryDbscheme;
+  // We are new enough that we will always support single file upgrades.
+  return result.compiledUpgrade;
+
+}
+
 /**
  * Prompts the user to save `document` if it has unsaved changes.
  *
@@ -344,7 +418,7 @@ async function promptUserToSaveChanges(document: TextDocument): Promise<boolean>
     else {
       const yesItem = { title: 'Yes', isCloseAffordance: false };
       const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
-      const noItem = { title: 'No (run anyway)', isCloseAffordance: false };
+      const noItem = { title: 'No (run version on disk)', isCloseAffordance: false };
       const cancelItem = { title: 'Cancel', isCloseAffordance: true };
       const message = 'Query file has unsaved changes. Save now?';
       const chosenItem = await window.showInformationMessage(
@@ -363,7 +437,7 @@ async function promptUserToSaveChanges(document: TextDocument): Promise<boolean>
       }
 
       if (chosenItem === cancelItem) {
-        throw new helpers.UserCancellationException('Query run cancelled.', true);
+        throw new UserCancellationException('Query run cancelled.', true);
       }
     }
   }
@@ -425,7 +499,7 @@ export async function determineSelectedQuery(selectedResourceUri: Uri | undefine
   // then prompt the user to save it first.
   if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
     if (await promptUserToSaveChanges(editor.document)) {
-      editor.document.save();
+      void editor.document.save();
     }
   }
 
@@ -453,7 +527,7 @@ export async function compileAndRunQueryAgainstDatabase(
   db: DatabaseItem,
   quickEval: boolean,
   selectedQueryUri: Uri | undefined,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
   templates?: messages.TemplateDefinitions,
 ): Promise<QueryWithResults> {
@@ -473,10 +547,14 @@ export async function compileAndRunQueryAgainstDatabase(
   }
 
   // Get the workspace folder paths.
-  const diskWorkspaceFolders = helpers.getOnDiskWorkspaceFolders();
+  const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
   // Figure out the library path for the query.
   const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, queryPath);
 
+  if (!packConfig.dbscheme) {
+    throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
+  }
+
   // Check whether the query has an entirely different schema from the
   // database. (Queries that merely need the database to be upgraded
   // won't trigger this check)
@@ -484,7 +562,7 @@ export async function compileAndRunQueryAgainstDatabase(
   const querySchemaName = path.basename(packConfig.dbscheme);
   const dbSchemaName = path.basename(db.contents.dbSchemeUri.fsPath);
   if (querySchemaName != dbSchemaName) {
-    logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
+    void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
     throw new Error(`The query ${path.basename(queryPath)} cannot be run against the selected database: their target languages are different. Please select a different database and try again.`);
   }
 
@@ -506,71 +584,89 @@ export async function compileAndRunQueryAgainstDatabase(
     metadata = await cliServer.resolveMetadata(qlProgram.queryPath);
   } catch (e) {
     // Ignore errors and provide no metadata.
-    logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
+    void logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
   }
 
   const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
-  await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
 
-  let errors;
+  const upgradeDir = await tmp.dir({ dir: upgradesTmpDir.name, unsafeCleanup: true });
   try {
-    errors = await query.compile(qs, progress, token);
-  } catch (e) {
-    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+    let upgradeQlo;
+    if (await hasNondestructiveUpgradeCapabilities(qs)) {
+      upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, progress, token);
     } else {
-      throw e;
+      await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
     }
-  }
-
-  if (errors.length == 0) {
-    const result = await query.run(qs, progress, token);
-    if (result.resultType !== messages.QueryResultType.SUCCESS) {
-      const message = result.message || 'Failed to run query';
-      logger.log(message);
-      helpers.showAndLogErrorMessage(message);
-    }
-    return {
-      query,
-      result,
-      database: {
-        name: db.name,
-        databaseUri: db.databaseUri.toString(true)
-      },
-      options: historyItemOptions,
-      logFileLocation: result.logFileLocation,
-      dispose: () => {
-        qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+    let errors;
+    try {
+      errors = await query.compile(qs, progress, token);
+    } catch (e) {
+      if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+        return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+      } else {
+        throw e;
       }
-    };
-  } else {
-    // Error dialogs are limited in size and scrollability,
-    // so we include a general description of the problem,
-    // and direct the user to the output window for the detailed compilation messages.
-    // However we don't show quick eval errors there so we need to display them anyway.
-    qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
-
-    const formattedMessages: string[] = [];
-
-    for (const error of errors) {
-      const message = error.message || '[no error message available]';
-      const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
-      formattedMessages.push(formatted);
-      qs.logger.log(formatted);
     }
-    if (quickEval && formattedMessages.length <= 3) {
-      helpers.showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
+
+    if (errors.length === 0) {
+      const result = await query.run(qs, upgradeQlo, progress, token);
+      if (result.resultType !== messages.QueryResultType.SUCCESS) {
+        const message = result.message || 'Failed to run query';
+        void logger.log(message);
+        void showAndLogErrorMessage(message);
+      }
+      return {
+        query,
+        result,
+        database: {
+          name: db.name,
+          databaseUri: db.databaseUri.toString(true)
+        },
+        options: historyItemOptions,
+        logFileLocation: result.logFileLocation,
+        dispose: () => {
+          qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+        }
+      };
     } else {
-      helpers.showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
-        ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
-        ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
-        ' and choose CodeQL Query Server from the dropdown.');
-    }
+      // Error dialogs are limited in size and scrollability,
+      // so we include a general description of the problem,
+      // and direct the user to the output window for the detailed compilation messages.
+      // However we don't show quick eval errors there so we need to display them anyway.
+      void qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
 
-    return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
+      const formattedMessages: string[] = [];
+
+      for (const error of errors) {
+        const message = error.message || '[no error message available]';
+        const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
+        formattedMessages.push(formatted);
+        void qs.logger.log(formatted);
+      }
+      if (quickEval && formattedMessages.length <= 2) {
+        // If there are more than 2 error messages, they will not be displayed well in a popup
+        // and will be trimmed by the function displaying the error popup. Accordingly, we only
+        // try to show the errors if there are 2 or less, otherwise we direct the user to the log.
+        void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
+      } else {
+        void showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
+      }
+
+      return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
+    }
+  } finally {
+    try {
+      await upgradeDir.cleanup();
+    } catch (e) {
+      void qs.logger.log(`Could not clean up the upgrades dir. Reason: ${e.message || e}`);
+    }
   }
 }
 
+const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
+  ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
+  ' and choose CodeQL Query Server from the dropdown.';
+
 function createSyntheticResult(
   query: QueryInfo,
   db: DatabaseItem,

extensions/ql-vscode/src/status-bar.ts

@@ -0,0 +1,48 @@
+import { ConfigurationChangeEvent, StatusBarAlignment, StatusBarItem, window, workspace } from 'vscode';
+import { CodeQLCliServer } from './cli';
+import { CANARY_FEATURES, CUSTOM_CODEQL_PATH_SETTING, DistributionConfigListener } from './config';
+import { DisposableObject } from './pure/disposable-object';
+
+/**
+ * Creates and manages a status bar item for codeql. THis item contains
+ * the current codeQL cli version as well as a notification if you are
+ * in canary mode
+ *
+ */
+export class CodeQlStatusBarHandler extends DisposableObject {
+
+  private readonly item: StatusBarItem;
+
+  constructor(private cli: CodeQLCliServer, distributionConfigListener: DistributionConfigListener) {
+    super();
+    this.item = window.createStatusBarItem(StatusBarAlignment.Right);
+    this.push(this.item);
+    this.push(workspace.onDidChangeConfiguration(this.handleDidChangeConfiguration, this));
+    this.push(distributionConfigListener.onDidChangeConfiguration(() => this.updateStatusItem()));
+    this.item.command = 'codeQL.copyVersion';
+    void this.updateStatusItem();
+  }
+
+  private handleDidChangeConfiguration(e: ConfigurationChangeEvent) {
+    if (
+      e.affectsConfiguration(CANARY_FEATURES.qualifiedName) ||
+      e.affectsConfiguration(CUSTOM_CODEQL_PATH_SETTING.qualifiedName)
+    ) {
+      // Wait a few seconds before updating the status item.
+      // This avoids a race condition where the cli's version
+      // is not updated before the status bar is refreshed.
+      setTimeout(() => this.updateStatusItem(), 3000);
+    }
+  }
+
+  private async updateStatusItem() {
+    const canary = CANARY_FEATURES.getValue() ? ' (Canary)' : '';
+    // since getting the version may take a few seconds, initialize with some
+    // meaningful text.
+    this.item.text = `CodeQL${canary}`;
+
+    const version = await this.cli.getVersion();
+    this.item.text = `CodeQL CLI v${version}${canary}`;
+    this.item.show();
+  }
+}

extensions/ql-vscode/src/telemetry.ts

@@ -0,0 +1,216 @@
+import { ConfigurationTarget, Extension, ExtensionContext, ConfigurationChangeEvent } from 'vscode';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY } from './config';
+import * as appInsights from 'applicationinsights';
+import { logger } from './logging';
+import { UserCancellationException } from './commandRunner';
+import { showBinaryChoiceWithUrlDialog } from './helpers';
+
+// Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
+const key = 'REPLACE-APP-INSIGHTS-KEY';
+
+export enum CommandCompletion {
+  Success = 'Success',
+  Failed = 'Failed',
+  Cancelled = 'Cancelled'
+}
+
+// Avoid sending the following data to App insights since we don't need it.
+const tagsToRemove = [
+  'ai.application.ver',
+  'ai.device.id',
+  'ai.cloud.roleInstance',
+  'ai.cloud.role',
+  'ai.device.id',
+  'ai.device.osArchitecture',
+  'ai.device.osPlatform',
+  'ai.device.osVersion',
+  'ai.internal.sdkVersion',
+  'ai.session.id'
+];
+
+const baseDataPropertiesToRemove = [
+  'common.os',
+  'common.platformversion',
+  'common.remotename',
+  'common.uikind',
+  'common.vscodesessionid'
+];
+
+export class TelemetryListener extends ConfigListener {
+
+  static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
+
+  private reporter?: TelemetryReporter;
+
+  constructor(
+    private readonly id: string,
+    private readonly version: string,
+    private readonly key: string,
+    private readonly ctx: ExtensionContext
+  ) {
+    super();
+  }
+
+  /**
+   * This function handles changes to relevant configuration elements. There are 2 configuration
+   * ids that this function cares about:
+   *
+   *     * `codeQL.telemetry.enableTelemetry`: If this one has changed, then we need to re-initialize
+   *        the reporter and the reporter may wind up being removed.
+   *     * `codeQL.canary`: A change here could possibly re-trigger a dialog popup.
+   *
+   * Note that the global telemetry setting also gate-keeps whether or not to send telemetry events
+   * to Application Insights. However, this gatekeeping happens inside of the vscode-extension-telemetry
+   * package. So, this does not need to be handled here.
+   *
+   * @param e the configuration change event
+   */
+  async handleDidChangeConfiguration(e: ConfigurationChangeEvent): Promise<void> {
+    if (
+      e.affectsConfiguration('codeQL.telemetry.enableTelemetry') ||
+      e.affectsConfiguration('telemetry.enableTelemetry')
+    ) {
+      await this.initialize();
+    }
+
+    // Re-request telemetry so that users can see the dialog again.
+    // Re-request if codeQL.canary is being set to `true` and telemetry
+    // is not currently enabled.
+    if (
+      e.affectsConfiguration('codeQL.canary') &&
+      CANARY_FEATURES.getValue() &&
+      !ENABLE_TELEMETRY.getValue()
+    ) {
+      await Promise.all([
+        this.setTelemetryRequested(false),
+        this.requestTelemetryPermission()
+      ]);
+    }
+  }
+
+  async initialize() {
+    await this.requestTelemetryPermission();
+
+    this.disposeReporter();
+
+    if (ENABLE_TELEMETRY.getValue<boolean>()) {
+      this.createReporter();
+    }
+  }
+
+  private createReporter() {
+    this.reporter = new TelemetryReporter(
+      this.id,
+      this.version,
+      this.key,
+      /* anonymize stack traces */ true
+    );
+    this.push(this.reporter);
+
+    const client = (this.reporter as any).appInsightsClient as appInsights.TelemetryClient;
+    if (client) {
+      // add a telemetry processor to delete unwanted properties
+      client.addTelemetryProcessor((envelope: any) => {
+        tagsToRemove.forEach(tag => delete envelope.tags[tag]);
+        const baseDataProperties = (envelope.data as any)?.baseData?.properties;
+        if (baseDataProperties) {
+          baseDataPropertiesToRemove.forEach(prop => delete baseDataProperties[prop]);
+        }
+
+        if (LOG_TELEMETRY.getValue<boolean>()) {
+          void logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
+        }
+        return true;
+      });
+    }
+  }
+
+  dispose() {
+    super.dispose();
+    void this.reporter?.dispose();
+  }
+
+  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+    if (!this.reporter) {
+      return;
+    }
+    const status = !error
+      ? CommandCompletion.Success
+      : error instanceof UserCancellationException
+        ? CommandCompletion.Cancelled
+        : CommandCompletion.Failed;
+
+    const isCanary = (!!CANARY_FEATURES.getValue<boolean>()).toString();
+
+    this.reporter.sendTelemetryEvent(
+      'command-usage',
+      {
+        name,
+        status,
+        isCanary
+      },
+      { executionTime }
+    );
+  }
+
+  /**
+   * Displays a popup asking the user if they want to enable telemetry
+   * for this extension.
+   */
+  async requestTelemetryPermission() {
+    if (!this.wasTelemetryRequested()) {
+      // if global telemetry is disabled, avoid showing the dialog or making any changes
+      let result = undefined;
+      if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
+        // Extension won't start until this completes.
+        result = await showBinaryChoiceWithUrlDialog(
+          'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?',
+          'https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code'
+        );
+      }
+      if (result !== undefined) {
+        await Promise.all([
+          this.setTelemetryRequested(true),
+          ENABLE_TELEMETRY.updateValue<boolean>(result, ConfigurationTarget.Global),
+        ]);
+      }
+    }
+  }
+
+  /**
+   * Exposed for testing
+   */
+  get _reporter() {
+    return this.reporter;
+  }
+
+  private disposeReporter() {
+    if (this.reporter) {
+      void this.reporter.dispose();
+      this.reporter = undefined;
+    }
+  }
+
+  private wasTelemetryRequested(): boolean {
+    return !!this.ctx.globalState.get<boolean>('telemetry-request-viewed');
+  }
+
+  private async setTelemetryRequested(newValue: boolean): Promise<void> {
+    await this.ctx.globalState.update('telemetry-request-viewed', newValue);
+  }
+}
+
+
+/**
+ * The global Telemetry instance
+ */
+export let telemetryListener: TelemetryListener;
+
+export async function initializeTelemetry(extension: Extension<any>, ctx: ExtensionContext): Promise<void> {
+  telemetryListener = new TelemetryListener(extension.id, extension.packageJSON.version, key, ctx);
+  // do not await initialization, since doing so will sometimes cause a modal popup.
+  // this is a particular problem during integration tests, which will hang if a modal popup is displayed.
+  void telemetryListener.initialize();
+  ctx.subscriptions.push(telemetryListener);
+}

extensions/ql-vscode/src/test-adapter.ts

@@ -1,3 +1,4 @@
+import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as vscode from 'vscode';
 import {
@@ -15,10 +16,11 @@ import {
 import { TestAdapterRegistrar } from 'vscode-test-adapter-util';
 import { QLTestFile, QLTestNode, QLTestDirectory, QLTestDiscovery } from './qltest-discovery';
 import { Event, EventEmitter, CancellationTokenSource, CancellationToken } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { CodeQLCliServer } from './cli';
-import { getOnDiskWorkspaceFolders } from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
 import { testLogger } from './logging';
+import { DatabaseItem, DatabaseManager } from './databases';
 
 /**
  * Get the full path of the `.expected` file for the specified QL test.
@@ -57,13 +59,13 @@ function getTestOutputFile(testPath: string, extension: string): string {
  * A factory service that creates `QLTestAdapter` objects for workspace folders on demand.
  */
 export class QLTestAdapterFactory extends DisposableObject {
-  constructor(testHub: TestHub, cliServer: CodeQLCliServer) {
+  constructor(testHub: TestHub, cliServer: CodeQLCliServer, databaseManager: DatabaseManager) {
     super();
 
     // this will register a QLTestAdapter for each WorkspaceFolder
     this.push(new TestAdapterRegistrar(
       testHub,
-      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer)
+      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer, databaseManager)
     ));
   }
 }
@@ -91,7 +93,8 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
 
   constructor(
     public readonly workspaceFolder: vscode.WorkspaceFolder,
-    private readonly cliServer: CodeQLCliServer
+    private readonly cliServer: CodeQLCliServer,
+    private readonly databaseManager: DatabaseManager
   ) {
     super();
 
@@ -182,19 +185,79 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
     testLogger.outputChannel.show(true);
 
     this.runningTask = this.track(new CancellationTokenSource());
+    const token = this.runningTask.token;
 
     this._testStates.fire({ type: 'started', tests: tests } as TestRunStartedEvent);
 
+    const currentDatabaseUri = this.databaseManager.currentDatabaseItem?.databaseUri;
+    const databasesUnderTest = this.databaseManager.databaseItems
+      .filter(database => tests.find(testPath => database.isAffectedByTest(testPath)));
+
+    await this.removeDatabasesBeforeTests(databasesUnderTest, token);
     try {
-      await this.runTests(tests, this.runningTask.token);
-    }
-    catch (e) {
-      /**/
+      await this.runTests(tests, token);
+    } catch (e) {
+      // CodeQL testing can throw exception even in normal scenarios. For example, if the test run
+      // produces no output (which is normal), the testing command would throw an exception on
+      // unexpected EOF during json parsing. So nothing needs to be done here - all the relevant
+      // error information (if any) should have already been written to the test logger.
     }
+    await this.reopenDatabasesAfterTests(databasesUnderTest, currentDatabaseUri, token);
+
     this._testStates.fire({ type: 'finished' } as TestRunFinishedEvent);
     this.clearTask();
   }
 
+  private async removeDatabasesBeforeTests(
+    databasesUnderTest: DatabaseItem[], token: vscode.CancellationToken): Promise<void> {
+    for (const database of databasesUnderTest) {
+      try {
+        await this.databaseManager
+          .removeDatabaseItem(_ => { /* no progress reporting */ }, token, database);
+      } catch (e) {
+        // This method is invoked from Test Explorer UI, and testing indicates that Test
+        // Explorer UI swallows any thrown exception without reporting it to the user.
+        // So we need to display the error message ourselves and then rethrow.
+        void showAndLogErrorMessage(`Cannot remove database ${database.name}: ${e}`);
+        throw e;
+      }
+    }
+  }
+
+  private async reopenDatabasesAfterTests(
+    databasesUnderTest: DatabaseItem[],
+    currentDatabaseUri: vscode.Uri | undefined,
+    token: vscode.CancellationToken): Promise<void> {
+    for (const closedDatabase of databasesUnderTest) {
+      const uri = closedDatabase.databaseUri;
+      if (await this.isFileAccessible(uri)) {
+        try {
+          const reopenedDatabase = await this.databaseManager
+            .openDatabase(_ => { /* no progress reporting */ }, token, uri);
+          await this.databaseManager.renameDatabaseItem(reopenedDatabase, closedDatabase.name);
+          if (currentDatabaseUri == uri) {
+            await this.databaseManager.setCurrentDatabaseItem(reopenedDatabase, true);
+          }
+        } catch (e) {
+          // This method is invoked from Test Explorer UI, and testing indicates that Test
+          // Explorer UI swallows any thrown exception without reporting it to the user.
+          // So we need to display the error message ourselves and then rethrow.
+          void showAndLogWarningMessage(`Cannot reopen database ${uri}: ${e}`);
+          throw e;
+        }
+      }
+    }
+  }
+
+  private async isFileAccessible(uri: vscode.Uri): Promise<boolean> {
+    try {
+      await fs.access(uri.fsPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
   private clearTask(): void {
     if (this.runningTask !== undefined) {
       const runningTask = this.runningTask;
@@ -205,7 +268,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
 
   public cancel(): void {
     if (this.runningTask !== undefined) {
-      testLogger.log('Cancelling test run...');
+      void testLogger.log('Cancelling test run...');
       this.runningTask.cancel();
       this.clearTask();
     }
@@ -220,14 +283,13 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
       const state = event.pass
         ? 'passed'
         : event.messages?.length
-        ? 'errored'
-        : 'failed';
+          ? 'errored'
+          : 'failed';
       let message: string | undefined;
-      if (event.diff?.length) {
-        message = ['', `${state}: ${event.test}`, ...event.diff, ''].join('\n');
-        testLogger.log(message);
+      if (event.failureDescription || event.diff?.length) {
+        message = ['', `${state}: ${event.test}`, event.failureDescription || event.diff?.join('\n'), ''].join('\n');
+        void testLogger.log(message);
       }
-      (event.diff || []).join('\n');
       this._testStates.fire({
         type: 'test',
         state,

extensions/ql-vscode/src/test-ui.ts

@@ -13,7 +13,7 @@ import {
 
 import { showAndLogWarningMessage } from './helpers';
 import { TestTreeNode } from './test-tree-node';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { UIService } from './vscode-utils/ui-service';
 import { QLTestAdapter, getExpectedFile, getActualFile } from './test-adapter';
 import { logger } from './logging';
@@ -44,7 +44,7 @@ export class TestUIService extends UIService implements TestController {
   constructor(private readonly testHub: TestHub) {
     super();
 
-    logger.log('Registering CodeQL test panel commands.');
+    void logger.log('Registering CodeQL test panel commands.');
     this.registerCommand('codeQLTests.showOutputDifferences', this.showOutputDifferences);
     this.registerCommand('codeQLTests.acceptOutput', this.acceptOutput);
 
@@ -90,7 +90,7 @@ export class TestUIService extends UIService implements TestController {
       };
 
       if (!await fs.pathExists(expectedPath)) {
-        showAndLogWarningMessage(`'${path.basename(expectedPath)}' does not exist. Creating an empty file.`);
+        void showAndLogWarningMessage(`'${path.basename(expectedPath)}' does not exist. Creating an empty file.`);
         await fs.createFile(expectedPath);
       }
 

extensions/ql-vscode/src/upgrades.ts

@@ -1,10 +1,14 @@
 import * as vscode from 'vscode';
-import { DatabaseItem } from './databases';
-import * as helpers from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 import * as messages from './pure/messages';
 import * as qsClient from './queryserver-client';
 import { upgradesTmpDir } from './run-queries';
+import * as tmp from 'tmp-promise';
+import * as path from 'path';
+import * as semver from 'semver';
+import { DatabaseItem } from './databases';
 
 /**
  * Maximum number of lines to include from database upgrade message,
@@ -14,74 +18,99 @@ import { upgradesTmpDir } from './run-queries';
 const MAX_UPGRADE_MESSAGE_LINES = 10;
 
 /**
- * Checks whether the given database can be upgraded to the given target DB scheme,
- * and whether the user wants to proceed with the upgrade.
- * Reports errors to both the user and the console.
- * @returns the `UpgradeParams` needed to start the upgrade, if the upgrade is possible and was confirmed by the user, or `undefined` otherwise.
+ * Check that we support non-destructive upgrades.
+ *
+ * This requires 3 features. The ability to compile an upgrade sequence; The ability to
+ * run a non-destructive upgrades as a query; the ability to specify a target when
+ * resolving upgrades. We check for a version of codeql that has all three features.
  */
-async function checkAndConfirmDatabaseUpgrade(
+export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QueryServerClient): Promise<boolean> {
+  return semver.gte(await qs.cliServer.getVersion(), '2.4.2');
+}
+
+
+/**
+ * Compile a database upgrade sequence.
+ * Callers must check that this is valid with the current queryserver first.
+ */
+export async function compileDatabaseUpgradeSequence(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
-  targetDbScheme: vscode.Uri,
-  upgradesDirectories: vscode.Uri[],
-  progress: helpers.ProgressCallback,
-  token: vscode.CancellationToken,
-): Promise<messages.UpgradeParams | undefined> {
+  resolvedSequence: string[],
+  currentUpgradeTmp: tmp.DirectoryResult,
+  progress: ProgressCallback,
+  token: vscode.CancellationToken
+): Promise<messages.CompileUpgradeSequenceResult> {
   if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
-  const params: messages.UpgradeParams = {
-    fromDbscheme: db.contents.dbSchemeUri.fsPath,
-    toDbscheme: targetDbScheme.fsPath,
-    additionalUpgrades: upgradesDirectories.map(uri => uri.fsPath)
-  };
+  if (!await hasNondestructiveUpgradeCapabilities(qs)) {
+    throw new Error('The version of codeql is too old to run non-destructive upgrades.');
+  }
+  // If possible just compile the upgrade sequence
+  return await qs.sendRequest(messages.compileUpgradeSequence, {
+    upgradeTempDir: currentUpgradeTmp.path,
+    upgradePaths: resolvedSequence
+  }, token, progress);
+}
 
-  let checkUpgradeResult: messages.CheckUpgradeResult;
-  try {
-    qs.logger.log('Checking database upgrade...');
-    checkUpgradeResult = await checkDatabaseUpgrade(qs, params, progress, token);
-  }
-  catch (e) {
-    throw new Error(`Database cannot be upgraded: ${e}`);
-  }
-  finally {
-    qs.logger.log('Done checking database upgrade.');
+async function compileDatabaseUpgrade(
+  qs: qsClient.QueryServerClient,
+  db: DatabaseItem,
+  targetDbScheme: string,
+  resolvedSequence: string[],
+  currentUpgradeTmp: tmp.DirectoryResult,
+  progress: ProgressCallback,
+  token: vscode.CancellationToken
+): Promise<messages.CompileUpgradeResult> {
+  if (!db.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
   }
+  // We have the upgrades we want but compileUpgrade
+  // requires searching for them.  So we use the parent directories of the upgrades
+  // as the upgrade path.
+  const parentDirs = resolvedSequence.map(dir => path.dirname(dir));
+  const uniqueParentDirs = new Set(parentDirs);
+  progress({
+    step: 1,
+    maxStep: 3,
+    message: 'Checking for database upgrades'
+  });
+  return qs.sendRequest(messages.compileUpgrade, {
+    upgrade: {
+      fromDbscheme: db.contents.dbSchemeUri.fsPath,
+      toDbscheme: targetDbScheme,
+      additionalUpgrades: Array.from(uniqueParentDirs)
+    },
+    upgradeTempDir: currentUpgradeTmp.path,
+    singleFileUpgrades: true,
+  }, token, progress);
+}
 
-  const checkedUpgrades = checkUpgradeResult.checkedUpgrades;
-  if (checkedUpgrades === undefined) {
-    const error = checkUpgradeResult.upgradeError || '[no error message available]';
-    throw new Error(`Database cannot be upgraded: ${error}`);
-  }
+/**
+ * Checks whether the user wants to proceed with the upgrade.
+ * Reports errors to both the user and the console.
+ */
+async function checkAndConfirmDatabaseUpgrade(
+  compiled: messages.CompiledUpgrades,
+  db: DatabaseItem,
+  quiet: boolean
+): Promise<void> {
 
-  if (checkedUpgrades.scripts.length === 0) {
-    progress({
-      step: 3,
-      maxStep: 3,
-      message: 'Database is already up to date; nothing to do.'
-    });
-    return;
-  }
-
-  let curSha = checkedUpgrades.initialSha;
   let descriptionMessage = '';
-  for (const script of checkedUpgrades.scripts) {
+  const descriptions = getUpgradeDescriptions(compiled);
+  for (const script of descriptions) {
     descriptionMessage += `Would perform upgrade: ${script.description}\n`;
     descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
-    curSha = script.newSha;
   }
+  void logger.log(descriptionMessage);
 
-  const targetSha = checkedUpgrades.targetSha;
-  if (curSha != targetSha) {
-    // Newlines aren't rendered in notifications: https://github.com/microsoft/vscode/issues/48900
-    // A modal dialog would be rendered better, but is more intrusive.
-    await helpers.showAndLogErrorMessage(`Database cannot be upgraded to the target database scheme.
-    Can upgrade from ${checkedUpgrades.initialSha} (current) to ${curSha}, but cannot reach ${targetSha} (target).`);
-    // TODO: give a more informative message if we think the DB is ahead of the target DB scheme
+
+  // If the quiet flag is set, do the upgrade without a popup.
+  if (quiet) {
     return;
   }
 
-  logger.log(descriptionMessage);
   // Ask the user to confirm the upgrade.
 
   const showLogItem: vscode.MessageItem = { title: 'No, Show Changes', isCloseAffordance: true };
@@ -103,11 +132,22 @@ async function checkAndConfirmDatabaseUpgrade(
     logger.outputChannel.show();
   }
 
-  if (chosenItem === yesItem) {
-    return params;
+  if (chosenItem !== yesItem) {
+    throw new UserCancellationException('User cancelled the database upgrade.');
   }
-  else {
-    throw new helpers.UserCancellationException('User cancelled the database upgrade.');
+}
+
+/**
+ * Get the descriptions from a compiled upgrade
+ */
+function getUpgradeDescriptions(compiled: messages.CompiledUpgrades): messages.UpgradeDescription[] {
+  // We use the presence of compiledUpgradeFile to check
+  // if it is multifile or not. We need to explicitly check undefined
+  // as the types claim the empty string is a valid value
+  if (compiled.compiledUpgradeFile === undefined) {
+    return compiled.scripts.map(script => script.description);
+  } else {
+    return compiled.descriptions;
   }
 }
 
@@ -117,91 +157,73 @@ async function checkAndConfirmDatabaseUpgrade(
  * First performs a dry-run and prompts the user to confirm the upgrade.
  * Reports errors during compilation and evaluation of upgrades to the user.
  */
-export async function upgradeDatabase(
+export async function upgradeDatabaseExplicit(
   qs: qsClient.QueryServerClient,
-  db: DatabaseItem, targetDbScheme: vscode.Uri,
-  upgradesDirectories: vscode.Uri[],
-  progress: helpers.ProgressCallback,
+  db: DatabaseItem,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.RunUpgradeResult | undefined> {
-  const upgradeParams = await checkAndConfirmDatabaseUpgrade(qs, db, targetDbScheme, upgradesDirectories, progress, token);
 
-  if (upgradeParams === undefined) {
-    return;
+  const searchPath: string[] = getOnDiskWorkspaceFolders();
+
+  if (!db?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
   }
+  const upgradeInfo = await qs.cliServer.resolveUpgrades(
+    db.contents.dbSchemeUri.fsPath,
+    searchPath,
+    false
+  );
 
-  let compileUpgradeResult: messages.CompileUpgradeResult;
+  const { scripts, finalDbscheme } = upgradeInfo;
+
+  if (finalDbscheme === undefined) {
+    throw new Error('Could not determine target dbscheme to upgrade to.');
+  }
+  const currentUpgradeTmp = await tmp.dir({ dir: upgradesTmpDir.name, prefix: 'upgrade_', keep: false, unsafeCleanup: true });
   try {
-    compileUpgradeResult = await compileDatabaseUpgrade(qs, upgradeParams, progress, token);
-  }
-  catch (e) {
-    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done compiling database upgrade.');
-  }
+    let compileUpgradeResult: messages.CompileUpgradeResult;
+    try {
+      compileUpgradeResult = await compileDatabaseUpgrade(qs, db, finalDbscheme, scripts, currentUpgradeTmp, progress, token);
+    }
+    catch (e) {
+      void showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
+      return;
+    }
+    finally {
+      void qs.logger.log('Done compiling database upgrade.');
+    }
 
-  if (compileUpgradeResult.compiledUpgrades === undefined) {
-    const error = compileUpgradeResult.error || '[no error message available]';
-    helpers.showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
-    return;
+    if (!compileUpgradeResult.compiledUpgrades) {
+      const error = compileUpgradeResult.error || '[no error message available]';
+      void showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
+      return;
+    }
+
+    await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, db, qs.cliServer.quiet);
+
+    try {
+      void qs.logger.log('Running the following database upgrade:');
+
+      getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n');
+      return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
+    }
+    catch (e) {
+      void showAndLogErrorMessage(`Database upgrade failed: ${e}`);
+      return;
+    } finally {
+      void qs.logger.log('Done running database upgrade.');
+    }
+  } finally {
+    await currentUpgradeTmp.cleanup();
   }
-
-  try {
-    qs.logger.log('Running the following database upgrade:');
-    qs.logger.log(compileUpgradeResult.compiledUpgrades.scripts.map(s => s.description.description).join('\n'));
-    return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
-  }
-  catch (e) {
-    helpers.showAndLogErrorMessage(`Database upgrade failed: ${e}`);
-    return;
-  }
-  finally {
-    qs.logger.log('Done running database upgrade.');
-  }
-}
-
-async function checkDatabaseUpgrade(
-  qs: qsClient.QueryServerClient,
-  upgradeParams: messages.UpgradeParams,
-  progress: helpers.ProgressCallback,
-  token: vscode.CancellationToken,
-): Promise<messages.CheckUpgradeResult> {
-  progress({
-    step: 1,
-    maxStep: 3,
-    message: 'Checking for database upgrades'
-  });
-
-  return qs.sendRequest(messages.checkUpgrade, upgradeParams, token, progress);
-}
-
-async function compileDatabaseUpgrade(
-  qs: qsClient.QueryServerClient,
-  upgradeParams: messages.UpgradeParams,
-  progress: helpers.ProgressCallback,
-  token: vscode.CancellationToken,
-): Promise<messages.CompileUpgradeResult> {
-  const params: messages.CompileUpgradeParams = {
-    upgrade: upgradeParams,
-    upgradeTempDir: upgradesTmpDir.name
-  };
-
-  progress({
-    step: 2,
-    maxStep: 3,
-    message: 'Compiling database upgrades'
-  });
-
-  return qs.sendRequest(messages.compileUpgrade, params, token, progress);
 }
 
 async function runDatabaseUpgrade(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
   upgrades: messages.CompiledUpgrades,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.RunUpgradeResult> {
 

extensions/ql-vscode/src/view/alert-table.tsx

@@ -3,7 +3,7 @@ import * as React from 'react';
 import * as Sarif from 'sarif';
 import * as Keys from '../pure/result-keys';
 import * as octicons from './octicons';
-import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection } from './result-table-utils';
+import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection, emptyQueryResultsMessage } from './result-table-utils';
 import { onNavigation, NavigationEvent } from './results';
 import { PathTableResultSet } from '../pure/interface-types';
 import {
@@ -79,7 +79,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
     if (this.props.nonemptyRawResults) {
       return <span>No Alerts. See <a href='#' onClick={this.props.showRawResults}>raw results</a>.</span>;
     } else {
-      return <span>No Alerts</span>;
+      return emptyQueryResultsMessage();
     }
   }
 

extensions/ql-vscode/src/view/raw-results-table.tsx

@@ -1,5 +1,5 @@
 import * as React from 'react';
-import { ResultTableProps, className } from './result-table-utils';
+import { ResultTableProps, className, emptyQueryResultsMessage } from './result-table-utils';
 import { RAW_RESULTS_LIMIT, RawResultsSortState } from '../pure/interface-types';
 import { RawTableResultSet } from '../pure/interface-types';
 import RawTableHeader from './RawTableHeader';
@@ -12,7 +12,7 @@ export type RawTableProps = ResultTableProps & {
   offset: number;
 };
 
-export class RawTable extends React.Component<RawTableProps, {}> {
+export class RawTable extends React.Component<RawTableProps, Record<string, never>> {
   constructor(props: RawTableProps) {
     super(props);
   }
@@ -21,6 +21,10 @@ export class RawTable extends React.Component<RawTableProps, {}> {
     const { resultSet, databaseUri } = this.props;
 
     let dataRows = resultSet.rows;
+    if (dataRows.length === 0) {
+      return emptyQueryResultsMessage();
+    }
+
     let numTruncatedResults = 0;
     if (dataRows.length > RAW_RESULTS_LIMIT) {
       numTruncatedResults = dataRows.length - RAW_RESULTS_LIMIT;

extensions/ql-vscode/src/view/result-table-utils.tsx

@@ -28,7 +28,8 @@ export interface ResultTableProps {
 }
 
 export const className = 'vscode-codeql__result-table';
-export const tableSelectionHeaderClassName = 'vscode-codeql__table-selection-header';
+export const tableHeaderClassName = 'vscode-codeql__table-selection-header';
+export const tableHeaderItemClassName = 'vscode-codeql__table-selection-header-item';
 export const alertExtrasClassName = `${className}-alert-extras`;
 export const toggleDiagnosticsClassName = `${className}-toggle-diagnostics`;
 export const evenRowClassName = 'vscode-codeql__result-table-row--even';
@@ -45,7 +46,9 @@ export function jumpToLocationHandler(
     jumpToLocation(loc, databaseUri);
     e.preventDefault();
     e.stopPropagation();
-    if (callback) callback();
+    if (callback) {
+      callback();
+    }
   };
 }
 
@@ -57,6 +60,13 @@ export function jumpToLocation(loc: ResolvableLocationValue, databaseUri: string
   });
 }
 
+export function openFile(filePath: string): void {
+  vscode.postMessage({
+    t: 'openFile',
+    filePath
+  });
+}
+
 /**
  * Render a location as a link which when clicked displays the original location.
  */
@@ -130,3 +140,9 @@ export function nextSortDirection(direction: SortDirection | undefined, includeU
       return assertNever(direction);
   }
 }
+
+export function emptyQueryResultsMessage(): JSX.Element {
+  return <div className='vscode-codeql__empty-query-message'><span>
+    This query returned no results. If this isn&apos;t what you were expecting, and for effective query-writing tips, check out the <a href="https://codeql.github.com/docs/codeql-language-guides/">CodeQL language guides</a>.
+  </span></div>;
+}

extensions/ql-vscode/src/view/result-tables.tsx

@@ -11,19 +11,23 @@ import {
   SELECT_TABLE_NAME,
   getDefaultResultSetName,
   ParsedResultSets,
-  IntoResultsViewMsg
+  IntoResultsViewMsg,
 } from '../pure/interface-types';
 import { PathTable } from './alert-table';
 import { RawTable } from './raw-results-table';
 import {
   ResultTableProps,
-  tableSelectionHeaderClassName,
+  tableHeaderClassName,
+  tableHeaderItemClassName,
   toggleDiagnosticsClassName,
-  alertExtrasClassName
+  alertExtrasClassName,
+  openFile
 } from './result-table-utils';
 import { vscode } from './vscode-api';
 
 
+const FILE_PATH_REGEX = /^(?:.+[\\/])*(.+)$/;
+
 /**
  * Properties for the `ResultTables` component.
  */
@@ -38,6 +42,8 @@ export interface ResultTablesProps {
   sortStates: Map<string, RawResultsSortState>;
   interpretedSortState?: InterpretedResultsSortState;
   isLoadingNewResults: boolean;
+  queryName: string;
+  queryPath: string;
 }
 
 /**
@@ -62,7 +68,7 @@ function getResultCount(resultSet: ResultSet): number {
 
 function renderResultCountString(resultSet: ResultSet): JSX.Element {
   const resultCount = getResultCount(resultSet);
-  return <span className="number-of-results">
+  return <span className={tableHeaderItemClassName}>
     {resultCount} {resultCount === 1 ? 'result' : 'results'}
   </span>;
 }
@@ -76,7 +82,7 @@ export class ResultTables
 
   private getResultSets(): ResultSet[] {
     const resultSets: ResultSet[] =
-      // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore 2783
       this.props.rawResultSets.map((rs) => ({ t: 'RawResultSet', ...rs }));
 
@@ -213,28 +219,44 @@ export class ResultTables
       });
     };
 
-    return <span className="vscode-codeql__table-selection-header">
-      <button onClick={prevPage} >&#xab;</button>
-      <input
-        type="number"
-        size={3}
-        value={this.state.selectedPage}
-        min="1"
-        max={numPages}
-        onChange={onChange}
-        onBlur={e => choosePage(e.target.value)}
-        onKeyDown={e => {
-          if (e.keyCode === 13) {
-            choosePage((e.target as HTMLInputElement).value);
-          }
-        }
-        }
-      />
-      <span>
-        / {numPages}
+    const openQuery = () => {
+      openFile(this.props.queryPath);
+    };
+    const fileName = FILE_PATH_REGEX.exec(this.props.queryPath)?.[1] || 'query';
+
+    return (
+      <span className="vscode-codeql__table-selection-pagination">
+        <button onClick={prevPage} >&#xab;</button>
+        <input
+          type="number"
+          size={3}
+          value={this.state.selectedPage}
+          min="1"
+          max={numPages}
+          onChange={onChange}
+          onBlur={e => choosePage(e.target.value)}
+          onKeyDown={e => {
+            if (e.keyCode === 13) {
+              choosePage((e.target as HTMLInputElement).value);
+            }
+          }}
+        />
+        <span>
+          /&nbsp;{numPages}
+        </span>
+        <button value=">" onClick={nextPage} >&#xbb;</button>
+        <div className={tableHeaderItemClassName}>
+          {this.props.queryName}
+        </div>
+        <div className={tableHeaderItemClassName}>
+          <a
+            href="#"
+            onClick={openQuery}
+            className="vscode-codeql__result-table-location-link"
+          >Open {fileName}</a>
+        </div>
       </span>
-      <button value=">" onClick={nextPage} >&#xbb;</button>
-    </span>;
+    );
   }
 
   render(): React.ReactNode {
@@ -248,32 +270,35 @@ export class ResultTables
 
     const resultSetOptions =
       resultSetNames.map(name => <option key={name} value={name}>{name}</option>);
-
-    return <div>
-      {this.renderPageButtons()}
-      <div className={tableSelectionHeaderClassName}>
-        <select value={selectedTable} onChange={this.onTableSelectionChange}>
-          {resultSetOptions}
-        </select>
-        {numberOfResults}
-        {selectedTable === ALERTS_TABLE_NAME ? this.alertTableExtras() : undefined}
+    return (
+      <div>
+        {this.renderPageButtons()}
+        <div className={tableHeaderClassName}>
+        </div>
+        <div className={tableHeaderClassName}>
+          <select value={selectedTable} onChange={this.onTableSelectionChange}>
+            {resultSetOptions}
+          </select>
+          {numberOfResults}
+          {selectedTable === ALERTS_TABLE_NAME ? this.alertTableExtras() : undefined}
+          {
+            this.props.isLoadingNewResults ?
+              <span className={UPDATING_RESULTS_TEXT_CLASS_NAME}>Updating results…</span>
+              : null
+          }
+        </div>
         {
-          this.props.isLoadingNewResults ?
-            <span className={UPDATING_RESULTS_TEXT_CLASS_NAME}>Updating results…</span>
-            : null
+          resultSet &&
+          <ResultTable key={resultSet.schema.name} resultSet={resultSet}
+            databaseUri={this.props.database.databaseUri}
+            resultsPath={this.props.resultsPath}
+            sortState={this.props.sortStates.get(resultSet.schema.name)}
+            nonemptyRawResults={nonemptyRawResults}
+            showRawResults={() => { this.setState({ selectedTable: SELECT_TABLE_NAME }); }}
+            offset={this.getOffset()} />
         }
       </div>
-      {
-        resultSet &&
-        <ResultTable key={resultSet.schema.name} resultSet={resultSet}
-          databaseUri={this.props.database.databaseUri}
-          resultsPath={this.props.resultsPath}
-          sortState={this.props.sortStates.get(resultSet.schema.name)}
-          nonemptyRawResults={nonemptyRawResults}
-          showRawResults={() => { this.setState({ selectedTable: SELECT_TABLE_NAME }); }}
-          offset={this.getOffset()} />
-      }
-    </div>;
+    );
   }
 
   handleMessage(msg: IntoResultsViewMsg): void {
@@ -303,13 +328,15 @@ export class ResultTables
   }
 
   private vscodeMessageHandler(evt: MessageEvent) {
+    // sanitize origin
+    const origin = evt.origin.replace(/\n|\r/g, '');
     evt.origin === window.origin
       ? this.handleMessage(evt.data as IntoResultsViewMsg)
-      : console.error(`Invalid event origin ${evt.origin}`);
+      : console.error(`Invalid event origin ${origin}`);
   }
 }
 
-class ResultTable extends React.Component<ResultTableProps, {}> {
+class ResultTable extends React.Component<ResultTableProps, Record<string, never>> {
 
   constructor(props: ResultTableProps) {
     super(props);

extensions/ql-vscode/src/view/results.tsx

@@ -11,7 +11,7 @@ import {
   QueryMetadata,
   ResultsPaths,
   ALERTS_TABLE_NAME,
-  ParsedResultSets
+  ParsedResultSets,
 } from '../pure/interface-types';
 import { EventHandlers as EventHandlerList } from './event-handler-list';
 import { ResultTables } from './result-tables';
@@ -37,6 +37,8 @@ interface ResultsInfo {
    */
   shouldKeepOldResultsWhileRendering: boolean;
   metadata?: QueryMetadata;
+  queryName: string;
+  queryPath: string;
 }
 
 interface Results {
@@ -69,7 +71,7 @@ export const onNavigation = new EventHandlerList<NavigationEvent>();
 /**
  * A minimal state container for displaying results.
  */
-class App extends React.Component<{}, ResultsViewState> {
+class App extends React.Component<Record<string, never>, ResultsViewState> {
   constructor(props: any) {
     super(props);
     this.state = {
@@ -96,9 +98,11 @@ class App extends React.Component<{}, ResultsViewState> {
           shouldKeepOldResultsWhileRendering:
             msg.shouldKeepOldResultsWhileRendering,
           metadata: msg.metadata,
+          queryName: msg.queryName,
+          queryPath: msg.queryPath,
         });
 
-        this.loadResults();
+        void this.loadResults();
         break;
       case 'showInterpretedPage':
         this.updateStateWithNewResultsInfo({
@@ -127,8 +131,10 @@ class App extends React.Component<{}, ResultsViewState> {
           interpretation: msg.interpretation,
           shouldKeepOldResultsWhileRendering: true,
           metadata: msg.metadata,
+          queryName: msg.queryName,
+          queryPath: msg.queryPath,
         });
-        this.loadResults();
+        void this.loadResults();
         break;
       case 'resultsUpdating':
         this.setState({
@@ -280,6 +286,8 @@ class App extends React.Component<{}, ResultsViewState> {
             this.state.isExpectingResultsUpdate ||
             this.state.nextResultsInfo !== null
           }
+          queryName={displayedResults.resultsInfo.queryName}
+          queryPath={displayedResults.resultsInfo.queryPath}
         />
       );
     } else {
@@ -299,9 +307,11 @@ class App extends React.Component<{}, ResultsViewState> {
   }
 
   private vscodeMessageHandler(evt: MessageEvent) {
+    // sanitize origin
+    const origin = evt.origin.replace(/\n|\r/g, '');
     evt.origin === window.origin
       ? this.handleMessage(evt.data as IntoResultsViewMsg)
-      : console.error(`Invalid event origin ${evt.origin}`);
+      : console.error(`Invalid event origin ${origin}`);
   }
 }
 

extensions/ql-vscode/src/view/resultsView.css

@@ -8,13 +8,24 @@
   display: flex;
   padding: 0.5em 0;
   align-items: center;
+  justify-content: space-between;
+}
+
+.vscode-codeql__table-selection-pagination {
+  display: flex;
+  padding: 0.5em 0;
+  align-items: center;
+}
+
+.vscode-codeql__table-selection-header-item {
+  padding-left: 2em;
 }
 
 .vscode-codeql__table-selection-header select {
   border: 0;
 }
 
-.vscode-codeql__table-selection-header button {
+.vscode-codeql__table-selection-pagination button {
   padding: 0.3rem;
   margin: 0.2rem;
   border: 0;
@@ -25,11 +36,11 @@
   opacity: 0.8;
 }
 
-.vscode-codeql__table-selection-header button:hover {
+.vscode-codeql__table-selection-pagination button:hover {
   opacity: 1;
 }
 
-.vscode-codeql__table-selection-header input[type="number"] {
+.vscode-codeql__table-selection-pagination input[type="number"] {
   border-radius: 0;
   padding: 0.3rem;
   margin: 0.2rem;
@@ -185,10 +196,6 @@ td.vscode-codeql__path-index-cell {
   opacity: 0.6;
 }
 
-.number-of-results {
-  padding-left: 3em;
-}
-
 .vscode-codeql__compare-header {
   display: flex;
 }
@@ -214,3 +221,16 @@ td.vscode-codeql__path-index-cell {
 .vscode-codeql__compare-body > tbody {
   vertical-align: top;
 }
+
+.vscode-codeql__empty-query-message {
+  height: 300px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.vscode-codeql__empty-query-message > span {
+  max-width: 80%;
+  font-size: 14px;
+  text-align: center;
+}

extensions/ql-vscode/src/vscode-tests/.eslintrc.js

@@ -1,5 +1,19 @@
 module.exports = {
   env: {
     mocha: true
+  },
+  rules: {
+    "@typescript-eslint/ban-types": [
+      "error",
+      { 
+        // For a full list of the default banned types, see:
+        // https://github.com/typescript-eslint/typescript-eslint/blob/master/packages/eslint-plugin/docs/rules/ban-types.md
+        extendDefaults: true,
+        "types": {
+          // Don't complain about the `Function` type in test files. (Default is `true`.)
+          "Function": false,
+        }
+      }
+    ]
   }
 }

extensions/ql-vscode/src/vscode-tests/cli-integration/data/qlpack.yml

@@ -0,0 +1,3 @@
+name: integration-test-queries-javascript
+version: 0.0.0
+libraryPathDependencies: codeql-javascript

extensions/ql-vscode/src/vscode-tests/cli-integration/data/simple-query.ql

@@ -0,0 +1,8 @@
+predicate edges(int i, int j) {
+  i = 1 and j = 2 or i = 2 and j = 3
+}
+
+
+from int i, int j
+where edges(i, j)
+select i, j

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -0,0 +1,84 @@
+import * as sinon from 'sinon';
+import * as path from 'path';
+import { fail } from 'assert';
+import { expect } from 'chai';
+import { extensions, CancellationToken, Uri, window } from 'vscode';
+
+import { CodeQLExtensionInterface } from '../../extension';
+import { DatabaseManager } from '../../databases';
+import { promptImportLgtmDatabase, importArchiveDatabase, promptImportInternetDatabase } from '../../databaseFetcher';
+import { ProgressCallback } from '../../commandRunner';
+import { dbLoc, DB_URL, storagePath } from './global.helper';
+
+/**
+ * Run various integration tests for databases
+ */
+describe('Databases', function() {
+  this.timeout(60000);
+
+  const LGTM_URL = 'https://lgtm.com/projects/g/aeisenberg/angular-bind-notifier/';
+
+  let databaseManager: DatabaseManager;
+  let sandbox: sinon.SinonSandbox;
+  let inputBoxStub: sinon.SinonStub;
+  let progressCallback: ProgressCallback;
+
+  beforeEach(async () => {
+    try {
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+      if ('databaseManager' in extension) {
+        databaseManager = extension.databaseManager;
+      } else {
+        throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+      }
+
+      sandbox = sinon.createSandbox();
+      // the uri.fsPath function on windows returns a lowercase drive letter
+      // so, force the storage path string to be lowercase, too.
+      progressCallback = sandbox.spy();
+      inputBoxStub = sandbox.stub(window, 'showInputBox');
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  afterEach(() => {
+    try {
+      sandbox.restore();
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should add a database from a folder', async () => {
+    const progressCallback = sandbox.spy() as ProgressCallback;
+    const uri = Uri.file(dbLoc);
+    let dbItem = await importArchiveDatabase(uri.toString(true), databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    expect(dbItem).to.be.eq(databaseManager.currentDatabaseItem);
+    expect(dbItem).to.be.eq(databaseManager.databaseItems[0]);
+    expect(dbItem).not.to.be.undefined;
+    dbItem = dbItem!;
+    expect(dbItem.name).to.eq('db');
+    expect(dbItem.databaseUri.fsPath).to.eq(path.join(storagePath, 'db', 'db'));
+  });
+
+  it('should add a database from lgtm with only one language', async () => {
+    inputBoxStub.resolves(LGTM_URL);
+    let dbItem = await promptImportLgtmDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    expect(dbItem).not.to.be.undefined;
+    dbItem = dbItem!;
+    expect(dbItem.name).to.eq('aeisenberg_angular-bind-notifier_106179a');
+    expect(dbItem.databaseUri.fsPath).to.eq(path.join(storagePath, 'javascript', 'aeisenberg_angular-bind-notifier_106179a'));
+  });
+
+  it('should add a database from a url', async () => {
+    inputBoxStub.resolves(DB_URL);
+
+    let dbItem = await promptImportInternetDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    expect(dbItem).not.to.be.undefined;
+    dbItem = dbItem!;
+    expect(dbItem.name).to.eq('db');
+    expect(dbItem.databaseUri.fsPath).to.eq(path.join(storagePath, 'simple-db', 'db'));
+  });
+
+});

extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts

@@ -0,0 +1,86 @@
+import * as path from 'path';
+import * as tmp from 'tmp';
+import * as fs from 'fs-extra';
+import fetch from 'node-fetch';
+
+import { fail } from 'assert';
+import { ConfigurationTarget, extensions, workspace } from 'vscode';
+import { CodeQLExtensionInterface } from '../../extension';
+
+// This file contains helpers shared between actual tests.
+
+export const DB_URL = 'https://github.com/github/vscode-codeql/files/5586722/simple-db.zip';
+
+// We need to resolve the path, but the final three segments won't exist until later, so we only resolve the
+// first portion of the path.
+export const dbLoc = path.join(fs.realpathSync(path.join(__dirname, '../../../')), 'build/tests/db.zip');
+export let storagePath: string;
+
+export default function(mocha: Mocha) {
+  // create an extension storage location
+  let removeStorage: tmp.DirResult['removeCallback'] | undefined;
+
+  // ensure the test database is downloaded
+  (mocha.options as any).globalSetup.push(
+    async () => {
+      fs.mkdirpSync(path.dirname(dbLoc));
+      if (!fs.existsSync(dbLoc)) {
+        try {
+          await new Promise((resolve, reject) => {
+            return fetch(DB_URL).then(response => {
+              const dest = fs.createWriteStream(dbLoc);
+              response.body.pipe(dest);
+
+              response.body.on('error', reject);
+              dest.on('error', reject);
+              dest.on('close', () => {
+                resolve(dbLoc);
+              });
+            });
+          });
+        } catch (e) {
+          fail('Failed to download test database: ' + e);
+        }
+      }
+    }
+  );
+
+  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+  (mocha.options as any).globalSetup.push(
+    async () => {
+      await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
+    }
+  );
+
+  // Create the temp directory to be used as extension local storage.
+  (mocha.options as any).globalSetup.push(
+    () => {
+      const dir = tmp.dirSync();
+      storagePath = fs.realpathSync(dir.name);
+      if (storagePath.substring(0, 2).match(/[A-Z]:/)) {
+        storagePath = storagePath.substring(0, 1).toLocaleLowerCase() + storagePath.substring(1);
+      }
+
+      removeStorage = dir.removeCallback;
+    }
+  );
+
+  // ensure etension is cleaned up.
+  (mocha.options as any).globalTeardown.push(
+    async () => {
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+      // This shuts down the extension and can only be run after all tests have completed.
+      // If this is not called, then the test process will hang.
+      if ('dispose' in extension) {
+        extension.dispose();
+      }
+    }
+  );
+
+  // ensure temp directory is cleaned up.
+  (mocha.options as any).globalTeardown.push(
+    () => {
+      removeStorage?.();
+    }
+  );
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts

@@ -0,0 +1,8 @@
+import 'mocha';
+import 'sinon-chai';
+import { runTestsInDirectory } from '../index-template';
+
+// The simple database used throughout the tests
+export function run(): Promise<void> {
+  return runTestsInDirectory(__dirname, true);
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts

@@ -0,0 +1,176 @@
+import { fail } from 'assert';
+import { CancellationToken, commands, ExtensionContext, extensions, Uri } from 'vscode';
+import * as sinon from 'sinon';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import 'mocha';
+import { expect } from 'chai';
+import * as yaml from 'js-yaml';
+
+import { DatabaseItem, DatabaseManager } from '../../databases';
+import { CodeQLExtensionInterface } from '../../extension';
+import { dbLoc, storagePath } from './global.helper';
+import { importArchiveDatabase } from '../../databaseFetcher';
+import { compileAndRunQueryAgainstDatabase } from '../../run-queries';
+import { CodeQLCliServer } from '../../cli';
+import { QueryServerClient } from '../../queryserver-client';
+import { skipIfNoCodeQL } from '../ensureCli';
+import { QueryResultType } from '../../pure/messages';
+
+
+/**
+ * Integration tests for queries
+ */
+describe('Queries', function() {
+  this.timeout(20000);
+
+  before(function() {
+    skipIfNoCodeQL(this);
+  });
+
+  let dbItem: DatabaseItem;
+  let databaseManager: DatabaseManager;
+  let cli: CodeQLCliServer;
+  let qs: QueryServerClient;
+  let sandbox: sinon.SinonSandbox;
+  let progress: sinon.SinonSpy;
+  let token: CancellationToken;
+  let ctx: ExtensionContext;
+
+  let qlpackFile: string;
+  let qlFile: string;
+
+
+  beforeEach(async () => {
+    sandbox = sinon.createSandbox();
+
+    try {
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+      if ('databaseManager' in extension) {
+        databaseManager = extension.databaseManager;
+        cli = extension.cliServer;
+        qs = extension.qs;
+        cli.quiet = true;
+        ctx = extension.ctx;
+        qlpackFile = `${ctx.storagePath}/quick-queries/qlpack.yml`;
+        qlFile = `${ctx.storagePath}/quick-queries/quick-query.ql`;
+      } else {
+        throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+      }
+
+      progress = sandbox.spy();
+      token = {} as CancellationToken;
+
+      // Add a database
+      const uri = Uri.file(dbLoc);
+      const maybeDbItem = await importArchiveDatabase(
+        uri.toString(true),
+        databaseManager,
+        storagePath,
+        progress,
+        token
+      );
+
+      if (!maybeDbItem) {
+        throw new Error('Could not import database');
+      }
+      dbItem = maybeDbItem;
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  afterEach(() => {
+    try {
+      sandbox.restore();
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should run a query', async () => {
+    try {
+      const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
+      const result = await compileAndRunQueryAgainstDatabase(
+        cli,
+        qs,
+        dbItem,
+        false,
+        Uri.file(queryPath),
+        progress,
+        token
+      );
+
+      // just check that the query was successful
+      expect(result.database.name).to.eq('db');
+      expect(result.options.queryText).to.eq(fs.readFileSync(queryPath, 'utf8'));
+      expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
+    } catch (e) {
+      console.error('Test Failed');
+      fail(e);
+    }
+  });
+
+  // Asserts a fix for bug https://github.com/github/vscode-codeql/issues/733
+  it('should restart the database and run a query', async () => {
+    try {
+      await commands.executeCommand('codeQL.restartQueryServer');
+      const queryPath = path.join(__dirname, 'data', 'simple-query.ql');
+      const result = await compileAndRunQueryAgainstDatabase(
+        cli,
+        qs,
+        dbItem,
+        false,
+        Uri.file(queryPath),
+        progress,
+        token
+      );
+
+      // this message would indicate that the databases were not properly reregistered
+      expect(result.result.message).not.to.eq('No result from server');
+      expect(result.options.queryText).to.eq(fs.readFileSync(queryPath, 'utf8'));
+      expect(result.result.resultType).to.eq(QueryResultType.SUCCESS);
+    } catch (e) {
+      console.error('Test Failed');
+      fail(e);
+    }
+  });
+
+  it('should create a quick query', async () => {
+    safeDel(qlFile);
+    safeDel(qlpackFile);
+
+    await commands.executeCommand('codeQL.quickQuery');
+
+    // should have created the quick query file and query pack file
+    expect(fs.pathExistsSync(qlFile)).to.be.true;
+    expect(fs.pathExistsSync(qlpackFile)).to.be.true;
+
+    const qlpackContents: any = await yaml.safeLoad(
+      fs.readFileSync(qlpackFile, 'utf8')
+    );
+    // Should have chosen the js libraries
+    expect(qlpackContents.libraryPathDependencies[0]).to.eq('codeql-javascript');
+  });
+
+  it('should avoid creating a quick query', async () => {
+    fs.writeFileSync(qlpackFile, yaml.safeDump({
+      name: 'quick-query',
+      version: '1.0.0',
+      libraryPathDependencies: ['codeql-javascript']
+    }));
+    fs.writeFileSync(qlFile, 'xxx');
+    await commands.executeCommand('codeQL.quickQuery');
+
+    // should not have created the quick query file because database schema hasn't changed
+    expect(fs.readFileSync(qlFile, 'utf8')).to.eq('xxx');
+  });
+
+  function safeDel(file: string) {
+    try {
+      fs.unlinkSync(file);
+    } catch (e) {
+      // ignore
+    }
+  }
+});

extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts

@@ -8,14 +8,14 @@ import { CancellationTokenSource } from 'vscode-jsonrpc';
 import * as messages from '../../pure/messages';
 import * as qsClient from '../../queryserver-client';
 import * as cli from '../../cli';
-import { ProgressReporter, Logger } from '../../logging';
 import { ColumnValue } from '../../pure/bqrs-cli-types';
-import { FindDistributionResultKind } from '../../distribution';
+import { extensions } from 'vscode';
+import { CodeQLExtensionInterface } from '../../extension';
+import { fail } from 'assert';
+import { skipIfNoCodeQL } from '../ensureCli';
 
 
-declare module 'url' {
-  export function pathToFileURL(urlStr: string): Url;
-}
+const baseDir = path.join(__dirname, '../../../test/data');
 
 const tmpDir = tmp.dirSync({ prefix: 'query_test_', keep: false, unsafeCleanup: true });
 
@@ -33,7 +33,10 @@ class Checkpoint<T> {
   constructor() {
     this.res = () => { /**/ };
     this.rej = () => { /**/ };
-    this.promise = new Promise((res, rej) => { this.res = res; this.rej = rej; });
+    this.promise = new Promise((res, rej) => {
+      this.res = res as () => Record<string, never>;
+      this.rej = rej;
+    });
   }
 
   async done(): Promise<T> {
@@ -61,13 +64,19 @@ type QueryTestCase = {
 // Test cases: queries to run and their expected results.
 const queryTestCases: QueryTestCase[] = [
   {
-    queryPath: path.join(__dirname, '../data/query.ql'),
+    queryPath: path.join(baseDir, 'query.ql'),
     expectedResultSets: {
       '#select': [[42, 3.14159, 'hello world', true]]
     }
   },
   {
-    queryPath: path.join(__dirname, '../data/multiple-result-sets.ql'),
+    queryPath: path.join(baseDir, 'compute-default-strings.ql'),
+    expectedResultSets: {
+      '#select': [[{ label: '(no string representation)' }]]
+    }
+  },
+  {
+    queryPath: path.join(baseDir, 'multiple-result-sets.ql'),
     expectedResultSets: {
       'edges': [[1, 2], [2, 3]],
       '#select': [['s']]
@@ -75,73 +84,42 @@ const queryTestCases: QueryTestCase[] = [
   }
 ];
 
+const db: messages.Dataset = {
+  dbDir: path.join(__dirname, '../test-db'),
+  workingSet: 'default',
+};
+
 describe('using the query server', function() {
   before(function() {
-    if (process.env['CODEQL_PATH'] === undefined) {
-      console.log('The environment variable CODEQL_PATH is not set. The query server tests, which require the CodeQL CLI, will be skipped.');
-      this.skip();
-    }
+    skipIfNoCodeQL(this);
   });
 
   // Note this does not work with arrow functions as the test case bodies:
   // ensure they are all written with standard anonymous functions.
-  this.timeout(10000);
+  this.timeout(20000);
 
-  const codeQlPath = process.env['CODEQL_PATH']!;
   let qs: qsClient.QueryServerClient;
   let cliServer: cli.CodeQLCliServer;
   const queryServerStarted = new Checkpoint<void>();
-  after(() => {
-    if (qs) {
-      qs.dispose();
-    }
-    if (cliServer) {
-      cliServer.dispose();
+
+  beforeEach(async () => {
+    try {
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+      if ('cliServer' in extension && 'qs' in extension) {
+        cliServer = extension.cliServer;
+        qs = extension.qs;
+        cliServer.quiet = true;
+      } else {
+        throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+      }
+    } catch (e) {
+      fail(e);
     }
   });
 
   it('should be able to start the query server', async function() {
-    const consoleProgressReporter: ProgressReporter = {
-      report: (v: { message: string }) => console.log(`progress reporter says ${v.message}`)
-    };
-    const logger: Logger = {
-      log: async (s: string) => console.log('logger says', s),
-      show: () => { /**/ },
-      removeAdditionalLogLocation: async () => { /**/ },
-      getBaseLocation: () => ''
-    };
-    cliServer = new cli.CodeQLCliServer(
-      {
-        async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
-          return codeQlPath;
-        },
-        getDistribution: async () => {
-          return {
-            kind: FindDistributionResultKind.NoDistribution
-          };
-        }
-      },
-      {
-        numberTestThreads: 2
-      },
-      logger
-    );
-    qs = new qsClient.QueryServerClient(
-      {
-        codeQlPath,
-        numThreads: 1,
-        queryMemoryMb: 1024,
-        timeoutSecs: 1000,
-        debug: false
-      },
-      cliServer,
-      {
-        logger
-      },
-      task => task(consoleProgressReporter, token)
-    );
     await qs.startQueryServer();
-    queryServerStarted.resolve();
+    await queryServerStarted.resolve();
   });
 
   for (const queryTestCase of queryTestCases) {
@@ -150,13 +128,19 @@ describe('using the query server', function() {
     const evaluationSucceeded = new Checkpoint<void>();
     const parsedResults = new Checkpoint<void>();
 
+    it('should register the database if necessary', async () => {
+      if (await cliServer.cliConstraints.supportsDatabaseRegistration()) {
+        await qs.sendRequest(messages.registerDatabases, { databases: [db] }, token, (() => { /**/ }) as any);
+      }
+    });
+
     it(`should be able to compile query ${queryName}`, async function() {
       await queryServerStarted.done();
       expect(fs.existsSync(queryTestCase.queryPath)).to.be.true;
       try {
         const qlProgram: messages.QlProgram = {
           libraryPath: [],
-          dbschemePath: path.join(__dirname, '../data/test.dbscheme'),
+          dbschemePath: path.join(baseDir, 'test.dbscheme'),
           queryPath: queryTestCase.queryPath
         };
         const params: messages.CompileQueryParams = {
@@ -168,6 +152,7 @@ describe('using the query server', function() {
             localChecking: false,
             noComputeGetUrl: false,
             noComputeToString: false,
+            computeDefaultStrings: true
           },
           queryToCheck: qlProgram,
           resultPath: COMPILED_QUERY_PATH,
@@ -175,10 +160,10 @@ describe('using the query server', function() {
         };
         const result = await qs.sendRequest(messages.compileQuery, params, token, () => { /**/ });
         expect(result.messages!.length).to.equal(0);
-        compilationSucceeded.resolve();
+        await compilationSucceeded.resolve();
       }
       catch (e) {
-        compilationSucceeded.reject(e);
+        await compilationSucceeded.reject(e);
       }
     });
 
@@ -186,7 +171,7 @@ describe('using the query server', function() {
       try {
         await compilationSucceeded.done();
         const callbackId = qs.registerCallback(_res => {
-          evaluationSucceeded.resolve();
+          void evaluationSucceeded.resolve();
         });
         const queryToRun: messages.QueryToRun = {
           resultsPath: RESULTS_PATH,
@@ -195,21 +180,17 @@ describe('using the query server', function() {
           id: callbackId,
           timeoutSecs: 1000,
         };
-        const db: messages.Dataset = {
-          dbDir: path.join(__dirname, '../test-db'),
-          workingSet: 'default',
-        };
         const params: messages.EvaluateQueriesParams = {
           db,
           evaluateId: callbackId,
           queries: [queryToRun],
-          stopOnError: false,
+          stopOnError: true,
           useSequenceHint: false
         };
         await qs.sendRequest(messages.runQueries, params, token, () => { /**/ });
       }
       catch (e) {
-        evaluationSucceeded.reject(e);
+        await evaluationSucceeded.reject(e);
       }
     });
 
@@ -222,7 +203,7 @@ describe('using the query server', function() {
         const decoded = await cliServer.bqrsDecode(RESULTS_PATH, resultSet.name);
         actualResultSets[resultSet.name] = decoded.tuples;
       }
-      parsedResults.resolve();
+      await parsedResults.resolve();
     });
 
     it(`should have correct results for query ${queryName}`, async function() {

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -0,0 +1,53 @@
+import { expect } from 'chai';
+import { extensions } from 'vscode';
+import { SemVer } from 'semver';
+
+import { CodeQLCliServer } from '../../cli';
+import { CodeQLExtensionInterface } from '../../extension';
+import { skipIfNoCodeQL } from '../ensureCli';
+import { getOnDiskWorkspaceFolders } from '../../helpers';
+
+/**
+ * Perform proper integration tests by running the CLI
+ */
+describe('Use cli', function() {
+  this.timeout(60000);
+
+  let cli: CodeQLCliServer;
+
+  beforeEach(async () => {
+    const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+    if ('cliServer' in extension) {
+      cli = extension.cliServer;
+    } else {
+      throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+    }
+  });
+
+  it('should have the correct version of the cli', async () => {
+    expect(
+      (await cli.getVersion()).toString()
+    ).to.eq(
+      new SemVer(process.env.CLI_VERSION || '').toString()
+    );
+  });
+
+  it('should resolve ram', async () => {
+    const result = await (cli as any).resolveRam(8192);
+    expect(result).to.deep.eq([
+      '-J-Xmx4096M',
+      '--off-heap-ram=4096'
+    ]);
+  });
+
+  it('should resolve query packs', async function() {
+    skipIfNoCodeQL(this);
+    const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
+    // should have a bunch of qlpacks. just check that a few known ones exist
+    expect(qlpacks['codeql-cpp']).not.to.be.undefined;
+    expect(qlpacks['codeql-csharp']).not.to.be.undefined;
+    expect(qlpacks['codeql-java']).not.to.be.undefined;
+    expect(qlpacks['codeql-javascript']).not.to.be.undefined;
+    expect(qlpacks['codeql-python']).not.to.be.undefined;
+  });
+});

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -0,0 +1,153 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { DistributionManager, extractZipArchive, codeQlLauncherName } from '../distribution';
+import fetch from 'node-fetch';
+import { workspace } from 'vscode';
+
+/**
+ * This module ensures that the proper CLI is available for tests of the extension.
+ * There are two environment variables to control this module:
+ *
+ *     - CLI_VERSION: The version of the CLI to install. Defaults to the most recent
+ *       version. Note that for now, we must maintain the default version by hand.
+ *
+ *     - CLI_BASE_DIR: The base dir where the CLI will be downloaded and unzipped.
+ *       The download location is `${CLI_BASE_DIR}/assets` and the unzip loction is
+ *       `${CLI_BASE_DIR}/${CLI_VERSION}`
+ *
+ * After downloading and unzipping, a new environment variable is set:
+ *
+ *     - CLI_PATH: Points to the cli executable for the specified CLI_VERSION. This
+ *       is variable is available in the unit tests and will be used as the value
+ *       for `codeQL.cli.executablePath`.
+ *
+ * As an optimization, the cli will not be unzipped again if the executable already
+ * exists. And the cli will not be re-downloaded if the zip already exists.
+ */
+
+process.on('unhandledRejection', e => {
+  console.error('Unhandled rejection.');
+  console.error(e);
+  // Must use a setTimeout in order to ensure the log is fully flushed before exiting
+  setTimeout(() => {
+    process.exit(-1);
+  }, 2000);
+});
+
+const _1MB = 1024 * 1024;
+const _10MB = _1MB * 10;
+
+// CLI version to test. Hard code the latest as default. And be sure
+// to update the env if it is not otherwise set.
+const CLI_VERSION = process.env.CLI_VERSION || 'v2.5.5';
+process.env.CLI_VERSION = CLI_VERSION;
+
+// Base dir where CLIs will be downloaded into
+// By default, put it in the `build` directory in the root of the extension.
+const CLI_BASE_DIR = process.env.CLI_DIR || path.normalize(path.join(__dirname, '../../build/cli'));
+
+export async function ensureCli(useCli: boolean) {
+  try {
+    if (!useCli) {
+      console.log('Not downloading CLI. It is not being used.');
+      return;
+    }
+
+    const assetName = DistributionManager.getRequiredAssetName();
+    const url = getCliDownloadUrl(assetName);
+    const unzipDir = getCliUnzipDir();
+    const downloadedFilePath = getDownloadFilePath(assetName);
+    const executablePath = path.join(getCliUnzipDir(), 'codeql', codeQlLauncherName());
+
+    // Use this environment variable to se to the `codeQL.cli.executablePath` in tests
+    process.env.CLI_PATH = executablePath;
+
+    if (fs.existsSync(executablePath)) {
+      console.log(`CLI version ${CLI_VERSION} is found ${executablePath}. Not going to download again.`);
+      return;
+    }
+
+    if (!fs.existsSync(downloadedFilePath)) {
+      console.log(`CLI version ${CLI_VERSION} zip file not found. Downloading from '${url}' into '${downloadedFilePath}'.`);
+
+      const assetStream = await fetch(url);
+      const contentLength = Number(assetStream.headers.get('content-length') || 0);
+      console.log('Total content size', Math.round(contentLength / _1MB), 'MB');
+      const archiveFile = fs.createWriteStream(downloadedFilePath);
+      const body = assetStream.body;
+      await new Promise<void>((resolve, reject) => {
+        let numBytesDownloaded = 0;
+        let lastMessage = 0;
+        body.on('data', (data) => {
+          numBytesDownloaded += data.length;
+          if (numBytesDownloaded - lastMessage > _10MB) {
+            console.log('Downloaded', Math.round(numBytesDownloaded / _1MB), 'MB');
+            lastMessage = numBytesDownloaded;
+          }
+          archiveFile.write(data);
+        });
+        body.on('finish', () => {
+          archiveFile.end(() => {
+            console.log('Finished download into', downloadedFilePath);
+            resolve();
+          });
+        });
+        body.on('error', reject);
+      });
+    } else {
+      console.log(`CLI version ${CLI_VERSION} zip file found at '${downloadedFilePath}'.`);
+    }
+
+    console.log(`Unzipping into '${unzipDir}'`);
+    fs.mkdirpSync(unzipDir);
+    await extractZipArchive(downloadedFilePath, unzipDir);
+    console.log('Done.');
+  } catch (e) {
+    console.error('Failed to download CLI.');
+    console.error(e);
+    process.exit(-1);
+  }
+}
+
+/**
+ * Heuristically determines if the codeql libraries are installed in this
+ * workspace. Looks for the existance of a folder whose path ends in `/codeql`
+ */
+function hasCodeQL() {
+  const folders = workspace.workspaceFolders;
+  return !!folders?.some(folder => folder.uri.path.endsWith('/codeql'));
+}
+
+export function skipIfNoCodeQL(context: Mocha.Context) {
+  if (!hasCodeQL()) {
+    console.log([
+      'The CodeQL libraries are not available as a folder in this workspace.',
+      'To fix in CI: checkout the github/codeql repository and set the \'TEST_CODEQL_PATH\' environment variable to the checked out directory.',
+      'To fix when running from vs code, see the comment in the launch.json file in the \'Launch Integration Tests - With CLI\' section.'
+    ].join('\n\n'));
+    context.skip();
+  }
+}
+
+/**
+ * Url to download from
+ */
+function getCliDownloadUrl(assetName: string) {
+  return `https://github.com/github/codeql-cli-binaries/releases/download/${CLI_VERSION}/${assetName}`;
+}
+
+/**
+ * Directory to place the downloaded cli into
+ */
+function getDownloadFilePath(assetName: string) {
+  const dir = path.join(CLI_BASE_DIR, 'assets', CLI_VERSION);
+  fs.mkdirpSync(dir);
+  return path.join(dir, assetName);
+}
+
+/**
+ * Directory to unzip the downloaded cli into.
+ */
+function getCliUnzipDir() {
+  return path.join(CLI_BASE_DIR, CLI_VERSION);
+}

extensions/ql-vscode/src/vscode-tests/index-template.ts

@@ -1,6 +1,19 @@
 import * as path from 'path';
 import * as Mocha from 'mocha';
 import * as glob from 'glob';
+import { ensureCli } from './ensureCli';
+import { env } from 'vscode';
+
+
+// Use this handler to avoid swallowing unhandled rejections.
+process.on('unhandledRejection', e => {
+  console.error('Unhandled rejection.');
+  console.error(e);
+  // Must use a setTimeout in order to ensure the log is fully flushed before exiting
+  setTimeout(() => {
+    process.exit(-1);
+  }, 2000);
+});
 
 /**
  * Helper function that runs all Mocha tests found in the
@@ -26,35 +39,61 @@ import * as glob from 'glob';
  * After https://github.com/microsoft/TypeScript/issues/420 is implemented,
  * this pattern can be expressed more neatly using a module interface.
  */
-export function runTestsInDirectory(testsRoot: string): Promise<void> {
+export async function runTestsInDirectory(testsRoot: string, useCli = false): Promise<void> {
   // Create the mocha test
   const mocha = new Mocha({
     ui: 'bdd',
-    color: true
-  });
+    color: true,
+    globalSetup: [],
+    globalTeardown: [],
+  } as any);
 
-  return new Promise((c, e) => {
-    console.log(`Adding test cases from ${testsRoot}`);
-    glob('**/**.test.js', { cwd: testsRoot }, (err, files) => {
+  (mocha.options as any).globalSetup.push(
+    // convert this function into an noop since it should not run during tests.
+    // If it does run during tests, then it can cause some testing environments
+    // to hang.
+    (env as any).openExternal = () => { /**/ }
+  );
+
+  await ensureCli(useCli);
+
+  return new Promise((resolve, reject) => {
+    console.log(`Adding test cases and helpers from ${testsRoot}`);
+    glob('**/**.js', { cwd: testsRoot }, (err, files) => {
       if (err) {
-        return e(err);
+        return reject(err);
       }
 
-
-      // Add files to the test suite
-      files.forEach(f => mocha.addFile(path.resolve(testsRoot, f)));
-
       try {
+        // Add test files to the test suite
+        files
+          .filter(f => f.endsWith('.test.js'))
+          .forEach(f => {
+            console.log(`Adding test file ${f}`);
+            mocha.addFile(path.resolve(testsRoot, f));
+          });
+
+        // Add helpers. Helper files add global setup and teardown blocks
+        // for a test run.
+        files
+          .filter(f => f.endsWith('.helper.js'))
+          .forEach(f => {
+            console.log(`Executing helper ${f}`);
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            const helper = require(path.resolve(testsRoot, f)).default;
+            helper(mocha);
+          });
+
         // Run the mocha test
         mocha.run(failures => {
           if (failures > 0) {
-            e(new Error(`${failures} tests failed.`));
+            reject(new Error(`${failures} tests failed.`));
           } else {
-            c();
+            resolve();
           }
         });
       } catch (err) {
-        e(err);
+        reject(err);
       }
     });
   });

extensions/ql-vscode/src/vscode-tests/minimal-workspace/activation.test.ts

@@ -9,6 +9,7 @@ import * as determiningSelectedQueryTest from './determining-selected-query-test
 chai.use(chaiAsPromised);
 
 describe('launching with a minimal workspace', async () => {
+
   const ext = vscode.extensions.getExtension('GitHub.vscode-codeql');
   it('should install the extension', () => {
     assert(ext);
@@ -19,6 +20,9 @@ describe('launching with a minimal workspace', async () => {
   });
 
   it('should activate the extension when a .ql file is opened', async function() {
+    this.timeout(60000);
+    await delay();
+
     const folders = vscode.workspace.workspaceFolders;
     assert(folders && folders.length === 1);
     const folderPath = folders![0].uri.fsPath;
@@ -26,10 +30,13 @@ describe('launching with a minimal workspace', async () => {
     const document = await vscode.workspace.openTextDocument(documentPath);
     assert(document.languageId === 'ql');
     // Delay slightly so that the extension has time to activate.
-    this.timeout(4000);
-    await new Promise(resolve => setTimeout(resolve, 2000));
+    await delay();
     assert(ext!.isActive);
   });
+
+  async function delay() {
+    await new Promise(resolve => setTimeout(resolve, 4000));
+  }
 });
 
 determiningSelectedQueryTest.run();

extensions/ql-vscode/src/vscode-tests/minimal-workspace/config.test.ts

@@ -15,7 +15,10 @@ import {
 chai.use(chaiAsPromised);
 const expect = chai.expect;
 
-describe('config listeners', () => {
+describe('config listeners', function() {
+  // Because we are adding some extra waiting, need to bump the test timeouts.
+  this.timeout(5000);
+
   let sandbox: Sinon.SinonSandbox;
   beforeEach(() => {
     sandbox = Sinon.createSandbox();
@@ -26,7 +29,7 @@ describe('config listeners', () => {
   });
 
   interface TestConfig<T> {
-    clazz: new() => {};
+    clazz: new () => unknown;
     settings: {
       name: string;
       property: string;
@@ -38,6 +41,10 @@ describe('config listeners', () => {
     {
       clazz: CliConfigListener,
       settings: [{
+        name: 'codeQL.runningQueries.numberOfThreads',
+        property: 'numberThreads',
+        values: [0, 1]
+      }, {
         name: 'codeQL.runningTests.numberOfThreads',
         property: 'numberTestThreads',
         values: [1, 0]
@@ -57,6 +64,14 @@ describe('config listeners', () => {
         name: 'codeQL.runningQueries.numberOfThreads',
         property: 'numThreads',
         values: [0, 1]
+      }, {
+        name: 'codeQL.runningQueries.saveCache',
+        property: 'saveCache',
+        values: [false, true]
+      }, {
+        name: 'codeQL.runningQueries.cacheSize',
+        property: 'cacheSize',
+        values: [0, 1]
       }, {
         name: 'codeQL.runningQueries.memory',
         property: 'queryMemoryMb',
@@ -84,19 +99,31 @@ describe('config listeners', () => {
         beforeEach(async () => {
           origValue = workspace.getConfiguration().get(setting.name);
           await workspace.getConfiguration().update(setting.name, setting.values[0]);
+          await wait();
           spy.resetHistory();
         });
 
         afterEach(async () => {
           await workspace.getConfiguration().update(setting.name, origValue);
+          await wait();
         });
 
         it(`should listen for changes to '${setting.name}'`, async () => {
           await workspace.getConfiguration().update(setting.name, setting.values[1]);
-          expect(spy.calledOnce).to.be.true;
+          await wait();
           expect(listener[setting.property]).to.eq(setting.values[1]);
+          expect(spy).to.have.been.calledOnce;
         });
       });
     });
   });
+
+  // Need to wait some time since the onDidChangeConfiguration listeners fire
+  // asynchronously and we sometimes need to wait for them to complete in
+  // order to have as successful test.
+  async function wait(ms = 50) {
+    return new Promise(resolve =>
+      setTimeout(resolve, ms)
+    );
+  }
 });

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -0,0 +1,508 @@
+import 'vscode-test';
+import 'mocha';
+import * as sinon from 'sinon';
+import * as tmp from 'tmp';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { expect } from 'chai';
+import { CancellationToken, ExtensionContext, Uri, workspace } from 'vscode';
+
+import {
+  DatabaseEventKind,
+  DatabaseManager,
+  DatabaseItemImpl,
+  DatabaseContents,
+  FullDatabaseOptions
+} from '../../databases';
+import { Logger } from '../../logging';
+import { QueryServerClient } from '../../queryserver-client';
+import { registerDatabases } from '../../pure/messages';
+import { ProgressCallback } from '../../commandRunner';
+import { CodeQLCliServer } from '../../cli';
+import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
+import { testDisposeHandler } from '../test-dispose-handler';
+
+describe('databases', () => {
+
+  const MOCK_DB_OPTIONS: FullDatabaseOptions = {
+    dateAdded: 123,
+    ignoreSourceArchive: false,
+    language: ''
+  };
+
+  let databaseManager: DatabaseManager;
+  let updateSpy: sinon.SinonSpy;
+  let getSpy: sinon.SinonStub;
+  let dbChangedHandler: sinon.SinonSpy;
+  let sendRequestSpy: sinon.SinonSpy;
+  let supportsDatabaseRegistrationSpy: sinon.SinonStub;
+  let supportsLanguageNameSpy: sinon.SinonStub;
+  let resolveDatabaseSpy: sinon.SinonStub;
+
+  let sandbox: sinon.SinonSandbox;
+  let dir: tmp.DirResult;
+
+  beforeEach(() => {
+    dir = tmp.dirSync();
+
+    sandbox = sinon.createSandbox();
+    updateSpy = sandbox.spy();
+    getSpy = sandbox.stub();
+    getSpy.returns([]);
+    sendRequestSpy = sandbox.stub();
+    dbChangedHandler = sandbox.spy();
+    supportsDatabaseRegistrationSpy = sandbox.stub();
+    supportsDatabaseRegistrationSpy.resolves(true);
+    supportsLanguageNameSpy = sandbox.stub();
+    resolveDatabaseSpy = sandbox.stub();
+    databaseManager = new DatabaseManager(
+      {
+        workspaceState: {
+          update: updateSpy,
+          get: getSpy
+        },
+        // pretend like databases added in the temp dir are controlled by the extension
+        // so that they are deleted upon removal
+        storagePath: dir.name
+      } as unknown as ExtensionContext,
+      {
+        sendRequest: sendRequestSpy,
+        onDidStartQueryServer: () => { /**/ }
+      } as unknown as QueryServerClient,
+      {
+        cliConstraints: {
+          supportsLanguageName: supportsLanguageNameSpy,
+          supportsDatabaseRegistration: supportsDatabaseRegistrationSpy,
+        },
+        resolveDatabase: resolveDatabaseSpy
+      } as unknown as CodeQLCliServer,
+      {} as Logger,
+    );
+
+    // Unfortunately, during a test it is not possible to convert from
+    // a single root workspace to multi-root, so must stub out relevant
+    // functions
+    sandbox.stub(workspace, 'updateWorkspaceFolders');
+    sandbox.spy(workspace, 'onDidChangeWorkspaceFolders');
+  });
+
+  afterEach(async () => {
+    dir.removeCallback();
+    databaseManager.dispose(testDisposeHandler);
+    sandbox.restore();
+  });
+
+  it('should fire events when adding and removing a db item', async () => {
+    const mockDbItem = createMockDB();
+    const spy = sinon.spy();
+    databaseManager.onDidChangeDatabaseItem(spy);
+    await (databaseManager as any).addDatabaseItem(
+      {} as ProgressCallback,
+      {} as CancellationToken,
+      mockDbItem
+    );
+
+    expect((databaseManager as any)._databaseItems).to.deep.eq([mockDbItem]);
+    expect(updateSpy).to.have.been.calledWith('databaseList', [{
+      options: MOCK_DB_OPTIONS,
+      uri: dbLocationUri().toString(true)
+    }]);
+    expect(spy).to.have.been.calledWith({
+      item: undefined,
+      kind: DatabaseEventKind.Add
+    });
+
+    sinon.reset();
+
+    // now remove the item
+    await databaseManager.removeDatabaseItem(
+      {} as ProgressCallback,
+      {} as CancellationToken,
+      mockDbItem,
+    );
+    expect((databaseManager as any)._databaseItems).to.deep.eq([]);
+    expect(updateSpy).to.have.been.calledWith('databaseList', []);
+    expect(spy).to.have.been.calledWith({
+      item: undefined,
+      kind: DatabaseEventKind.Remove
+    });
+  });
+
+  it('should rename a db item and emit an event', async () => {
+    const mockDbItem = createMockDB();
+    const spy = sinon.spy();
+    databaseManager.onDidChangeDatabaseItem(spy);
+    await (databaseManager as any).addDatabaseItem(
+      {} as ProgressCallback,
+      {} as CancellationToken,
+      mockDbItem
+    );
+
+    sinon.restore();
+
+    await databaseManager.renameDatabaseItem(mockDbItem, 'new name');
+
+    expect(mockDbItem.name).to.eq('new name');
+    expect(updateSpy).to.have.been.calledWith('databaseList', [{
+      options: { ...MOCK_DB_OPTIONS, displayName: 'new name' },
+      uri: dbLocationUri().toString(true)
+    }]);
+
+    expect(spy).to.have.been.calledWith({
+      item: undefined,
+      kind: DatabaseEventKind.Rename
+    });
+  });
+
+  describe('add / remove database items', () => {
+    it('should add a database item', async () => {
+      const spy = sandbox.spy();
+      databaseManager.onDidChangeDatabaseItem(spy);
+      const mockDbItem = createMockDB();
+
+      await (databaseManager as any).addDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+
+      expect(databaseManager.databaseItems).to.deep.eq([mockDbItem]);
+      expect(updateSpy).to.have.been.calledWith('databaseList', [{
+        uri: dbLocationUri().toString(true),
+        options: MOCK_DB_OPTIONS
+      }]);
+
+      const mockEvent = {
+        item: undefined,
+        kind: DatabaseEventKind.Add
+      };
+      expect(spy).to.have.been.calledWith(mockEvent);
+    });
+
+    it('should add a database item source archive', async function () {
+      const mockDbItem = createMockDB();
+      mockDbItem.name = 'xxx';
+      await (databaseManager as any).addDatabaseSourceArchiveFolder(mockDbItem);
+
+      // workspace folders should be updated. We can only check the mocks since
+      // when running as a test, we are not allowed to update the workspace folders
+      expect(workspace.updateWorkspaceFolders).to.have.been.calledWith(1, 0, {
+        name: '[xxx source archive]',
+        // must use a matcher here since vscode URIs with the same path
+        // are not always equal due to internal state.
+        uri: sinon.match.has('fsPath', encodeArchiveBasePath(sourceLocationUri().fsPath).fsPath)
+      });
+    });
+
+    it('should remove a database item', async () => {
+      const mockDbItem = createMockDB();
+      sandbox.stub(fs, 'remove').resolves();
+
+      // pretend that this item is the first workspace folder in the list
+      sandbox.stub(mockDbItem, 'belongsToSourceArchiveExplorerUri').returns(true);
+
+      await (databaseManager as any).addDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+
+      updateSpy.resetHistory();
+
+      await databaseManager.removeDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+
+      expect(databaseManager.databaseItems).to.deep.eq([]);
+      expect(updateSpy).to.have.been.calledWith('databaseList', []);
+      // should remove the folder
+      expect(workspace.updateWorkspaceFolders).to.have.been.calledWith(0, 1);
+
+      // should also delete the db contents
+      expect(fs.remove).to.have.been.calledWith(mockDbItem.databaseUri.fsPath);
+    });
+
+    it('should remove a database item outside of the extension controlled area', async () => {
+      const mockDbItem = createMockDB();
+      sandbox.stub(fs, 'remove').resolves();
+
+      // pretend that this item is the first workspace folder in the list
+      sandbox.stub(mockDbItem, 'belongsToSourceArchiveExplorerUri').returns(true);
+
+      await (databaseManager as any).addDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+      updateSpy.resetHistory();
+
+      // pretend that the database location is not controlled by the extension
+      (databaseManager as any).ctx.storagePath = 'hucairz';
+
+      await databaseManager.removeDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+
+      expect(databaseManager.databaseItems).to.deep.eq([]);
+      expect(updateSpy).to.have.been.calledWith('databaseList', []);
+      // should remove the folder
+      expect(workspace.updateWorkspaceFolders).to.have.been.calledWith(0, 1);
+
+      // should NOT delete the db contents
+      expect(fs.remove).not.to.have.been.called;
+    });
+
+    it('should register and deregister a database when adding and removing it', async () => {
+      // similar test as above, but also check the call to sendRequestSpy to make sure they send the
+      // registration messages.
+      const mockDbItem = createMockDB();
+      const registration = {
+        databases: [{
+          dbDir: mockDbItem.contents!.datasetUri.fsPath,
+          workingSet: 'default'
+        }]
+      };
+
+      sandbox.stub(fs, 'remove').resolves();
+
+      await (databaseManager as any).addDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+      // Should have registered this database
+      expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {});
+
+      await databaseManager.removeDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+
+      // Should have deregistered this database
+      expect(sendRequestSpy).to.have.been.calledWith(registerDatabases, registration, {}, {});
+    });
+
+    it('should avoid registration when query server does not support it', async () => {
+      // similar test as above, but now pretend query server doesn't support registration
+      supportsDatabaseRegistrationSpy.resolves(false);
+      const mockDbItem = createMockDB();
+      sandbox.stub(fs, 'remove').resolves();
+
+      await (databaseManager as any).addDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+      // Should NOT have registered this database
+      expect(sendRequestSpy).not.to.have.been.called;
+
+      await databaseManager.removeDatabaseItem(
+        {} as ProgressCallback,
+        {} as CancellationToken,
+        mockDbItem
+      );
+
+      // Should NOT have deregistered this database
+      expect(sendRequestSpy).not.to.have.been.called;
+    });
+  });
+
+  describe('resolveSourceFile', () => {
+    it('should fail to resolve when not a uri', () => {
+      const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
+      (db as any)._contents.sourceArchiveUri = undefined;
+      expect(() => db.resolveSourceFile('abc')).to.throw('Scheme is missing');
+    });
+
+    it('should fail to resolve when not a file uri', () => {
+      const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
+      (db as any)._contents.sourceArchiveUri = undefined;
+      expect(() => db.resolveSourceFile('http://abc')).to.throw('Invalid uri scheme');
+    });
+
+    describe('no source archive', () => {
+      it('should resolve undefined', () => {
+        const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
+        (db as any)._contents.sourceArchiveUri = undefined;
+        const resolved = db.resolveSourceFile(undefined);
+        expect(resolved.toString(true)).to.eq(dbLocationUri().toString(true));
+      });
+
+      it('should resolve an empty file', () => {
+        const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
+        (db as any)._contents.sourceArchiveUri = undefined;
+        const resolved = db.resolveSourceFile('file:');
+        expect(resolved.toString()).to.eq('file:///');
+      });
+    });
+
+    describe('zipped source archive', () => {
+      it('should encode a source archive url', () => {
+        const db = createMockDB(encodeSourceArchiveUri({
+          sourceArchiveZipPath: 'sourceArchive-uri',
+          pathWithinSourceArchive: 'def'
+        }));
+        const resolved = db.resolveSourceFile(Uri.file('abc').toString());
+
+        // must recreate an encoded archive uri instead of typing out the
+        // text since the uris will be different on windows and ubuntu.
+        expect(resolved.toString()).to.eq(encodeSourceArchiveUri({
+          sourceArchiveZipPath: 'sourceArchive-uri',
+          pathWithinSourceArchive: 'def/abc'
+        }).toString());
+      });
+
+      it('should encode a source archive url with trailing slash', () => {
+        const db = createMockDB(encodeSourceArchiveUri({
+          sourceArchiveZipPath: 'sourceArchive-uri',
+          pathWithinSourceArchive: 'def/'
+        }));
+        const resolved = db.resolveSourceFile(Uri.file('abc').toString());
+
+        // must recreate an encoded archive uri instead of typing out the
+        // text since the uris will be different on windows and ubuntu.
+        expect(resolved.toString()).to.eq(encodeSourceArchiveUri({
+          sourceArchiveZipPath: 'sourceArchive-uri',
+          pathWithinSourceArchive: 'def/abc'
+        }).toString());
+      });
+
+      it('should encode an empty source archive url', () => {
+        const db = createMockDB(encodeSourceArchiveUri({
+          sourceArchiveZipPath: 'sourceArchive-uri',
+          pathWithinSourceArchive: 'def'
+        }));
+        const resolved = db.resolveSourceFile('file:');
+        expect(resolved.toString()).to.eq('codeql-zip-archive://1-18/sourceArchive-uri/def/');
+      });
+    });
+
+    it('should handle an empty file', () => {
+      const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
+      const resolved = db.resolveSourceFile('');
+      expect(resolved.toString()).to.eq('file:///sourceArchive-uri/');
+    });
+  });
+
+  it('should not support the primary language', async () => {
+    supportsLanguageNameSpy.resolves(false);
+
+    const result = (await (databaseManager as any).getPrimaryLanguage('hucairz'));
+    expect(result).to.be.undefined;
+  });
+
+  it('should get the primary language', async () => {
+    supportsLanguageNameSpy.resolves(true);
+    resolveDatabaseSpy.resolves({
+      languages: ['python']
+    });
+    const result = (await (databaseManager as any).getPrimaryLanguage('hucairz'));
+    expect(result).to.eq('python');
+  });
+
+  it('should handle missing the primary language', async () => {
+    supportsLanguageNameSpy.resolves(true);
+    resolveDatabaseSpy.resolves({
+      languages: []
+    });
+    const result = (await (databaseManager as any).getPrimaryLanguage('hucairz'));
+    expect(result).to.eq('');
+  });
+
+  describe('isAffectedByTest', () => {
+    const directoryStats = new fs.Stats();
+    const fileStats = new fs.Stats();
+
+    before(() => {
+      sinon.stub(directoryStats, 'isDirectory').returns(true);
+      sinon.stub(fileStats, 'isDirectory').returns(false);
+    });
+
+    it('should return true for testproj database in test directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.true;
+    });
+
+    it('should return false for non-existent test directory', async () => {
+      sandbox.stub(fs, 'stat').throws('Simulated Error: ENOENT');
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.false;
+    });
+
+    it('should return false for non-testproj database in test directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.proj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.false;
+    });
+
+    it('should return false for testproj database outside test directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/other/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.false;
+    });
+
+    it('should return false for testproj database for prefix directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      // /path/to/d is a prefix of /path/to/dir/dir.testproj, but
+      // /path/to/dir/dir.testproj is not under /path/to/d
+      expect(await db.isAffectedByTest('/path/to/d')).to.false;
+    });
+
+    it('should return true for testproj database for test file', async () => {
+      sandbox.stub(fs, 'stat').resolves(fileStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir/test.ql')).to.true;
+    });
+
+    it('should return false for non-existent test file', async () => {
+      sandbox.stub(fs, 'stat').throws('Simulated Error: ENOENT');
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir/test.ql')).to.false;
+    });
+
+    it('should return false for non-testproj database for test file', async () => {
+      sandbox.stub(fs, 'stat').resolves(fileStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.proj'));
+      expect(await db.isAffectedByTest('/path/to/dir/test.ql')).to.false;
+    });
+
+    it('should return false for testproj database not matching test file', async () => {
+      sandbox.stub(fs, 'stat').resolves(fileStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/test.ql')).to.false;
+    });
+  });
+
+  function createMockDB(
+    // source archive location must be a real(-ish) location since
+    // tests will add this to the workspace location
+    sourceArchiveUri = sourceLocationUri(),
+    databaseUri = dbLocationUri()
+  ): DatabaseItemImpl {
+
+    return new DatabaseItemImpl(
+      databaseUri,
+      {
+        sourceArchiveUri,
+        datasetUri: databaseUri
+      } as DatabaseContents,
+      MOCK_DB_OPTIONS,
+      dbChangedHandler,
+    );
+  }
+
+  function sourceLocationUri() {
+    return Uri.file(path.join(dir.name, 'src.zip'));
+  }
+
+  function dbLocationUri() {
+    return Uri.file(path.join(dir.name, 'db'));
+  }
+});

extensions/ql-vscode/src/vscode-tests/minimal-workspace/index.ts

@@ -1,4 +1,12 @@
 import { runTestsInDirectory } from '../index-template';
+
+import * as sinonChai from 'sinon-chai';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+chai.use(chaiAsPromised);
+chai.use(sinonChai);
+
+
 export function run(): Promise<void> {
   return runTestsInDirectory(__dirname);
 }

extensions/ql-vscode/src/vscode-tests/minimal-workspace/qltest-discovery.test.ts

@@ -2,20 +2,27 @@ import 'vscode-test';
 import 'mocha';
 import { Uri, WorkspaceFolder } from 'vscode';
 import { expect } from 'chai';
+import * as fs from 'fs-extra';
+import * as sinon from 'sinon';
 
 import { QLTestDiscovery } from '../../qltest-discovery';
 
 describe('qltest-discovery', () => {
   describe('discoverTests', () => {
-    it('should run discovery', async () => {
-      const baseUri = Uri.parse('file:/a/b');
-      const baseDir = baseUri.fsPath;
-      const cDir = Uri.parse('file:/a/b/c').fsPath;
-      const dFile = Uri.parse('file:/a/b/c/d.ql').fsPath;
-      const eFile = Uri.parse('file:/a/b/c/e.ql').fsPath;
-      const hDir = Uri.parse('file:/a/b/c/f/g/h').fsPath;
-      const iFile = Uri.parse('file:/a/b/c/f/g/h/i.ql').fsPath;
-      const qlTestDiscover = new QLTestDiscovery(
+    let sandbox: sinon.SinonSandbox;
+
+    const baseUri = Uri.parse('file:/a/b');
+    const baseDir = baseUri.fsPath;
+    const cDir = Uri.parse('file:/a/b/c').fsPath;
+    const dFile = Uri.parse('file:/a/b/c/d.ql').fsPath;
+    const eFile = Uri.parse('file:/a/b/c/e.ql').fsPath;
+    const hDir = Uri.parse('file:/a/b/c/f/g/h').fsPath;
+    const iFile = Uri.parse('file:/a/b/c/f/g/h/i.ql').fsPath;
+    let qlTestDiscover: QLTestDiscovery;
+
+    beforeEach(() => {
+      sandbox = sinon.createSandbox();
+      qlTestDiscover = new QLTestDiscovery(
         {
           uri: baseUri,
           name: 'My tests'
@@ -31,6 +38,15 @@ describe('qltest-discovery', () => {
         } as any
       );
 
+    });
+
+    afterEach(() => {
+      sandbox.restore();
+    });
+
+    it('should run discovery', async () => {
+      sandbox.stub(fs, 'pathExists').resolves(true);
+
       const result = await (qlTestDiscover as any).discover();
       expect(result.watchPath).to.eq(baseDir);
       expect(result.testDirectory.path).to.eq(baseDir);
@@ -56,5 +72,15 @@ describe('qltest-discovery', () => {
       expect(children[0].path).to.eq(iFile);
       expect(children[0].name).to.eq('i.ql');
     });
+
+    it('should avoid discovery if a folder does not exist', async () => {
+      sandbox.stub(fs, 'pathExists').resolves(false);
+      const result = await (qlTestDiscover as any).discover();
+      expect(result.watchPath).to.eq(baseDir);
+      expect(result.testDirectory.path).to.eq(baseDir);
+      expect(result.testDirectory.name).to.eq('My tests');
+
+      expect(result.testDirectory.children.length).to.eq(0);
+    });
   });
 });

extensions/ql-vscode/src/vscode-tests/no-workspace/astViewer.test.ts

@@ -7,6 +7,7 @@ import * as yaml from 'js-yaml';
 import { AstViewer, AstItem } from '../../astViewer';
 import { commands, Range } from 'vscode';
 import { DatabaseItem } from '../../databases';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 chai.use(chaiAsPromised);
 const expect = chai.expect;
@@ -15,19 +16,25 @@ const expect = chai.expect;
 
 describe('AstViewer', () => {
   let astRoots: AstItem[];
-  let viewer: AstViewer;
+  let viewer: AstViewer | undefined;
+  let sandbox: sinon.SinonSandbox;
   beforeEach(async () => {
+    sandbox = sinon.createSandbox();
     // the ast is stored in yaml because there are back pointers
     // making a json representation impossible.
     // The complication here is that yaml files are not copied into the 'out' directory by tsc.
     astRoots = await buildAst();
 
-    sinon.stub(commands, 'registerCommand');
-    sinon.stub(commands, 'executeCommand');
+    sandbox.stub(commands, 'registerCommand');
+    sandbox.stub(commands, 'executeCommand');
   });
 
   afterEach(() => {
-    sinon.restore();
+    sandbox.restore();
+    if (viewer) {
+      viewer.dispose(testDisposeHandler);
+      viewer = undefined;
+    }
   });
 
   it('should update the viewer roots', () => {
@@ -56,7 +63,6 @@ describe('AstViewer', () => {
     doSelectionTest(undefined, new Range(2, 3, 4, 5));
   });
 
-
   function doSelectionTest(
     expectedSelection: any,
     selectionRange: Range | undefined,
@@ -65,7 +71,7 @@ describe('AstViewer', () => {
     const item = {} as DatabaseItem;
     viewer = new AstViewer();
     viewer.updateRoots(astRoots, item, fsPath);
-    const spy = sinon.spy();
+    const spy = sandbox.spy();
     (viewer as any).treeView.reveal = spy;
     Object.defineProperty((viewer as any).treeView, 'visible', {
       value: true

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts

@@ -19,24 +19,23 @@ This test uses an AST generated from this file (already BQRS-decoded in ../data/
 
 int interrupt_init(void)
 {
-	return 0;
+    return 0;
 }
 
 void enable_interrupts(void)
 {
-	return;
+    return;
 }
 int disable_interrupts(void)
 {
-	return 0;
+    return 0;
 }
  */
 
 
 describe('AstBuilder', () => {
   let mockCli: CodeQLCliServer;
-  let overrides: Record<string, object | undefined>;
-
+  let overrides: Record<string, Record<string, unknown> | undefined>;
 
   beforeEach(() => {
     mockCli = {
@@ -136,7 +135,7 @@ describe('AstBuilder', () => {
     };
 
     const astBuilder = createAstBuilder();
-    expect(astBuilder.getRoots()).to.be.rejectedWith('AST is invalid');
+    await expect(astBuilder.getRoots()).to.be.rejectedWith('AST is invalid');
   });
 
   function createAstBuilder() {

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts

@@ -16,7 +16,8 @@ const expect = chai.expect;
 describe('queryResolver', () => {
   let module: Record<string, Function>;
   let writeFileSpy: sinon.SinonSpy;
-  let resolveDatasetFolderSpy: sinon.SinonStub;
+  let getQlPackForDbschemeSpy: sinon.SinonStub;
+  let getPrimaryDbschemeSpy: sinon.SinonStub;
   let mockCli: Record<string, sinon.SinonStub>;
   beforeEach(() => {
     mockCli = {
@@ -60,7 +61,7 @@ describe('queryResolver', () => {
 
   describe('qlpackOfDatabase', () => {
     it('should get the qlpack of a database', async () => {
-      resolveDatasetFolderSpy.returns({ qlpack: 'my-qlpack' });
+      getQlPackForDbschemeSpy.resolves('my-qlpack');
       const db = {
         contents: {
           datasetUri: {
@@ -70,20 +71,22 @@ describe('queryResolver', () => {
       };
       const result = await module.qlpackOfDatabase(mockCli, db);
       expect(result).to.eq('my-qlpack');
-      expect(resolveDatasetFolderSpy).to.have.been.calledWith(mockCli, '/path/to/database');
+      expect(getPrimaryDbschemeSpy).to.have.been.calledWith('/path/to/database');
     });
   });
 
   function createModule() {
     writeFileSpy = sinon.spy();
-    resolveDatasetFolderSpy = sinon.stub();
+    getQlPackForDbschemeSpy = sinon.stub();
+    getPrimaryDbschemeSpy = sinon.stub();
     return proxyquire('../../../contextual/queryResolver', {
       'fs-extra': {
         writeFile: writeFileSpy
       },
 
       '../helpers': {
-        resolveDatasetFolder: resolveDatasetFolderSpy,
+        getQlPackForDbscheme: getQlPackForDbschemeSpy,
+        getPrimaryDbscheme: getPrimaryDbschemeSpy,
         getOnDiskWorkspaceFolders: () => ({}),
         showAndLogErrorMessage: () => ({})
       }

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -21,17 +21,19 @@ describe('databaseFetcher', function() {
   this.timeout(10000);
 
   describe('convertToDatabaseUrl', () => {
+    let sandbox: sinon.SinonSandbox;
     let quickPickSpy: sinon.SinonStub;
     beforeEach(() => {
-      quickPickSpy = sinon.stub(window, 'showQuickPick');
+      sandbox = sinon.createSandbox();
+      quickPickSpy = sandbox.stub(window, 'showQuickPick');
     });
 
     afterEach(() => {
-      (window.showQuickPick as sinon.SinonStub).restore();
+      sandbox.restore();
     });
 
     it('should convert a project url to a database url', async () => {
-      quickPickSpy.returns('javascript' as any);
+      quickPickSpy.resolves('javascript');
       const lgtmUrl = 'https://lgtm.com/projects/g/github/codeql';
       const dbUrl = await convertToDatabaseUrl(lgtmUrl);
 
@@ -43,7 +45,7 @@ describe('databaseFetcher', function() {
     });
 
     it('should convert a project url to a database url with extra path segments', async () => {
-      quickPickSpy.returns('python' as any);
+      quickPickSpy.resolves('python');
       const lgtmUrl =
         'https://lgtm.com/projects/g/github/codeql/subpage/subpage2?query=xxx';
       const dbUrl = await convertToDatabaseUrl(lgtmUrl);
@@ -53,10 +55,21 @@ describe('databaseFetcher', function() {
       );
     });
 
-    it('should fail on a nonexistant prohect', async () => {
-      quickPickSpy.returns('javascript' as any);
+    it('should convert a raw slug to a database url with extra path segments', async () => {
+      quickPickSpy.resolves('python');
+      const lgtmUrl =
+        'g/github/codeql';
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+
+      expect(dbUrl).to.equal(
+        'https://lgtm.com/api/v1.0/snapshots/1506465042581/python'
+      );
+    });
+
+    it('should fail on a nonexistent project', async () => {
+      quickPickSpy.resolves('javascript');
       const lgtmUrl = 'https://lgtm.com/projects/g/github/hucairz';
-      expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
+      await expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
     });
   });
 
@@ -69,6 +82,10 @@ describe('databaseFetcher', function() {
         .to.be.false;
       expect(looksLikeLgtmUrl('https://ww.lgtm.com/projects/g/github')).to.be
         .false;
+      expect(looksLikeLgtmUrl('g/github')).to.be
+        .false;
+      expect(looksLikeLgtmUrl('ggg/github/myproj')).to.be
+        .false;
     });
 
     it('should handle valid urls', () => {
@@ -84,6 +101,10 @@ describe('databaseFetcher', function() {
           'https://lgtm.com/projects/g/github/codeql/sub/pages?query=string'
         )
       ).to.be.true;
+      expect(looksLikeLgtmUrl('g/github/myproj')).to.be
+        .true;
+      expect(looksLikeLgtmUrl('git/github/myproj')).to.be
+        .true;
     });
   });
 

extensions/ql-vscode/src/vscode-tests/no-workspace/databases-ui.test.ts

@@ -7,6 +7,7 @@ import { expect } from 'chai';
 import { Uri } from 'vscode';
 
 import { DatabaseUI } from '../../databases-ui';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 describe('databases-ui', () => {
   describe('fixDbUri', () => {
@@ -68,7 +69,6 @@ describe('databases-ui', () => {
     const db5 = createDatabase(storageDir, 'db2-notimported-with-codeql-database.yml', 'cpp', 'codeql-database.yml');
 
     const databaseUI = new DatabaseUI(
-      {} as any,
       {
         databaseItems: [
           { databaseUri: Uri.file(db1) }
@@ -90,7 +90,7 @@ describe('databases-ui', () => {
     expect(fs.pathExistsSync(db4)).to.be.false;
     expect(fs.pathExistsSync(db5)).to.be.false;
 
-    databaseUI.dispose();
+    databaseUI.dispose(testDisposeHandler);
   });
 
   function createDatabase(storageDir: string, dbName: string, language: string, extraFile?: string) {

extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts

@@ -1,188 +0,0 @@
-import 'vscode-test';
-import 'mocha';
-import * as sinon from 'sinon';
-import { expect } from 'chai';
-import { ExtensionContext, Uri } from 'vscode';
-
-import {
-  DatabaseEventKind,
-  DatabaseItem,
-  DatabaseManager,
-  DatabaseItemImpl,
-  DatabaseContents,
-  isLikelyDbLanguageFolder
-} from '../../databases';
-import { QueryServerConfig } from '../../config';
-import { Logger } from '../../logging';
-import { encodeSourceArchiveUri } from '../../archive-filesystem-provider';
-
-describe('databases', () => {
-  let databaseManager: DatabaseManager;
-  let updateSpy: sinon.SinonSpy;
-
-  beforeEach(() => {
-    updateSpy = sinon.spy();
-    databaseManager = new DatabaseManager(
-      {
-        workspaceState: {
-          update: updateSpy,
-          get: sinon.stub()
-        },
-      } as unknown as ExtensionContext,
-      {} as QueryServerConfig,
-      {} as Logger,
-    );
-  });
-
-  it('should fire events when adding and removing a db item', () => {
-    const mockDbItem = {
-      databaseUri: { path: 'file:/abc' },
-      name: 'abc',
-      getPersistedState() {
-        return this.name;
-      }
-    };
-    const spy = sinon.spy();
-    databaseManager.onDidChangeDatabaseItem(spy);
-    (databaseManager as any).addDatabaseItem(mockDbItem);
-
-    expect((databaseManager as any)._databaseItems).to.deep.eq([mockDbItem]);
-    expect(updateSpy).to.have.been.calledWith('databaseList', ['abc']);
-    expect(spy).to.have.been.calledWith({
-      item: undefined,
-      kind: DatabaseEventKind.Add
-    });
-
-    sinon.reset();
-
-    // now remove the item
-    databaseManager.removeDatabaseItem(mockDbItem as unknown as DatabaseItem);
-    expect((databaseManager as any)._databaseItems).to.deep.eq([]);
-    expect(updateSpy).to.have.been.calledWith('databaseList', []);
-    expect(spy).to.have.been.calledWith({
-      item: undefined,
-      kind: DatabaseEventKind.Remove
-    });
-  });
-
-  it('should rename a db item and emit an event', () => {
-    const mockDbItem = {
-      databaseUri: 'file:/abc',
-      name: 'abc',
-      getPersistedState() {
-        return this.name;
-      }
-    };
-    const spy = sinon.spy();
-    databaseManager.onDidChangeDatabaseItem(spy);
-    (databaseManager as any).addDatabaseItem(mockDbItem);
-    sinon.restore();
-
-    databaseManager.renameDatabaseItem(mockDbItem as unknown as DatabaseItem, 'new name');
-
-    expect(mockDbItem.name).to.eq('new name');
-    expect(updateSpy).to.have.been.calledWith('databaseList', ['new name']);
-    expect(spy).to.have.been.calledWith({
-      item: undefined,
-      kind: DatabaseEventKind.Rename
-    });
-  });
-
-  describe('resolveSourceFile', () => {
-    it('should fail to resolve when not a uri', () => {
-      const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
-      (db as any)._contents.sourceArchiveUri = undefined;
-      expect(() => db.resolveSourceFile('abc')).to.throw('Scheme is missing');
-    });
-
-    it('should fail to resolve when not a file uri', () => {
-      const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
-      (db as any)._contents.sourceArchiveUri = undefined;
-      expect(() => db.resolveSourceFile('http://abc')).to.throw('Invalid uri scheme');
-    });
-
-    describe('no source archive', () => {
-      it('should resolve undefined', () => {
-        const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
-        (db as any)._contents.sourceArchiveUri = undefined;
-        const resolved = db.resolveSourceFile(undefined);
-        expect(resolved.toString()).to.eq('file:///database-uri');
-      });
-
-      it('should resolve an empty file', () => {
-        const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
-        (db as any)._contents.sourceArchiveUri = undefined;
-        const resolved = db.resolveSourceFile('file:');
-        expect(resolved.toString()).to.eq('file:///');
-      });
-    });
-
-    describe('zipped source archive', () => {
-      it('should encode a source archive url', () => {
-        const db = createMockDB(encodeSourceArchiveUri({
-          sourceArchiveZipPath: 'sourceArchive-uri',
-          pathWithinSourceArchive: 'def'
-        }));
-        const resolved = db.resolveSourceFile(Uri.file('abc').toString());
-
-        // must recreate an encoded archive uri instead of typing out the
-        // text since the uris will be different on windows and ubuntu.
-        expect(resolved.toString()).to.eq(encodeSourceArchiveUri({
-          sourceArchiveZipPath: 'sourceArchive-uri',
-          pathWithinSourceArchive: 'def/abc'
-        }).toString());
-      });
-
-      it('should encode a source archive url with trailing slash', () => {
-        const db = createMockDB(encodeSourceArchiveUri({
-          sourceArchiveZipPath: 'sourceArchive-uri',
-          pathWithinSourceArchive: 'def/'
-        }));
-        const resolved = db.resolveSourceFile(Uri.file('abc').toString());
-
-        // must recreate an encoded archive uri instead of typing out the
-        // text since the uris will be different on windows and ubuntu.
-        expect(resolved.toString()).to.eq(encodeSourceArchiveUri({
-          sourceArchiveZipPath: 'sourceArchive-uri',
-          pathWithinSourceArchive: 'def/abc'
-        }).toString());
-      });
-
-      it('should encode an empty source archive url', () => {
-        const db = createMockDB(encodeSourceArchiveUri({
-          sourceArchiveZipPath: 'sourceArchive-uri',
-          pathWithinSourceArchive: 'def'
-        }));
-        const resolved = db.resolveSourceFile('file:');
-        expect(resolved.toString()).to.eq('codeql-zip-archive://1-18/sourceArchive-uri/def/');
-      });
-    });
-
-    it('should handle an empty file', () => {
-      const db = createMockDB(Uri.parse('file:/sourceArchive-uri/'));
-      const resolved = db.resolveSourceFile('');
-      expect(resolved.toString()).to.eq('file:///sourceArchive-uri/');
-    });
-    function createMockDB(
-      sourceArchiveUri = Uri.parse('file:/sourceArchive-uri'),
-      databaseUri = Uri.parse('file:/database-uri')
-    ) {
-      return new DatabaseItemImpl(
-        databaseUri,
-        {
-          sourceArchiveUri
-        } as DatabaseContents,
-        {
-          dateAdded: 123,
-          ignoreSourceArchive: false
-        },
-        () => { /**/ }
-      );
-    }
-  });
-
-  it('should find likely db language folders', () => {
-    expect(isLikelyDbLanguageFolder('db-javascript')).to.be.true;
-    expect(isLikelyDbLanguageFolder('dbnot-a-db')).to.be.false;
-  });
-});

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -190,10 +190,11 @@ describe('Launcher path', () => {
   let launcherThatExists = '';
 
   beforeEach(() => {
+    sandbox = sinon.createSandbox();
     getExecutableFromDirectory = createModule().getExecutableFromDirectory;
   });
 
-  beforeEach(() => {
+  afterEach(() => {
     sandbox.restore();
   });
 
@@ -214,7 +215,6 @@ describe('Launcher path', () => {
 
   it('should warn when using a hard-coded deprecated launcher name', async () => {
     launcherThatExists = 'codeql.cmd';
-    path.sep;
     const result = await getExecutableFromDirectory('abc');
     expect(fsSpy).to.have.been.calledWith(pathToExe);
     expect(fsSpy).to.have.been.calledWith(pathToCmd);
@@ -252,8 +252,12 @@ describe('Launcher path', () => {
     expect(result).to.equal(undefined);
   });
 
-  it('should not warn when deprecated launcher is used, but no new launcher is available', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+  it('should not warn when deprecated launcher is used, but no new launcher is available', async function() {
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
     launcherThatExists = 'codeql.cmd';
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -265,7 +269,11 @@ describe('Launcher path', () => {
   });
 
   it('should warn when deprecated launcher is used, and new launcher is available', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
     launcherThatExists = ''; // pretend both launchers exist
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -277,7 +285,11 @@ describe('Launcher path', () => {
   });
 
   it('should warn when launcher path is incorrect', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
     launcherThatExists = 'xxx'; // pretend neither launcher exists
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -289,7 +301,6 @@ describe('Launcher path', () => {
   });
 
   function createModule() {
-    sandbox = sinon.createSandbox();
     warnSpy = sandbox.spy();
     errorSpy = sandbox.spy();
     logSpy = sandbox.spy();

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -1,126 +1,376 @@
 import { expect } from 'chai';
 import 'mocha';
-import { ExtensionContext, Memento } from 'vscode';
-import { InvocationRateLimiter } from '../../helpers';
+import { EnvironmentVariableCollection, EnvironmentVariableMutator, Event, ExtensionContext, ExtensionMode, Memento, SecretStorage, SecretStorageChangeEvent, Uri, window } from 'vscode';
+import * as yaml from 'js-yaml';
+import * as tmp from 'tmp';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import * as sinon from 'sinon';
 
-describe('Invocation rate limiter', () => {
-  // 1 January 2020
-  let currentUnixTime = 1577836800;
+import {
+  getInitialQueryContents,
+  InvocationRateLimiter,
+  isLikelyDbLanguageFolder,
+  showBinaryChoiceDialog,
+  showBinaryChoiceWithUrlDialog,
+  showInformationMessageWithAction
+} from '../../helpers';
+import { reportStreamProgress } from '../../commandRunner';
+import Sinon = require('sinon');
+import { fail } from 'assert';
 
-  function createDate(dateString?: string): Date {
-    if (dateString) {
-      return new Date(dateString);
+describe('helpers', () => {
+  let sandbox: sinon.SinonSandbox;
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox();
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  describe('Invocation rate limiter', () => {
+    // 1 January 2020
+    let currentUnixTime = 1577836800;
+
+    function createDate(dateString?: string): Date {
+      if (dateString) {
+        return new Date(dateString);
+      }
+      const numMillisecondsPerSecond = 1000;
+      return new Date(currentUnixTime * numMillisecondsPerSecond);
     }
-    const numMillisecondsPerSecond = 1000;
-    return new Date(currentUnixTime * numMillisecondsPerSecond);
+
+    function createInvocationRateLimiter<T>(funcIdentifier: string, func: () => Promise<T>): InvocationRateLimiter<T> {
+      return new InvocationRateLimiter(new MockExtensionContext(), funcIdentifier, func, s => createDate(s));
+    }
+
+    it('initially invokes function', async () => {
+      let numTimesFuncCalled = 0;
+      const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncCalled++;
+      });
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+      expect(numTimesFuncCalled).to.equal(1);
+    });
+
+    it('doesn\'t invoke function again if no time has passed', async () => {
+      let numTimesFuncCalled = 0;
+      const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncCalled++;
+      });
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+      expect(numTimesFuncCalled).to.equal(1);
+    });
+
+    it('doesn\'t invoke function again if requested time since last invocation hasn\'t passed', async () => {
+      let numTimesFuncCalled = 0;
+      const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncCalled++;
+      });
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
+      currentUnixTime += 1;
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
+      expect(numTimesFuncCalled).to.equal(1);
+    });
+
+    it('invokes function again immediately if requested time since last invocation is 0 seconds', async () => {
+      let numTimesFuncCalled = 0;
+      const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncCalled++;
+      });
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
+      expect(numTimesFuncCalled).to.equal(2);
+    });
+
+    it('invokes function again after requested time since last invocation has elapsed', async () => {
+      let numTimesFuncCalled = 0;
+      const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncCalled++;
+      });
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
+      currentUnixTime += 1;
+      await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
+      expect(numTimesFuncCalled).to.equal(2);
+    });
+
+    it('invokes functions with different rate limiters', async () => {
+      let numTimesFuncACalled = 0;
+      const invocationRateLimiterA = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncACalled++;
+      });
+      let numTimesFuncBCalled = 0;
+      const invocationRateLimiterB = createInvocationRateLimiter('funcid', async () => {
+        numTimesFuncBCalled++;
+      });
+      await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
+      await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
+      expect(numTimesFuncACalled).to.equal(1);
+      expect(numTimesFuncBCalled).to.equal(1);
+    });
+  });
+
+  describe('codeql-database.yml tests', () => {
+    let dir: tmp.DirResult;
+    beforeEach(() => {
+      dir = tmp.dirSync();
+      const contents = yaml.safeDump({
+        primaryLanguage: 'cpp'
+      });
+      fs.writeFileSync(path.join(dir.name, 'codeql-database.yml'), contents, 'utf8');
+    });
+
+    afterEach(() => {
+      dir.removeCallback();
+    });
+
+    it('should get initial query contents when language is known', () => {
+      expect(getInitialQueryContents('cpp', 'hucairz')).to.eq('import cpp\n\nselect ""');
+    });
+
+    it('should get initial query contents when dbscheme is known', () => {
+      expect(getInitialQueryContents('', 'semmlecode.cpp.dbscheme')).to.eq('import cpp\n\nselect ""');
+    });
+
+    it('should get initial query contents when nothing is known', () => {
+      expect(getInitialQueryContents('', 'hucairz')).to.eq('select ""');
+    });
+  });
+
+  it('should find likely db language folders', () => {
+    expect(isLikelyDbLanguageFolder('db-javascript')).to.be.true;
+    expect(isLikelyDbLanguageFolder('dbnot-a-db')).to.be.false;
+  });
+
+  class MockExtensionContext implements ExtensionContext {
+    extensionMode: ExtensionMode = 3;
+    subscriptions: { dispose(): unknown }[] = [];
+    workspaceState: Memento = new MockMemento();
+    globalState = new MockGlobalStorage();
+    extensionPath = '';
+    asAbsolutePath(_relativePath: string): string {
+      throw new Error('Method not implemented.');
+    }
+    storagePath = '';
+    globalStoragePath = '';
+    logPath = '';
+    extensionUri = Uri.parse('');
+    environmentVariableCollection = new MockEnvironmentVariableCollection();
+    secrets = new MockSecretStorage();
+    storageUri = Uri.parse('');
+    globalStorageUri = Uri.parse('');
+    logUri = Uri.parse('');
+    extension: any;
   }
 
-  function createInvocationRateLimiter<T>(funcIdentifier: string, func: () => Promise<T>): InvocationRateLimiter<T> {
-    return new InvocationRateLimiter(new MockExtensionContext(), funcIdentifier, func, s => createDate(s));
+  class MockEnvironmentVariableCollection implements EnvironmentVariableCollection {
+    persistent = false;
+    replace(_variable: string, _value: string): void {
+      throw new Error('Method not implemented.');
+    }
+    append(_variable: string, _value: string): void {
+      throw new Error('Method not implemented.');
+    }
+    prepend(_variable: string, _value: string): void {
+      throw new Error('Method not implemented.');
+    }
+    get(_variable: string): EnvironmentVariableMutator | undefined {
+      throw new Error('Method not implemented.');
+    }
+    forEach(_callback: (variable: string, mutator: EnvironmentVariableMutator, collection: EnvironmentVariableCollection) => any, _thisArg?: any): void {
+      throw new Error('Method not implemented.');
+    }
+    delete(_variable: string): void {
+      throw new Error('Method not implemented.');
+    }
+    clear(): void {
+      throw new Error('Method not implemented.');
+    }
   }
 
-  it('initially invokes function', async () => {
-    let numTimesFuncCalled = 0;
-    const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncCalled++;
+  class MockMemento implements Memento {
+    map = new Map<any, any>();
+
+    /**
+     * Return a value.
+     *
+     * @param key A string.
+     * @param defaultValue A value that should be returned when there is no
+     * value (`undefined`) with the given key.
+     * @return The stored value or the defaultValue.
+     */
+    get<T>(key: string, defaultValue?: T): T {
+      return this.map.has(key) ? this.map.get(key) : defaultValue;
+    }
+
+    /**
+     * Store a value. The value must be JSON-stringifyable.
+     *
+     * @param key A string.
+     * @param value A value. MUST not contain cyclic references.
+     */
+    async update(key: string, value: any): Promise<void> {
+      this.map.set(key, value);
+    }
+  }
+
+  class MockGlobalStorage extends MockMemento {
+    public setKeysForSync(_keys: string[]): void {
+      return;
+    }
+  }
+
+  class MockSecretStorage implements SecretStorage {
+    get(_key: string): Thenable<string | undefined> {
+      throw new Error('Method not implemented.');
+    }
+    store(_key: string, _value: string): Thenable<void> {
+      throw new Error('Method not implemented.');
+    }
+    delete(_key: string): Thenable<void> {
+      throw new Error('Method not implemented.');
+    }
+    onDidChange!: Event<SecretStorageChangeEvent>;
+  }
+
+  it('should report stream progress', () => {
+    const spy = sandbox.spy();
+    const mockReadable = {
+      on: sandbox.spy()
+    };
+    const max = 1024 * 1024 * 4;
+    const firstStep = (1024 * 1024) + (1024 * 600);
+    const secondStep = 1024 * 1024 * 2;
+
+    (reportStreamProgress as any)(mockReadable, 'My prefix', max, spy);
+
+    // now pretend that we have received some messages
+    mockReadable.on.getCall(0).args[1]({ length: firstStep });
+    mockReadable.on.getCall(0).args[1]({ length: secondStep });
+
+    expect(spy).to.have.callCount(3);
+    expect(spy).to.have.been.calledWith({
+      step: 0,
+      maxStep: max,
+      message: 'My prefix [0.0 MB of 4.0 MB]',
+    });
+    expect(spy).to.have.been.calledWith({
+      step: firstStep,
+      maxStep: max,
+      message: 'My prefix [1.6 MB of 4.0 MB]',
+    });
+    expect(spy).to.have.been.calledWith({
+      step: firstStep + secondStep,
+      maxStep: max,
+      message: 'My prefix [3.6 MB of 4.0 MB]',
     });
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-    expect(numTimesFuncCalled).to.equal(1);
   });
 
-  it('doesn\'t invoke function again if no time has passed', async () => {
-    let numTimesFuncCalled = 0;
-    const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncCalled++;
+  it('should report stream progress when total bytes unknown', () => {
+    const spy = sandbox.spy();
+    const mockReadable = {
+      on: sandbox.spy()
+    };
+    (reportStreamProgress as any)(mockReadable, 'My prefix', undefined, spy);
+
+    // There are no listeners registered to this readable
+    expect(mockReadable.on).not.to.have.been.called;
+
+    expect(spy).to.have.callCount(1);
+    expect(spy).to.have.been.calledWith({
+      step: 1,
+      maxStep: 2,
+      message: 'My prefix (Size unknown)',
     });
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-    expect(numTimesFuncCalled).to.equal(1);
   });
 
-  it('doesn\'t invoke function again if requested time since last invocation hasn\'t passed', async () => {
-    let numTimesFuncCalled = 0;
-    const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncCalled++;
+  describe('open dialog', () => {
+    let showInformationMessageSpy: Sinon.SinonStub;
+    beforeEach(() => {
+      showInformationMessageSpy = sandbox.stub(window, 'showInformationMessage');
     });
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(100);
-    currentUnixTime += 1;
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(2);
-    expect(numTimesFuncCalled).to.equal(1);
-  });
 
-  it('invokes function again immediately if requested time since last invocation is 0 seconds', async () => {
-    let numTimesFuncCalled = 0;
-    const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncCalled++;
+    it('should show a binary choice dialog and return `yes`', (done) => {
+      // pretend user chooses 'yes'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      const res = showBinaryChoiceDialog('xxx');
+      res.then((val) => {
+        expect(val).to.eq(true);
+        done();
+      }).catch(e => fail(e));
     });
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(0);
-    expect(numTimesFuncCalled).to.equal(2);
-  });
 
-  it('invokes function again after requested time since last invocation has elapsed', async () => {
-    let numTimesFuncCalled = 0;
-    const invocationRateLimiter = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncCalled++;
+    it('should show a binary choice dialog and return `no`', (done) => {
+      // pretend user chooses 'no'
+      showInformationMessageSpy.onCall(0).resolvesArg(3);
+      const res = showBinaryChoiceDialog('xxx');
+      res.then((val) => {
+        expect(val).to.eq(false);
+        done();
+      }).catch(e => fail(e));
     });
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
-    currentUnixTime += 1;
-    await invocationRateLimiter.invokeFunctionIfIntervalElapsed(1);
-    expect(numTimesFuncCalled).to.equal(2);
-  });
 
-  it('invokes functions with different rate limiters', async () => {
-    let numTimesFuncACalled = 0;
-    const invocationRateLimiterA = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncACalled++;
+    it('should show an info dialog and confirm the action', (done) => {
+      // pretend user chooses to run action
+      showInformationMessageSpy.onCall(0).resolvesArg(1);
+      const res = showInformationMessageWithAction('xxx', 'yyy');
+      res.then((val) => {
+        expect(val).to.eq(true);
+        done();
+      }).catch(e => fail(e));
     });
-    let numTimesFuncBCalled = 0;
-    const invocationRateLimiterB = createInvocationRateLimiter('funcid', async () => {
-      numTimesFuncBCalled++;
+
+    it('should show an action dialog and avoid choosing the action', (done) => {
+      // pretend user does not choose to run action
+      showInformationMessageSpy.onCall(0).resolves(undefined);
+      const res = showInformationMessageWithAction('xxx', 'yyy');
+      res.then((val) => {
+        expect(val).to.eq(false);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog with a url and return `yes`', (done) => {
+      // pretend user clicks on the url twice and then clicks 'yes'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      showInformationMessageSpy.onCall(1).resolvesArg(2);
+      showInformationMessageSpy.onCall(2).resolvesArg(3);
+      const res = showBinaryChoiceWithUrlDialog('xxx', 'invalid:url');
+      res.then((val) => {
+        expect(val).to.eq(true);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog with a url and return `no`', (done) => {
+      // pretend user clicks on the url twice and then clicks 'no'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      showInformationMessageSpy.onCall(1).resolvesArg(2);
+      showInformationMessageSpy.onCall(2).resolvesArg(4);
+      const res = showBinaryChoiceWithUrlDialog('xxx', 'invalid:url');
+      res.then((val) => {
+        expect(val).to.eq(false);
+        done();
+      }).catch(e => fail(e));
+    });
+
+    it('should show a binary choice dialog and exit after clcking `more info` 5 times', (done) => {
+      // pretend user clicks on the url twice and then clicks 'no'
+      showInformationMessageSpy.onCall(0).resolvesArg(2);
+      showInformationMessageSpy.onCall(1).resolvesArg(2);
+      showInformationMessageSpy.onCall(2).resolvesArg(2);
+      showInformationMessageSpy.onCall(3).resolvesArg(2);
+      showInformationMessageSpy.onCall(4).resolvesArg(2);
+      const res = showBinaryChoiceWithUrlDialog('xxx', 'invalid:url');
+      res.then((val) => {
+        // No choie was made
+        expect(val).to.eq(undefined);
+        expect(showInformationMessageSpy.getCalls().length).to.eq(5);
+        done();
+      }).catch(e => fail(e));
     });
-    await invocationRateLimiterA.invokeFunctionIfIntervalElapsed(100);
-    await invocationRateLimiterB.invokeFunctionIfIntervalElapsed(100);
-    expect(numTimesFuncACalled).to.equal(1);
-    expect(numTimesFuncBCalled).to.equal(1);
   });
 });
-
-class MockExtensionContext implements ExtensionContext {
-  subscriptions: { dispose(): unknown }[] = [];
-  workspaceState: Memento = new MockMemento();
-  globalState: Memento = new MockMemento();
-  extensionPath = '';
-  asAbsolutePath(_relativePath: string): string {
-    throw new Error('Method not implemented.');
-  }
-  storagePath = '';
-  globalStoragePath = '';
-  logPath = '';
-}
-
-class MockMemento implements Memento {
-  map = new Map<any, any>();
-
-  /**
-   * Return a value.
-   *
-   * @param key A string.
-   * @param defaultValue A value that should be returned when there is no
-   * value (`undefined`) with the given key.
-   * @return The stored value or the defaultValue.
-   */
-  get<T>(key: string, defaultValue?: T): T {
-    return this.map.has(key) ? this.map.get(key) : defaultValue;
-  }
-
-  /**
-   * Store a value. The value must be JSON-stringifyable.
-   *
-   * @param key A string.
-   * @param value A value. MUST not contain cyclic references.
-   */
-  async update(key: string, value: any): Promise<void> {
-    this.map.set(key, value);
-  }
-}

extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts

@@ -5,7 +5,9 @@ import * as vscode from 'vscode';
 import * as sinon from 'sinon';
 import * as chaiAsPromised from 'chai-as-promised';
 import { logger } from '../../logging';
-import { QueryHistoryManager } from '../../query-history';
+import { QueryHistoryManager, HistoryTreeDataProvider } from '../../query-history';
+import { CompletedQuery } from '../../query-results';
+import { QueryInfo } from '../../run-queries';
 
 chai.use(chaiAsPromised);
 const expect = chai.expect;
@@ -19,32 +21,30 @@ describe('query-history', () => {
   let showQuickPickSpy: sinon.SinonStub;
 
   let tryOpenExternalFile: Function;
+  let sandbox: sinon.SinonSandbox;
 
   beforeEach(() => {
-    showTextDocumentSpy = sinon.stub(vscode.window, 'showTextDocument');
-    showInformationMessageSpy = sinon.stub(
+    sandbox = sinon.createSandbox();
+
+    showTextDocumentSpy = sandbox.stub(vscode.window, 'showTextDocument');
+    showInformationMessageSpy = sandbox.stub(
       vscode.window,
       'showInformationMessage'
     );
-    showQuickPickSpy = sinon.stub(
+    showQuickPickSpy = sandbox.stub(
       vscode.window,
       'showQuickPick'
     );
-    executeCommandSpy = sinon.stub(vscode.commands, 'executeCommand');
-    sinon.stub(logger, 'log');
+    executeCommandSpy = sandbox.stub(vscode.commands, 'executeCommand');
+    sandbox.stub(logger, 'log');
     tryOpenExternalFile = (QueryHistoryManager.prototype as any).tryOpenExternalFile;
   });
 
   afterEach(() => {
-    (vscode.window.showTextDocument as sinon.SinonStub).restore();
-    (vscode.commands.executeCommand as sinon.SinonStub).restore();
-    (logger.log as sinon.SinonStub).restore();
-    (vscode.window.showInformationMessage as sinon.SinonStub).restore();
-    (vscode.window.showQuickPick as sinon.SinonStub).restore();
+    sandbox.restore();
   });
 
   describe('tryOpenExternalFile', () => {
-
     it('should open an external file', async () => {
       await tryOpenExternalFile('xxx');
       expect(showTextDocumentSpy).to.have.been.calledOnceWith(
@@ -205,9 +205,70 @@ describe('query-history', () => {
       expect(queryHistory.compareWithItem).to.be.eq('a');
     });
   });
+
+  describe('HistoryTreeDataProvider', () => {
+    let historyTreeDataProvider: HistoryTreeDataProvider;
+    beforeEach(() => {
+      historyTreeDataProvider = new HistoryTreeDataProvider(vscode.Uri.file('/a/b/c').fsPath);
+    });
+
+    it('should get a tree item with raw results', async () => {
+      const mockQuery = {
+        query: {
+          hasInterpretedResults: () => Promise.resolve(false)
+        } as QueryInfo,
+        didRunSuccessfully: true,
+        toString: () => 'mock label'
+      } as CompletedQuery;
+      const treeItem = await historyTreeDataProvider.getTreeItem(mockQuery);
+      expect(treeItem.command).to.deep.eq({
+        title: 'Query History Item',
+        command: 'codeQLQueryHistory.itemClicked',
+        arguments: [mockQuery],
+      });
+      expect(treeItem.label).to.eq('mock label');
+      expect(treeItem.contextValue).to.eq('rawResultsItem');
+      expect(treeItem.iconPath).to.be.undefined;
+    });
+
+    it('should get a tree item with interpreted results', async () => {
+      const mockQuery = {
+        query: {
+          // as above, except for this line
+          hasInterpretedResults: () => Promise.resolve(true)
+        } as QueryInfo,
+        didRunSuccessfully: true,
+        toString: () => 'mock label'
+      } as CompletedQuery;
+      const treeItem = await historyTreeDataProvider.getTreeItem(mockQuery);
+      expect(treeItem.contextValue).to.eq('interpretedResultsItem');
+    });
+
+    it('should get a tree item that did not complete successfully', async () => {
+      const mockQuery = {
+        query: {
+          hasInterpretedResults: () => Promise.resolve(true)
+        } as QueryInfo,
+        // as above, except for this line
+        didRunSuccessfully: false,
+        toString: () => 'mock label'
+      } as CompletedQuery;
+      const treeItem = await historyTreeDataProvider.getTreeItem(mockQuery);
+      expect(treeItem.iconPath).to.eq(vscode.Uri.file('/a/b/c/media/red-x.svg').fsPath);
+    });
+
+    it('should get children', () => {
+      const mockQuery = {
+        databaseName: 'abc'
+      } as CompletedQuery;
+      historyTreeDataProvider.allHistory.push(mockQuery);
+      expect(historyTreeDataProvider.getChildren()).to.deep.eq([mockQuery]);
+      expect(historyTreeDataProvider.getChildren(mockQuery)).to.deep.eq([]);
+    });
+  });
 });
 
-function createMockQueryHistory(allHistory: {}[]) {
+function createMockQueryHistory(allHistory: Record<string, unknown>[]) {
   return {
     assertSingleQuery: (QueryHistoryManager.prototype as any).assertSingleQuery,
     findOtherQueryToCompare: (QueryHistoryManager.prototype as any).findOtherQueryToCompare,

extensions/ql-vscode/src/vscode-tests/no-workspace/query-results.test.ts

@@ -152,7 +152,8 @@ describe('CompletedQuery', () => {
     const sourceInfo = {};
     const metadata = {
       kind: 'my-kind',
-      id: 'my-id' as string | undefined
+      id: 'my-id' as string | undefined,
+      scored: undefined
     };
     const results1 = await interpretResults(
       mockServer,
@@ -183,7 +184,7 @@ describe('CompletedQuery', () => {
     );
     expect(results2).to.eq('1234');
     expect(spy).to.have.been.calledWith(
-      { kind: 'my-kind', id: 'dummy-id' },
+      { kind: 'my-kind', id: 'dummy-id', scored: undefined },
       resultsPath, interpretedResultsPath, sourceInfo
     );
 

extensions/ql-vscode/src/vscode-tests/no-workspace/run-queries.test.ts

@@ -24,6 +24,23 @@ describe('run-queries', () => {
     expect(info.dataset).to.eq('file:///abc');
   });
 
+  it('should check if interpreted results can be created', async () => {
+    const info = createMockQueryInfo();
+    (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(true);
+
+    expect(await info.canHaveInterpretedResults()).to.eq(true);
+
+    (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(false);
+    expect(await info.canHaveInterpretedResults()).to.eq(false);
+
+    (info.dbItem.hasMetadataFile as sinon.SinonStub).returns(true);
+    info.metadata!.kind = undefined;
+    expect(await info.canHaveInterpretedResults()).to.eq(false);
+
+    info.metadata!.kind = 'table';
+    expect(await info.canHaveInterpretedResults()).to.eq(false);
+  });
+
   describe('compile', () => {
     it('should compile', async () => {
       const info = createMockQueryInfo();
@@ -52,6 +69,7 @@ describe('run-queries', () => {
             localChecking: false,
             noComputeGetUrl: false,
             noComputeToString: false,
+            computeDefaultStrings: true
           },
           extraOptions: {
             timeoutSecs: 5
@@ -72,9 +90,14 @@ describe('run-queries', () => {
       {
         contents: {
           datasetUri: 'file:///abc'
-        }
+        },
+        hasMetadataFile: sinon.stub()
       } as unknown as DatabaseItem,
-      'my-scheme' // queryDbscheme
+      'my-scheme', // queryDbscheme,
+      undefined,
+      {
+        kind: 'problem'
+      }
     );
   }
 

extensions/ql-vscode/src/vscode-tests/no-workspace/telemetry.test.ts

@@ -0,0 +1,380 @@
+import * as chai from 'chai';
+import 'mocha';
+import 'sinon-chai';
+import * as sinon from 'sinon';
+import * as chaiAsPromised from 'chai-as-promised';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ExtensionContext, workspace, ConfigurationTarget, window } from 'vscode';
+import { TelemetryListener, telemetryListener as globalTelemetryListener } from '../../telemetry';
+import { UserCancellationException } from '../../commandRunner';
+import { fail } from 'assert';
+import { ENABLE_TELEMETRY } from '../../config';
+
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
+const sandbox = sinon.createSandbox();
+
+describe('telemetry reporting', function() {
+  // setting preferences can trigger lots of background activity
+  // so need to bump up the timeout of this test.
+  this.timeout(10000);
+
+  let originalTelemetryExtension: boolean | undefined;
+  let originalTelemetryGlobal: boolean | undefined;
+  let isCanary: string;
+  let ctx: ExtensionContext;
+  let telemetryListener: TelemetryListener;
+
+  beforeEach(async () => {
+    try {
+      // in case a previous test has accidentally activated this extension,
+      // need to disable it first.
+      // Accidentaly activation may happen asynchronously due to activationEvents
+      // specified in the package.json.
+      globalTelemetryListener?.dispose();
+
+      ctx = createMockExtensionContext();
+
+      sandbox.stub(TelemetryReporter.prototype, 'sendTelemetryEvent');
+      sandbox.stub(TelemetryReporter.prototype, 'sendTelemetryException');
+      sandbox.stub(TelemetryReporter.prototype, 'dispose');
+
+      originalTelemetryExtension = workspace.getConfiguration().get<boolean>('codeQL.telemetry.enableTelemetry');
+      originalTelemetryGlobal = workspace.getConfiguration().get<boolean>('telemetry.enableTelemetry');
+      isCanary = (!!workspace.getConfiguration().get<boolean>('codeQL.canary')).toString();
+
+      // each test will default to telemetry being enabled
+      await enableTelemetry('telemetry', true);
+      await enableTelemetry('codeQL.telemetry', true);
+
+      telemetryListener = new TelemetryListener('my-id', '1.2.3', 'fake-key', ctx);
+      await wait(100);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+
+  afterEach(async () => {
+    telemetryListener?.dispose();
+    // await wait(100);
+    try {
+      sandbox.restore();
+      await enableTelemetry('telemetry', originalTelemetryGlobal);
+      await enableTelemetry('codeQL.telemetry', originalTelemetryExtension);
+    } catch (e) {
+      console.error(e);
+    }
+  });
+
+  it('should initialize telemetry when both options are enabled', async () => {
+    await telemetryListener.initialize();
+
+    expect(telemetryListener._reporter).not.to.be.undefined;
+
+    const reporter: any = telemetryListener._reporter;
+    expect(reporter.extensionId).to.eq('my-id');
+    expect(reporter.extensionVersion).to.eq('1.2.3');
+    expect(reporter.userOptIn).to.eq(true); // enabled
+  });
+
+  it('should initialize telemetry when global option disabled', async () => {
+    try {
+      await enableTelemetry('telemetry', false);
+      await telemetryListener.initialize();
+      expect(telemetryListener._reporter).not.to.be.undefined;
+
+      const reporter: any = telemetryListener._reporter;
+      expect(reporter.userOptIn).to.eq(false); // disabled
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should not initialize telemetry when extension option disabled', async () => {
+    try {
+      await enableTelemetry('codeQL.telemetry', false);
+      await telemetryListener.initialize();
+
+      expect(telemetryListener._reporter).to.be.undefined;
+    } catch (e) {
+      fail(e);
+    }
+  });
+
+  it('should not initialize telemetry when both options disabled', async () => {
+    await enableTelemetry('codeQL.telemetry', false);
+    await enableTelemetry('telemetry', false);
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).to.be.undefined;
+  });
+
+  it('should dispose telemetry object when re-initializing and should not add multiple', async () => {
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    const firstReporter = telemetryListener._reporter;
+    await telemetryListener.initialize();
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    expect(telemetryListener._reporter).not.to.eq(firstReporter);
+
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+
+    // initializing a third time continues to dispose
+    await telemetryListener.initialize();
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledTwice;
+  });
+
+  it('should reinitialize reporter when extension setting changes', async () => {
+    await telemetryListener.initialize();
+
+    expect(TelemetryReporter.prototype.dispose).not.to.have.been.called;
+    expect(telemetryListener._reporter).not.to.be.undefined;
+
+    // this disables the reporter
+    await enableTelemetry('codeQL.telemetry', false);
+
+    expect(telemetryListener._reporter).to.be.undefined;
+
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+
+    // creates a new reporter, but does not dispose again
+    await enableTelemetry('codeQL.telemetry', true);
+
+    expect(telemetryListener._reporter).not.to.be.undefined;
+    expect(TelemetryReporter.prototype.dispose).to.have.been.calledOnce;
+  });
+
+  it('should set userOprIn to false when global setting changes', async () => {
+    await telemetryListener.initialize();
+
+    const reporter: any = telemetryListener._reporter;
+    expect(reporter.userOptIn).to.eq(true); // enabled
+
+    await enableTelemetry('telemetry', false);
+    expect(reporter.userOptIn).to.eq(false); // disabled
+  });
+
+  it('should send an event', async () => {
+    await telemetryListener.initialize();
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Success',
+        isCanary
+      },
+      { executionTime: 1234 });
+
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should send a command usage event with an error', async () => {
+    await telemetryListener.initialize();
+
+    telemetryListener.sendCommandUsage('command-id', 1234, new UserCancellationException());
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Cancelled',
+        isCanary
+      },
+      { executionTime: 1234 });
+
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should avoid sending an event when telemetry is disabled', async () => {
+    await telemetryListener.initialize();
+    await enableTelemetry('codeQL.telemetry', false);
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+    telemetryListener.sendCommandUsage('command-id', 1234, new Error());
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).not.to.have.been.called;
+    expect(TelemetryReporter.prototype.sendTelemetryException).not.to.have.been.called;
+  });
+
+  it('should send an event when telemetry is re-enabled', async () => {
+    await telemetryListener.initialize();
+    await enableTelemetry('codeQL.telemetry', false);
+    await enableTelemetry('codeQL.telemetry', true);
+
+    telemetryListener.sendCommandUsage('command-id', 1234, undefined);
+
+    expect(TelemetryReporter.prototype.sendTelemetryEvent).to.have.been.calledOnceWith('command-usage',
+      {
+        name: 'command-id',
+        status: 'Success',
+        isCanary
+      },
+      { executionTime: 1234 });
+  });
+
+  it('should filter undesired properties from telemetry payload', async () => {
+    await telemetryListener.initialize();
+    // Reach into the internal appInsights client to grab our telemetry processor.
+    const telemetryProcessor: Function =
+      ((telemetryListener._reporter as any).appInsightsClient._telemetryProcessors)[0];
+    const envelop = {
+      tags: {
+        'ai.cloud.roleInstance': true,
+        other: true
+      },
+      data: {
+        baseData: {
+          properties: {
+            'common.remotename': true,
+            other: true
+          }
+        }
+      }
+    };
+    const res = telemetryProcessor(envelop);
+    expect(res).to.eq(true);
+    expect(envelop).to.deep.eq({
+      tags: {
+        other: true
+      },
+      data: {
+        baseData: {
+          properties: {
+            other: true
+          }
+        }
+      }
+    });
+  });
+
+  it('should request permission if popup has never been seen before', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolvesArg(3 /* "yes" item */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    await enableTelemetry('codeQL.telemetry', false);
+
+    await telemetryListener.initialize();
+
+    // Dialog opened, user clicks "yes" and telemetry enabled
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(true);
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.true;
+  });
+
+  it('should prevent telemetry if permission is denied', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolvesArg(4 /* "no" item */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    await enableTelemetry('codeQL.telemetry', true);
+
+    await telemetryListener.initialize();
+
+    // Dialog opened, user clicks "no" and telemetry disabled
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(false);
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.true;
+  });
+
+  it('should unchange telemetry if permission dialog is dismissed', async () => {
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+
+    // this causes requestTelemetryPermission to be called
+    await enableTelemetry('codeQL.telemetry', false);
+
+    // Dialog opened, and user closes without interacting with it
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(false);
+    // dialog was canceled, so should not have marked as viewed
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+  });
+
+  it('should unchange telemetry if permission dialog is cancelled if starting as true', async () => {
+    await enableTelemetry('codeQL.telemetry', false);
+
+    // as before, except start with telemetry enabled. It should _stay_ enabled if the
+    // dialog is canceled.
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+
+    // this causes requestTelemetryPermission to be called
+    await enableTelemetry('codeQL.telemetry', true);
+
+    // Dialog opened, and user closes without interacting with it
+    // Telemetry state should not have changed
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+    expect(ENABLE_TELEMETRY.getValue()).to.eq(true);
+    // dialog was canceled, so should not have marked as viewed
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+  });
+
+  it('should avoid showing dialog if global telemetry is disabled', async () => {
+    // when telemetry is disabled globally, we never want to show the
+    // opt in/out dialog. We just assume that codeql telemetry should
+    // remain disabled as well.
+    // If the user ever turns global telemetry back on, then we can
+    // show the dialog.
+
+    await enableTelemetry('telemetry', false);
+    await ctx.globalState.update('telemetry-request-viewed', false);
+    sandbox.stub(window, 'showInformationMessage');
+
+    await telemetryListener.initialize();
+
+    // popup should not be shown even though we have initialized telemetry
+    expect(window.showInformationMessage).not.to.have.been.called;
+  });
+
+  // This test is failing because codeQL.canary is not a registered configuration.
+  // We do not want to have it registered because we don't want this item
+  // appearing in the settings page. It needs to olny be set by users we tell
+  // about it to.
+  // At this point, I see no other way of testing re-requesting permission.
+  xit('should request permission again when user changes canary setting', async () => {
+    // initially, both canary and telemetry are false
+    await workspace.getConfiguration().update('codeQL.canary', false);
+    await enableTelemetry('codeQL.telemetry', false);
+    await ctx.globalState.update('telemetry-request-viewed', true);
+    await telemetryListener.initialize();
+    sandbox.stub(window, 'showInformationMessage').resolves(undefined /* cancelled */);
+
+    // set canary to true
+    await workspace.getConfiguration().update('codeQL.canary', true);
+
+    // now, we should have to click through the telemetry requestor again
+    expect(ctx.globalState.get('telemetry-request-viewed')).to.be.false;
+    expect(window.showInformationMessage).to.have.been.calledOnce;
+  });
+
+  function createMockExtensionContext(): ExtensionContext {
+    return {
+      globalState: {
+        _state: {
+          'telemetry-request-viewed': true
+        } as Record<string, any>,
+        get(key: string) {
+          return this._state[key];
+        },
+        update(key: string, val: any) {
+          this._state[key] = val;
+        }
+      }
+    } as any;
+  }
+
+  async function enableTelemetry(section: string, value: boolean | undefined) {
+    await workspace.getConfiguration(section).update(
+      'enableTelemetry', value, ConfigurationTarget.Global
+    );
+
+    // Need to wait some time since the onDidChangeConfiguration listeners fire
+    // asynchronously and we sometimes need to wait for them to complete in
+    // order to have as successful test.
+    await wait(50);
+  }
+
+  async function wait(ms = 0) {
+    return new Promise(resolve =>
+      setTimeout(resolve, ms)
+    );
+  }
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/test-adapter.test.ts

@@ -1,24 +1,61 @@
 import 'vscode-test';
 import 'mocha';
 import * as sinon from 'sinon';
+import * as fs from 'fs-extra';
 import { Uri, WorkspaceFolder } from 'vscode';
 import { expect } from 'chai';
 
 import { QLTestAdapter } from '../../test-adapter';
 import { CodeQLCliServer } from '../../cli';
+import { DatabaseItem, DatabaseItemImpl, DatabaseManager, FullDatabaseOptions } from '../../databases';
 
 describe('test-adapter', () => {
   let adapter: QLTestAdapter;
+  let fakeDatabaseManager: DatabaseManager;
+  let currentDatabaseItem: DatabaseItem | undefined;
+  let databaseItems: DatabaseItem[] = [];
+  let openDatabaseSpy: sinon.SinonStub;
+  let removeDatabaseItemSpy: sinon.SinonStub;
+  let renameDatabaseItemSpy: sinon.SinonStub;
+  let setCurrentDatabaseItemSpy: sinon.SinonStub;
   let runTestsSpy: sinon.SinonStub;
   let resolveTestsSpy: sinon.SinonStub;
   let resolveQlpacksSpy: sinon.SinonStub;
   let sandox: sinon.SinonSandbox;
 
+  const preTestDatabaseItem = new DatabaseItemImpl(
+    Uri.file('/path/to/test/dir/dir.testproj'),
+    undefined,
+    { displayName: 'custom display name' } as unknown as FullDatabaseOptions,
+    (_) => { /* no change event listener */ }
+  );
+  const postTestDatabaseItem = new DatabaseItemImpl(
+    Uri.file('/path/to/test/dir/dir.testproj'),
+    undefined,
+    { displayName: 'default name' } as unknown as FullDatabaseOptions,
+    (_) => { /* no change event listener */ }
+  );
+
   beforeEach(() => {
     sandox = sinon.createSandbox();
     mockRunTests();
+    openDatabaseSpy = sandox.stub().resolves(postTestDatabaseItem);
+    removeDatabaseItemSpy = sandox.stub().resolves();
+    renameDatabaseItemSpy = sandox.stub().resolves();
+    setCurrentDatabaseItemSpy = sandox.stub().resolves();
     resolveQlpacksSpy = sandox.stub().resolves({});
     resolveTestsSpy = sandox.stub().resolves([]);
+    fakeDatabaseManager = {
+      currentDatabaseItem: undefined,
+      databaseItems: undefined,
+      openDatabase: openDatabaseSpy,
+      removeDatabaseItem: removeDatabaseItemSpy,
+      renameDatabaseItem: renameDatabaseItemSpy,
+      setCurrentDatabaseItem: setCurrentDatabaseItemSpy,
+    } as unknown as DatabaseManager;
+    sandox.stub(fakeDatabaseManager, 'currentDatabaseItem').get(() => currentDatabaseItem);
+    sandox.stub(fakeDatabaseManager, 'databaseItems').get(() => databaseItems);
+    sandox.stub(preTestDatabaseItem, 'isAffectedByTest').resolves(true);
     adapter = new QLTestAdapter({
       name: 'ABC',
       uri: Uri.parse('file:/ab/c')
@@ -26,7 +63,8 @@ describe('test-adapter', () => {
       runTests: runTestsSpy,
       resolveQlpacks: resolveQlpacksSpy,
       resolveTests: resolveTestsSpy
-    } as unknown as CodeQLCliServer);
+    } as unknown as CodeQLCliServer,
+      fakeDatabaseManager);
   });
 
   afterEach(() => {
@@ -74,12 +112,33 @@ describe('test-adapter', () => {
     expect(listenerSpy).to.have.callCount(5);
   });
 
+  it('should reregister testproj databases around test run', async () => {
+    sandox.stub(fs, 'access').resolves();
+    currentDatabaseItem = preTestDatabaseItem;
+    databaseItems = [preTestDatabaseItem];
+    await adapter.run(['/path/to/test/dir']);
+
+    removeDatabaseItemSpy.getCall(0).calledBefore(runTestsSpy.getCall(0));
+    openDatabaseSpy.getCall(0).calledAfter(runTestsSpy.getCall(0));
+    renameDatabaseItemSpy.getCall(0).calledAfter(openDatabaseSpy.getCall(0));
+    setCurrentDatabaseItemSpy.getCall(0).calledAfter(openDatabaseSpy.getCall(0));
+
+    sinon.assert.calledOnceWithExactly(
+      removeDatabaseItemSpy, sinon.match.any, sinon.match.any, preTestDatabaseItem);
+    sinon.assert.calledOnceWithExactly(
+      openDatabaseSpy, sinon.match.any, sinon.match.any, preTestDatabaseItem.databaseUri);
+    sinon.assert.calledOnceWithExactly(
+      renameDatabaseItemSpy, postTestDatabaseItem, preTestDatabaseItem.name);
+    sinon.assert.calledOnceWithExactly(
+      setCurrentDatabaseItemSpy, postTestDatabaseItem, true);
+  });
+
   function mockRunTests() {
     // runTests is an async generator function. This is not directly supported in sinon
     // However, we can pretend the same thing by just returning an async array.
     runTestsSpy = sandox.stub();
     runTestsSpy.returns(
-      (async function*() {
+      (async function* () {
         yield Promise.resolve({
           test: Uri.parse('file:/ab/c/d.ql').fsPath,
           pass: true,

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -1,16 +1,17 @@
 import * as path from 'path';
 import * as os from 'os';
-import { runTests } from 'vscode-test';
+import * as cp from 'child_process';
+import {
+  runTests,
+  downloadAndUnzipVSCode,
+  resolveCliPathFromVSCodeExecutablePath
+} from 'vscode-test';
+import { assertNever } from '../pure/helpers-pure';
+
+// For some reason, `TestOptions` is not exported directly from `vscode-test`,
+// but we can be tricky and import directly from the out file.
+import { TestOptions } from 'vscode-test/out/runTest';
 
-// A subset of the fields in TestOptions from vscode-test, which we
-// would simply use instead, but for the fact that it doesn't export
-// it.
-type Suite = {
-  extensionDevelopmentPath: string;
-  extensionTestsPath: string;
-  launchArgs: string[];
-  version?: string;
-};
 
 // Which version of vscode to test against. Can set to 'stable' or
 // 'insiders' or an explicit version number. See runTest.d.ts in
@@ -22,11 +23,21 @@ type Suite = {
 // testing against old versions if necessary.
 const VSCODE_VERSION = 'stable';
 
+// List if test dirs
+//   - no-workspace - Tests with no workspace selected upon launch.
+//   - minimal-workspace - Tests with a simple workspace selected upon launch.
+//   - cli-integration - Tests that require a cli to invoke actual commands
+enum TestDir {
+  NoWorksspace = 'no-workspace',
+  MinimalWorksspace = 'minimal-workspace',
+  CliIntegration = 'cli-integration'
+}
+
 /**
  * Run an integration test suite `suite`, retrying if it segfaults, at
  * most `tries` times.
  */
-async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise<void> {
+async function runTestsWithRetryOnSegfault(suite: TestOptions, tries: number): Promise<void> {
   for (let t = 0; t < tries; t++) {
     try {
       // Download and unzip VS Code if necessary, and run the integration test suite.
@@ -58,34 +69,35 @@ async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise
  */
 async function main() {
   try {
-    // The folder containing the Extension Manifest package.json
-    // Passed to `--extensionDevelopmentPath`.
     const extensionDevelopmentPath = path.resolve(__dirname, '../..');
+    const vscodeExecutablePath = await downloadAndUnzipVSCode(VSCODE_VERSION);
 
-    // List of integration test suites.
-    // The path to the extension test runner script is passed to --extensionTestsPath.
-    const integrationTestSuites: Suite[] = [
-      // Tests with no workspace selected upon launch.
-      {
-        version: VSCODE_VERSION,
-        extensionDevelopmentPath: extensionDevelopmentPath,
-        extensionTestsPath: path.resolve(__dirname, 'no-workspace', 'index'),
-        launchArgs: ['--disable-extensions'],
-      },
-      // Tests with a simple workspace selected upon launch.
-      {
-        version: VSCODE_VERSION,
-        extensionDevelopmentPath: extensionDevelopmentPath,
-        extensionTestsPath: path.resolve(__dirname, 'minimal-workspace', 'index'),
-        launchArgs: [
-          path.resolve(__dirname, '../../test/data'),
-          '--disable-extensions',
-        ]
-      }
-    ];
 
-    for (const integrationTestSuite of integrationTestSuites) {
-      await runTestsWithRetryOnSegfault(integrationTestSuite, 3);
+    // Which tests to run. Use a comma-separated list of directories.
+    const testDirsString = process.argv[2];
+    const dirs = testDirsString.split(',').map(dir => dir.trim().toLocaleLowerCase());
+
+    if (dirs.includes(TestDir.CliIntegration)) {
+      console.log('Installing required extensions');
+      const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath);
+      cp.spawnSync(cliPath, ['--install-extension', 'hbenl.vscode-test-explorer'], {
+        encoding: 'utf-8',
+        stdio: 'inherit'
+      });
+    }
+
+    console.log(`Running integration tests in these directories: ${dirs}`);
+    for (const dir of dirs) {
+      const launchArgs = getLaunchArgs(dir as TestDir);
+      console.log(`Next integration test dir: ${dir}`);
+      console.log(`Launch args: ${launchArgs}`);
+      await runTestsWithRetryOnSegfault({
+        version: VSCODE_VERSION,
+        vscodeExecutablePath,
+        extensionDevelopmentPath,
+        extensionTestsPath: path.resolve(__dirname, dir, 'index'),
+        launchArgs
+      }, 3);
     }
   } catch (err) {
     console.error(`Unexpected exception while running tests: ${err}`);
@@ -93,4 +105,42 @@ async function main() {
   }
 }
 
-main();
+void main();
+
+
+function getLaunchArgs(dir: TestDir) {
+  switch (dir) {
+    case TestDir.NoWorksspace:
+      return [
+        '--disable-extensions',
+        '--disable-gpu'
+      ];
+
+    case TestDir.MinimalWorksspace:
+      return [
+        '--disable-extensions',
+        '--disable-gpu',
+        path.resolve(__dirname, '../../test/data')
+      ];
+
+    case TestDir.CliIntegration:
+      // CLI integration tests requires a multi-root workspace so that the data and the QL sources are accessible.
+      return [
+        '--disable-gpu',
+        path.resolve(__dirname, '../../test/data'),
+
+        // explicitly disable extensions that are known to interfere with the CLI integration tests
+        '--disable-extension',
+        'eamodio.gitlens',
+        '--disable-extension',
+        'github.codespaces',
+        '--disable-extension',
+        'github.copilot',
+        process.env.TEST_CODEQL_PATH!
+      ];
+
+    default:
+      assertNever(dir);
+  }
+  return undefined;
+}

extensions/ql-vscode/src/vscode-tests/test-dispose-handler.ts

@@ -0,0 +1,14 @@
+import { Disposable } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+
+export function testDisposeHandler(disposable: any & Disposable) {
+  if (disposable.onDidExpandElement && disposable.onDidCollapseElement && disposable.reveal) {
+    // This looks like a treeViewer. Don't dispose
+    return;
+  }
+  if (disposable instanceof DisposableObject) {
+    disposable.dispose(testDisposeHandler);
+  } else {
+    disposable.dispose();
+  }
+}

extensions/ql-vscode/src/vscode-utils/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from './disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from 'vscode';
 
 /**
@@ -62,4 +62,3 @@ export class MultiFileSystemWatcher extends DisposableObject {
     this._onDidChange.fire(uri);
   }
 }
-