v1.3.8

Clarify a few entries.

.gitattributes

@@ -12,3 +12,10 @@ yarn.lock                    merge=binary
 # For more information, see this issue: https://github.com/Microsoft/web-build-tools/issues/1088
 #
 *.json                       linguist-language=JSON-with-Comments
+
+# Reduce incidence of needless merge conflicts on CHANGELOG.md
+# The man page at
+# https://mirrors.edge.kernel.org/pub/software/scm/git/docs/gitattributes.html
+# suggests that this might interleave lines arbitrarily, but empirically
+# it keeps added chunks contiguous
+CHANGELOG.md                merge=union

.github/ISSUE_TEMPLATE/new-extension-release.md

@@ -9,7 +9,7 @@ assignees: ''
 ---
 
 - [ ] Update this issue title to refer to the version of the release
-- [ ] Trigger a release build on Actions by adding a new tag on master of the format `vxx.xx.xx`
+- [ ] Trigger a release build on Actions by adding a new tag on branch `main` of the format `vxx.xx.xx`
 - [ ] Monitor the status of the release build in the `Release` workflow in the Actions tab.
 - [ ] Download the VSIX from the draft GitHub release that is created when the release build finishes.
 - [ ] Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).

.github/codeql/codeql-config.yml

@@ -0,0 +1,12 @@
+name: "CodeQL config"
+queries:
+  - name: Run standard queries
+    uses: security-and-quality
+  - name: Run custom javascript queries
+    uses: ./.github/codeql/queries
+paths:
+  - ./extensions/ql-vscode
+paths-ignore:
+  - '**/node_modules'
+  - '**/build'
+  - '**/out'

.github/codeql/queries/assert-pure.ql

@@ -0,0 +1,21 @@
+/**
+ * @name Unwanted dependency on vscode API
+ * @kind problem
+ * @problem.severity error
+ * @id vscode-codeql/assert-pure
+ * @description The modules stored under `pure` and tested in the `pure-tests`
+ * are intended to be "pure".
+ */
+import javascript
+
+class VSCodeImport extends ASTNode {
+  VSCodeImport() {
+    this.(Import).getImportedPath().getValue() = "vscode"
+  }
+}
+
+from Module m, VSCodeImport v
+where
+  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
+  m.getAnImportedModule*().getAnImport() = v
+select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/qlpack.yml

@@ -0,0 +1,3 @@
+name: vscode-codeql-custom-queries-javascript
+version: 0.0.0
+libraryPathDependencies: codeql-javascript

.github/pull_request_template.md

@@ -1,12 +1,12 @@
 <!-- Thank you for submitting a pull request. Please read our pull request guidelines before
   submitting your pull request:
-  https://github.com/github/vscode-codeql/blob/master/CONTRIBUTING.md#submitting-a-pull-request.
+  https://github.com/github/vscode-codeql/blob/main/CONTRIBUTING.md#submitting-a-pull-request.
 -->
 
 Replace this with a description of the changes your pull request makes.
 
 ## Checklist
 
-- [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
+- [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
 - [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] `@github/product-docs-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.
+- [ ] `@github/docs-content-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -2,6 +2,7 @@ name: "Code Scanning - CodeQL"
 
 on:
   push:
+  pull_request:
   schedule:
     - cron: '0 0 * * 0'
 
@@ -16,6 +17,9 @@ jobs:
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v1
+        with:
+            languages: javascript
+            config-file: ./.github/codeql/codeql-config.yml
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v1

.github/workflows/label-issue.yml

@@ -0,0 +1,15 @@
+name: Label issue
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  label:
+    name: Label issue
+    runs-on: ubuntu-latest
+    steps:
+      - name: Label issue
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo '{"labels": ["VSCode"]}' | gh api repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels --input -

.github/workflows/main.yml

@@ -1,5 +1,9 @@
 name: Build Extension
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   build:
@@ -16,14 +20,18 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
       - name: Install dependencies
-        run: node common/scripts/install-run-rush.js install
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
         shell: bash
 
       - name: Build
-        run: node common/scripts/install-run-rush.js build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
         shell: bash
 
       - name: Prepare artifacts
@@ -33,7 +41,7 @@ jobs:
           cp dist/*.vsix artifacts
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@master
+        uses: actions/upload-artifact@v2
         if: matrix.os == 'ubuntu-latest'
         with:
           name: vscode-codeql-extension
@@ -53,20 +61,23 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
-      # We have to build the dependencies in `lib` before running any tests.
       - name: Install dependencies
-        run: node common/scripts/install-run-rush.js install
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
         shell: bash
 
       - name: Build
-        run: node common/scripts/install-run-rush.js build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
         shell: bash
 
       - name: Lint
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run lint
 
       - name: Install CodeQL
@@ -79,27 +90,76 @@ jobs:
         shell: bash
 
       - name: Run unit tests (Linux)
+        working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
-          cd extensions/ql-vscode
           CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
 
       - name: Run unit tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
         if: matrix.os == 'ubuntu-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           sudo apt-get install xvfb
           /usr/bin/xvfb-run npm run integration
 
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run integration
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ['v2.2.6', 'v2.3.3', 'v2.4.0']
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.14.0'
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Checkout QL
+        uses: actions/checkout@v2
+        with:
+          repository: github/codeql
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          /usr/bin/xvfb-run npm run cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run cli-integration

.github/workflows/release.yml

@@ -6,20 +6,16 @@
 
 name: Release
 on:
-  push:
-    # Path filters are not evaluated for pushes to tags.
-    # (source: https://help.github.com/en/github/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#onpushpull_requestpaths)
-    # So this workflow is triggered in the following events:
-    # - Release event: a SemVer tag, e.g. v1.0.0 or v1.0.0-alpha, is pushed
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+*'
-    # OR
-    # - Test event: this file is modified on a branch in the main repo containing `/actions/` in the name.
-    branches:
-      - '**/actions/**'
+  pull_request:
     paths:
       - '**/workflows/release.yml'
 
+  workflow_dispatch:
+
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+
 jobs:
   build:
     name: Release
@@ -34,11 +30,15 @@ jobs:
           node-version: '10.18.1'
 
       - name: Install dependencies
-        run: node common/scripts/install-run-rush.js install
+        run: |
+          cd extensions/ql-vscode
+          npm install
         shell: bash
 
       - name: Build
-        run: node common/scripts/install-run-rush.js build --release
+        run: |
+          cd extensions/ql-vscode
+          npm run build -- --release
         shell: bash
 
       - name: Prepare artifacts
@@ -59,7 +59,7 @@ jobs:
       # This is just in case the release itself fails and we want to access the built artifacts from Actions.
       # TODO Remove if not useful.
       - name: Upload artifacts
-        uses: actions/upload-artifact@master
+        uses: actions/upload-artifact@v2
         with:
           name: vscode-codeql-extension
           path: artifacts
@@ -93,12 +93,12 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
-      # The checkout action does not fetch the master branch.
-      # Fetch the master branch so that we can base the version bump PR against master.
-      - name: Fetch master branch
+      # The checkout action does not fetch the main branch.
+      # Fetch the main branch so that we can base the version bump PR against main.
+      - name: Fetch main branch
         run: |
-          git fetch --depth=1 origin master:master
-          git checkout master
+          git fetch --depth=1 origin main:main
+          git checkout main
 
       - name: Bump patch version
         id: bump-patch-version
@@ -110,8 +110,14 @@ jobs:
           NEXT_VERSION="$(npm version patch)"
           echo "::set-output name=next_version::$NEXT_VERSION"
 
+      - name: Add changelog for next release
+        if: success()
+        run: |
+          cd extensions/ql-vscode
+          perl -i -pe 's/^/## \[UNRELEASED\]\n\n/ if($.==3)' CHANGELOG.md
+
       - name: Create version bump PR
-        uses: peter-evans/create-pull-request@c7b64af0a489eae91f7890f2c1b63d13cc2d8ab7 # v2.4.2
+        uses: peter-evans/create-pull-request@c7f493a8000b8aeb17a1332e326ba76b57cb83eb # v3.4.1
         if: success()
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -119,4 +125,4 @@ jobs:
           title: Bump version to ${{ steps.bump-patch-version.outputs.next_version }}
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
-          base: master
+          base: main

.gitignore

@@ -4,6 +4,7 @@
 # Generated files
 /dist/
 out/
+build/
 server/
 node_modules/
 gen/

.vscode/extensions.json

@@ -3,8 +3,10 @@
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
-    "eamodio.tsl-problem-matcher"
+    "eamodio.tsl-problem-matcher",
+    "dbaeumer.vscode-eslint",
+    "eternalphane.tsfmt-vscode"
   ],
   // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
   "unwantedRecommendations": []
-}
+}

.vscode/launch.json

@@ -8,18 +8,18 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-bqrs/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-io-node/out/**/*.js",
-        "${workspaceRoot}/dist/vscode-codeql/node_modules/semmle-vscode-utils/out/**/*.js"
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
-      "preLaunchTask": "Build"
+      "env": {
+        // change to 'true' debug the IDE or Query servers
+        "IDE_SERVER_JAVA_DEBUG": "false",
+        "QUERY_SERVER_JAVA_DEBUG": "false",
+      }
     },
     {
       "name": "Launch Unit Tests (vscode-codeql)",
@@ -44,7 +44,6 @@
       "port": 9229,
       "stopOnEntry": false,
       "sourceMaps": true,
-      "preLaunchTask": "Build",
       "console": "integratedTerminal",
       "internalConsoleOptions": "neverOpen"
     },
@@ -54,16 +53,14 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/**/*.js"
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
-      "preLaunchTask": "Build"
     },
     {
       "name": "Launch Integration Tests - Minimal Workspace (vscode-codeql)",
@@ -71,17 +68,31 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
         "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/dist/vscode-codeql/out/**/*.js",
-        "${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/**/*.js"
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+      ],
+    },
+    {
+      "name": "Launch Integration Tests - With CLI",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
+      ],
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "outFiles": [
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
-      "preLaunchTask": "Build"
     }
   ]
 }

.vscode/settings.json

@@ -33,6 +33,8 @@
     // This is necessary so that eslint can properly resolve its plugins
     "resolvePluginsRelativeTo": "./extensions/ql-vscode"
   },
-  // Force this to false since this will cause too many changes on each commit
-  "editor.formatOnSave": false
+  "editor.formatOnSave": false,
+  "typescript.preferences.quoteStyle": "single",
+  "javascript.preferences.quoteStyle": "single",
+  "editor.wordWrapColumn": 100
 }

.vscode/tasks.json

@@ -10,7 +10,10 @@
         "kind": "build",
         "isDefault": true
       },
-      "command": "node common/scripts/install-run-rush.js build --verbose",
+      "command": "npm run build",
+      "options": {
+        "cwd": "extensions/ql-vscode/"
+      },
       "presentation": {
         "echo": true,
         "reveal": "always",
@@ -33,64 +36,13 @@
         "$ts-webpack"
       ]
     },
-    {
-      "label": "Rebuild",
-      "type": "shell",
-      "group": "build",
-      "command": "node common/scripts/install-run-rush.js rebuild --verbose",
-      "presentation": {
-        "echo": true,
-        "reveal": "always",
-        "focus": false,
-        "panel": "shared",
-        "showReuseMessage": true,
-        "clear": true
-      },
-      "problemMatcher": [
-        {
-          "owner": "typescript",
-          "fileLocation": "absolute",
-          "pattern": {
-            "regexp": "^\\[gulp-typescript\\] ([^(]+)\\((\\d+|\\d+,\\d+|\\d+,\\d+,\\d+,\\d+)\\): error TS\\d+: (.*)$",
-            "file": 1,
-            "location": 2,
-            "message": 3
-          }
-        }
-      ]
-    },
-    {
-      "label": "Update",
-      "type": "shell",
-      "command": "node common/scripts/install-run-rush.js update",
-      "presentation": {
-        "echo": true,
-        "reveal": "always",
-        "focus": false,
-        "panel": "shared",
-        "showReuseMessage": true,
-        "clear": true
-      },
-      "problemMatcher": []
-    },
-    {
-      "label": "Update (full)",
-      "type": "shell",
-      "command": "node common/scripts/install-run-rush.js update --full",
-      "presentation": {
-        "echo": true,
-        "reveal": "always",
-        "focus": false,
-        "panel": "shared",
-        "showReuseMessage": true,
-        "clear": true
-      },
-      "problemMatcher": []
-    },
     {
       "label": "Format",
       "type": "shell",
-      "command": "node common/scripts/install-run-rush.js format",
+      "command": "npm run format",
+      "options": {
+        "cwd": "extensions/ql-vscode/"
+      },
       "presentation": {
         "echo": true,
         "reveal": "always",
@@ -111,4 +63,4 @@
       "group": "build"
     }
   ]
-}
+}

CONTRIBUTING.md

@@ -32,87 +32,34 @@ Here are a few things you can do that will increase the likelihood of your pull
 Make sure you have a fairly recent version of vscode (>1.32) and are using nodejs
 version >=v10.13.0. (Tested on v10.15.1 and v10.16.0).
 
-This repo uses [Rush](https://rushjs.io) to handle package management, building, and other
-operations across multiple projects. See the Rush "[Getting started as a developer](https://rushjs.io/pages/developer/new_developer/)" docs
-for more details.
+### Installing all packages
 
-If you plan on building from the command line, it's easiest if Rush is installed globally:
+From the command line, go to the directory `extensions/ql-vscode` and run
 
 ```shell
-npm install -g @microsoft/rush
+npm install
 ```
 
-To get started, run:
+### Building the extension
+
+From the command line, go to the directory `extensions/ql-vscode` and run
 
 ```shell
-rush update && rush build
+npm run build
+npm run watch
 ```
 
-Note that when you run the `rush` command from the globally installed version, it will examine the
-`rushVersion` property in the repo's `rush.json`, and if it differs from the globally installed
-version, it will download, cache, and run the version of Rush specified in the `rushVersion`
-property.
+Alternatively, you can build the extension within VS Code via `Terminal > Run Build Task...` (or `Ctrl+Shift+B` with the default key bindings). And you can run the watch command via `Terminal > Run Task` and then select `npm watch` from the menu.
 
-A few more things to know about using rush:
+Before running any of the launch commands, be sure to have run the `build` command to ensure that the JavaScript is compiled and the resources are copied to the proper location.
 
-* Avoid running `npm` for any commands that install/link dependencies
-* Instead use the *rush* equivalent: `rush add <package>`, `rush update`, etc.
-* If you plan on only building via VS Code tasks, you don't need Rush installed at all, since those
-tasks run `common/scripts/install-run-rush.js` to bootstrap a locally installed and cached copy of
-Rush.
+We recommend that you keep `npm run watch` running in the backgound and you only need to re-run `npm run build` in the following situations:
 
-### Building
+1. on first checkout
+2. whenever any of the non-TypeScript resources have changed
+3. on any change to files included in the webview
 
-#### Installing all packages (instead of `npm install`)
-
-After updating any `package.json` file, or after checking or pulling a new branch, you need to
-make sure all the right npm packages are installed, which you would normally do via `npm install` in
-a single-project repo. With Rush, you need to do an "update" instead:
-
-##### From VS Code
-
-`Terminal > Run Task... > Update`
-
-##### From the command line
-
-```shell
-rush update
-```
-
-#### Building all projects (instead of `gulp`)
-
-Rush builds all projects in the repo, in dependency order, building multiple projects in parallel
-where possible. By default, the build also packages the extension itself into a .vsix file in the
-`dist` directory. To build:
-
-##### From VS Code
-
-`Terminal > Run Build Task...` (or just `Ctrl+Shift+B` with the default key bindings)
-
-##### From the command line
-
-```shell
-rush build --verbose
-```
-
-#### Forcing a clean build
-
-Rush does a reasonable job of detecting on its own which projects need to be rebuilt, but if you need to
-force a full rebuild of all projects:
-
-##### From VS Code
-
-`Terminal > Run Task... > Rebuild`
-
-##### From the command line
-
-```shell
-rush rebuild --verbose
-```
-
-Note that `rush rebuild` performs a complete rebuild, whereas `rush build` performs an incremental build and in many cases will not need to do anything at all.
-
-### Installing
+### Installing the extension
 
 You can install the `.vsix` file from within VS Code itself, from the Extensions container in the sidebar:
 
@@ -144,15 +91,28 @@ Alternatively, you can run the tests inside of vscode. There are several vscode
 
 ## Releasing (write access required)
 
-1. Double-check the `CHANGELOG.md` contains all desired change comments
-   and has the version to be released with date at the top.
-1. Double-check that the extension `package.json` has the version you intend to release.
-   If you are doing a patch release (as opposed to minor or major version) this should already
-   be correct.
-1. Trigger a release build on Actions by adding a new tag on master of the format `vxx.xx.xx`
+1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
+    * Go through all recent PRs and make sure they are properly accounted for.
+    * Make sure all changelog entries have links back to their PR(s) if appropriate.
+1. Double-check that the extension `package.json` has the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
+1. Create a PR for this release:
+    * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
+    * Create a new branch for the release named after the new version. For example: `v1.3.6`
+    * Create a new commit with a message the same as the branch name.
+    * Create a PR for this branch.
+    * Wait for the PR to be merged into `main`
+1. Trigger a release build on Actions by adding a new tag on branch `main` named after the release, as above. Note that when you push to upstream, you will need to fully qualify the ref. A command like this will work:
+
+    ```bash
+    git push upstream refs/tags/v1.3.6
+    ```
+
+    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
+    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
+
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
-1. Optionally unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
+1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
 1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
 1. Click the `...` menu in the CodeQL row and click **Update**.

README.md

@@ -4,7 +4,7 @@ This project is an extension for Visual Studio Code that adds rich language supp
 
 The extension is released. You can download it from the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql).
 
-To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
+To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
 [![CI status badge](https://github.com/github/vscode-codeql/workflows/Build%20Extension/badge.svg)](https://github.com/github/vscode-codeql/actions?query=workflow%3A%22Build+Extension%22+branch%3Amaster)
 [![VS Marketplace badge](https://vsmarketplacebadge.apphb.com/version/github.vscode-codeql.svg)](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql)
@@ -13,7 +13,7 @@ To see what has changed in the last few versions of the extension, see the [Chan
 
 * Enables you to use CodeQL to query databases and discover problems in codebases.
 * Shows the flow of data through the results of path queries, which is essential for triaging security results.
-* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/Semmle/ql).
+* Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
 

common/config/LICENSE

@@ -1,28 +0,0 @@
-This directory contains content from https://github.com/microsoft/rushstack,
-used under the MIT license as follows.
-See https://github.com/microsoft/rushstack/blob/master/stack/rush-stack/LICENSE.
-
-@microsoft/rush-stack
-
-Copyright (c) Microsoft Corporation. All rights reserved.
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

common/config/rush/.npmrc

@@ -1,12 +0,0 @@
-# Rush uses this file to configure the package registry, regardless of whether the
-# package manager is PNPM, NPM, or Yarn.  Prior to invoking the package manager,
-# Rush will always copy this file to the folder where installation is performed.
-# When NPM is the package manager, Rush works around NPM's processing of
-# undefined environment variables by deleting any lines that reference undefined
-# environment variables.
-#
-# DO NOT SPECIFY AUTHENTICATION CREDENTIALS IN THIS FILE.  It should only be used
-# to configure registry sources.
-
-registry=https://registry.npmjs.org/
-always-auth=false

common/config/rush/command-line.json

@@ -1,32 +0,0 @@
-/**
- * This configuration file defines custom commands for the "rush" command-line.
- * For full documentation, please see https://rushjs.io/pages/configs/command_line_json/
- */
-{
-  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",
-  "commands": [
-    {
-      "commandKind": "bulk",
-      "name": "format",
-      "summary": "Reformat source code in all projects",
-      "description": "Runs the `format` npm task in each project, if present.",
-      "safeForSimultaneousRushProcesses": false,
-      "enableParallelism": true,
-      "ignoreDependencyOrder": true,
-      "ignoreMissingScript": true,
-      "allowWarningsInSuccessfulBuild": false
-    }
-  ],
-  "parameters": [
-    {
-      "parameterKind": "flag",
-      "longName": "--release",
-      "shortName": "-r",
-      "description": "Perform a release build",
-      "associatedCommands": [
-        "build",
-        "rebuild"
-      ],
-    }
-  ]
-}

common/config/rush/common-versions.json

@@ -1,43 +0,0 @@
-/**
- * This configuration file specifies NPM dependency version selections that affect all projects
- * in a Rush repo.  For full documentation, please see https://rushjs.io
- */
-{
-  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/common-versions.schema.json",
-
-  /**
-   * A table that specifies a "preferred version" for a dependency package. The "preferred version"
-   * is typically used to hold an indirect dependency back to a specific version, however generally
-   * it can be any SemVer range specifier (e.g. "~1.2.3"), and it will narrow any (compatible)
-   * SemVer range specifier.  See the Rush documentation for details about this feature.
-   */
-  "preferredVersions": {
-
-    /**
-     * When someone asks for "^1.0.0" make sure they get "1.2.3" when working in this repo,
-     * instead of the latest version.
-     */
-    // "some-library": "1.2.3"
-  },
-
-  /**
-   * The "rush check" command can be used to enforce that every project in the repo must specify
-   * the same SemVer range for a given dependency.  However, sometimes exceptions are needed.
-   * The allowedAlternativeVersions table allows you to list other SemVer ranges that will be
-   * accepted by "rush check" for a given dependency.
-   *
-   * IMPORTANT: THIS TABLE IS FOR *ADDITIONAL* VERSION RANGES THAT ARE ALTERNATIVES TO THE
-   * USUAL VERSION (WHICH IS INFERRED BY LOOKING AT ALL PROJECTS IN THE REPO).
-   * This design avoids unnecessary churn in this file.
-   */
-   "allowedAlternativeVersions": {
-
-    /**
-     * For example, allow some projects to use an older TypeScript compiler
-     * (in addition to whatever "usual" version is being used by other projects in the repo):
-     */
-    // "typescript": [
-    //   "~2.4.0"
-    // ]
-  }
-}

common/config/rush/pnpmfile.js

@@ -1,32 +0,0 @@
-"use strict";
-
-/**
- * When using the PNPM package manager, you can use pnpmfile.js to workaround
- * dependencies that have mistakes in their package.json file.  (This feature is
- * functionally similar to Yarn's "resolutions".)
- *
- * For details, see the PNPM documentation:
- * https://pnpm.js.org/docs/en/hooks.html
- *
- * IMPORTANT: SINCE THIS FILE CONTAINS EXECUTABLE CODE, MODIFYING IT IS LIKELY
- * TO INVALIDATE ANY CACHED DEPENDENCY ANALYSIS.  We recommend to run "rush update --full"
- * after any modification to pnpmfile.js.
- *
- */
-module.exports = {
-  hooks: {
-    readPackage
-  }
-};
-
-/**
- * This hook is invoked during installation before a package's dependencies
- * are selected.
- * The `packageJson` parameter is the deserialized package.json
- * contents for the package that is about to be installed.
- * The `context` parameter provides a log() function.
- * The return value is the updated object.
- */
-function readPackage(packageJson, context) {
-  return packageJson;
-}

common/config/rush/version-policies.json

@@ -1,6 +0,0 @@
-/**
- * This is configuration file is used for advanced publishing configurations with Rush.
- * For full documentation, please see https://rushjs.io/pages/configs/version_policies_json/
- */
-
-[]

common/scripts/install-run-rush.js

@@ -1,67 +0,0 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where the Rush command may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
-// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it.
-// An example usage would be:
-//
-//    node common/scripts/install-run-rush.js install
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const path = require("path");
-const fs = require("fs");
-const install_run_1 = require("./install-run");
-const PACKAGE_NAME = '@microsoft/rush';
-const RUSH_PREVIEW_VERSION = 'RUSH_PREVIEW_VERSION';
-function _getRushVersion() {
-    const rushPreviewVersion = process.env[RUSH_PREVIEW_VERSION];
-    if (rushPreviewVersion !== undefined) {
-        console.log(`Using Rush version from environment variable ${RUSH_PREVIEW_VERSION}=${rushPreviewVersion}`);
-        return rushPreviewVersion;
-    }
-    const rushJsonFolder = install_run_1.findRushJsonFolder();
-    const rushJsonPath = path.join(rushJsonFolder, install_run_1.RUSH_JSON_FILENAME);
-    try {
-        const rushJsonContents = fs.readFileSync(rushJsonPath, 'utf-8');
-        // Use a regular expression to parse out the rushVersion value because rush.json supports comments,
-        // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script.
-        const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/);
-        return rushJsonMatches[1];
-    }
-    catch (e) {
-        throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` +
-            'The \'rushVersion\' field is either not assigned in rush.json or was specified ' +
-            'using an unexpected syntax.');
-    }
-}
-function _run() {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ ...packageBinArgs /* [build, --to, myproject] */] = process.argv;
-    // Detect if this script was directly invoked, or if the install-run-rushx script was invokved to select the
-    // appropriate binary inside the rush package to run
-    const scriptName = path.basename(scriptPath);
-    const bin = scriptName.toLowerCase() === 'install-run-rushx.js' ? 'rushx' : 'rush';
-    if (!nodePath || !scriptPath) {
-        throw new Error('Unexpected exception: could not detect node path or script path');
-    }
-    if (process.argv.length < 3) {
-        console.log(`Usage: ${scriptName} <command> [args...]`);
-        if (scriptName === 'install-run-rush.js') {
-            console.log(`Example: ${scriptName} build --to myproject`);
-        }
-        else {
-            console.log(`Example: ${scriptName} custom-command`);
-        }
-        process.exit(1);
-    }
-    install_run_1.runWithErrorAndStatusCode(() => {
-        const version = _getRushVersion();
-        console.log(`The rush.json configuration requests Rush version ${version}`);
-        return install_run_1.installAndRun(PACKAGE_NAME, version, bin, packageBinArgs);
-    });
-}
-_run();
-//# sourceMappingURL=install-run-rush.js.map

common/scripts/install-run-rushx.js

@@ -1,18 +0,0 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where the Rush command may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the version of Rush
-// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the
-// rushx command.
-//
-// An example usage would be:
-//
-//    node common/scripts/install-run-rushx.js custom-command
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-require("./install-run-rush");
-//# sourceMappingURL=install-run-rushx.js.map

common/scripts/install-run.js

@@ -1,433 +0,0 @@
-"use strict";
-// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
-// See the @microsoft/rush package's LICENSE file for license information.
-Object.defineProperty(exports, "__esModule", { value: true });
-// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED.
-//
-// This script is intended for usage in an automated build environment where a Node tool may not have
-// been preinstalled, or may have an unpredictable version.  This script will automatically install the specified
-// version of the specified tool (if not already installed), and then pass a command-line to it.
-// An example usage would be:
-//
-//    node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io
-//
-// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/
-const childProcess = require("child_process");
-const fs = require("fs");
-const os = require("os");
-const path = require("path");
-exports.RUSH_JSON_FILENAME = 'rush.json';
-const RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME = 'RUSH_TEMP_FOLDER';
-const INSTALLED_FLAG_FILENAME = 'installed.flag';
-const NODE_MODULES_FOLDER_NAME = 'node_modules';
-const PACKAGE_JSON_FILENAME = 'package.json';
-/**
- * Parse a package specifier (in the form of name\@version) into name and version parts.
- */
-function _parsePackageSpecifier(rawPackageSpecifier) {
-    rawPackageSpecifier = (rawPackageSpecifier || '').trim();
-    const separatorIndex = rawPackageSpecifier.lastIndexOf('@');
-    let name;
-    let version = undefined;
-    if (separatorIndex === 0) {
-        // The specifier starts with a scope and doesn't have a version specified
-        name = rawPackageSpecifier;
-    }
-    else if (separatorIndex === -1) {
-        // The specifier doesn't have a version
-        name = rawPackageSpecifier;
-    }
-    else {
-        name = rawPackageSpecifier.substring(0, separatorIndex);
-        version = rawPackageSpecifier.substring(separatorIndex + 1);
-    }
-    if (!name) {
-        throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`);
-    }
-    return { name, version };
-}
-/**
- * As a workaround, copyAndTrimNpmrcFile() copies the .npmrc file to the target folder, and also trims
- * unusable lines from the .npmrc file.
- *
- * Why are we trimming the .npmrc lines?  NPM allows environment variables to be specified in
- * the .npmrc file to provide different authentication tokens for different registry.
- * However, if the environment variable is undefined, it expands to an empty string, which
- * produces a valid-looking mapping with an invalid URL that causes an error.  Instead,
- * we'd prefer to skip that line and continue looking in other places such as the user's
- * home directory.
- *
- * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._copyNpmrcFile()
- */
-function _copyAndTrimNpmrcFile(sourceNpmrcPath, targetNpmrcPath) {
-    console.log(`Copying ${sourceNpmrcPath} --> ${targetNpmrcPath}`); // Verbose
-    let npmrcFileLines = fs.readFileSync(sourceNpmrcPath).toString().split('\n');
-    npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim());
-    const resultLines = [];
-    // Trim out lines that reference environment variables that aren't defined
-    for (const line of npmrcFileLines) {
-        // This finds environment variable tokens that look like "${VAR_NAME}"
-        const regex = /\$\{([^\}]+)\}/g;
-        const environmentVariables = line.match(regex);
-        let lineShouldBeTrimmed = false;
-        if (environmentVariables) {
-            for (const token of environmentVariables) {
-                // Remove the leading "${" and the trailing "}" from the token
-                const environmentVariableName = token.substring(2, token.length - 1);
-                if (!process.env[environmentVariableName]) {
-                    lineShouldBeTrimmed = true;
-                    break;
-                }
-            }
-        }
-        if (lineShouldBeTrimmed) {
-            // Example output:
-            // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}"
-            resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line);
-        }
-        else {
-            resultLines.push(line);
-        }
-    }
-    fs.writeFileSync(targetNpmrcPath, resultLines.join(os.EOL));
-}
-/**
- * syncNpmrc() copies the .npmrc file to the target folder, and also trims unusable lines from the .npmrc file.
- * If the source .npmrc file not exist, then syncNpmrc() will delete an .npmrc that is found in the target folder.
- *
- * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc()
- */
-function _syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder, useNpmrcPublish) {
-    const sourceNpmrcPath = path.join(sourceNpmrcFolder, !useNpmrcPublish ? '.npmrc' : '.npmrc-publish');
-    const targetNpmrcPath = path.join(targetNpmrcFolder, '.npmrc');
-    try {
-        if (fs.existsSync(sourceNpmrcPath)) {
-            _copyAndTrimNpmrcFile(sourceNpmrcPath, targetNpmrcPath);
-        }
-        else if (fs.existsSync(targetNpmrcPath)) {
-            // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target
-            console.log(`Deleting ${targetNpmrcPath}`); // Verbose
-            fs.unlinkSync(targetNpmrcPath);
-        }
-    }
-    catch (e) {
-        throw new Error(`Error syncing .npmrc file: ${e}`);
-    }
-}
-let _npmPath = undefined;
-/**
- * Get the absolute path to the npm executable
- */
-function getNpmPath() {
-    if (!_npmPath) {
-        try {
-            if (os.platform() === 'win32') {
-                // We're on Windows
-                const whereOutput = childProcess.execSync('where npm', { stdio: [] }).toString();
-                const lines = whereOutput.split(os.EOL).filter((line) => !!line);
-                // take the last result, we are looking for a .cmd command
-                // see https://github.com/microsoft/rushstack/issues/759
-                _npmPath = lines[lines.length - 1];
-            }
-            else {
-                // We aren't on Windows - assume we're on *NIX or Darwin
-                _npmPath = childProcess.execSync('which npm', { stdio: [] }).toString();
-            }
-        }
-        catch (e) {
-            throw new Error(`Unable to determine the path to the NPM tool: ${e}`);
-        }
-        _npmPath = _npmPath.trim();
-        if (!fs.existsSync(_npmPath)) {
-            throw new Error('The NPM executable does not exist');
-        }
-    }
-    return _npmPath;
-}
-exports.getNpmPath = getNpmPath;
-function _ensureFolder(folderPath) {
-    if (!fs.existsSync(folderPath)) {
-        const parentDir = path.dirname(folderPath);
-        _ensureFolder(parentDir);
-        fs.mkdirSync(folderPath);
-    }
-}
-/**
- * Create missing directories under the specified base directory, and return the resolved directory.
- *
- * Does not support "." or ".." path segments.
- * Assumes the baseFolder exists.
- */
-function _ensureAndJoinPath(baseFolder, ...pathSegments) {
-    let joinedPath = baseFolder;
-    try {
-        for (let pathSegment of pathSegments) {
-            pathSegment = pathSegment.replace(/[\\\/]/g, '+');
-            joinedPath = path.join(joinedPath, pathSegment);
-            if (!fs.existsSync(joinedPath)) {
-                fs.mkdirSync(joinedPath);
-            }
-        }
-    }
-    catch (e) {
-        throw new Error(`Error building local installation folder (${path.join(baseFolder, ...pathSegments)}): ${e}`);
-    }
-    return joinedPath;
-}
-function _getRushTempFolder(rushCommonFolder) {
-    const rushTempFolder = process.env[RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME];
-    if (rushTempFolder !== undefined) {
-        _ensureFolder(rushTempFolder);
-        return rushTempFolder;
-    }
-    else {
-        return _ensureAndJoinPath(rushCommonFolder, 'temp');
-    }
-}
-/**
- * Resolve a package specifier to a static version
- */
-function _resolvePackageVersion(rushCommonFolder, { name, version }) {
-    if (!version) {
-        version = '*'; // If no version is specified, use the latest version
-    }
-    if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) {
-        // If the version contains only characters that we recognize to be used in static version specifiers,
-        // pass the version through
-        return version;
-    }
-    else {
-        // version resolves to
-        try {
-            const rushTempFolder = _getRushTempFolder(rushCommonFolder);
-            const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-            _syncNpmrc(sourceNpmrcFolder, rushTempFolder);
-            const npmPath = getNpmPath();
-            // This returns something that looks like:
-            //  @microsoft/rush@3.0.0 '3.0.0'
-            //  @microsoft/rush@3.0.1 '3.0.1'
-            //  ...
-            //  @microsoft/rush@3.0.20 '3.0.20'
-            //  <blank line>
-            const npmVersionSpawnResult = childProcess.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], {
-                cwd: rushTempFolder,
-                stdio: []
-            });
-            if (npmVersionSpawnResult.status !== 0) {
-                throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`);
-            }
-            const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString();
-            const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line);
-            const latestVersion = versionLines[versionLines.length - 1];
-            if (!latestVersion) {
-                throw new Error('No versions found for the specified version range.');
-            }
-            const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/);
-            if (!versionMatches) {
-                throw new Error(`Invalid npm output ${latestVersion}`);
-            }
-            return versionMatches[1];
-        }
-        catch (e) {
-            throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`);
-        }
-    }
-}
-let _rushJsonFolder;
-/**
- * Find the absolute path to the folder containing rush.json
- */
-function findRushJsonFolder() {
-    if (!_rushJsonFolder) {
-        let basePath = __dirname;
-        let tempPath = __dirname;
-        do {
-            const testRushJsonPath = path.join(basePath, exports.RUSH_JSON_FILENAME);
-            if (fs.existsSync(testRushJsonPath)) {
-                _rushJsonFolder = basePath;
-                break;
-            }
-            else {
-                basePath = tempPath;
-            }
-        } while (basePath !== (tempPath = path.dirname(basePath))); // Exit the loop when we hit the disk root
-        if (!_rushJsonFolder) {
-            throw new Error('Unable to find rush.json.');
-        }
-    }
-    return _rushJsonFolder;
-}
-exports.findRushJsonFolder = findRushJsonFolder;
-/**
- * Detects if the package in the specified directory is installed
- */
-function _isPackageAlreadyInstalled(packageInstallFolder) {
-    try {
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        if (!fs.existsSync(flagFilePath)) {
-            return false;
-        }
-        const fileContents = fs.readFileSync(flagFilePath).toString();
-        return fileContents.trim() === process.version;
-    }
-    catch (e) {
-        return false;
-    }
-}
-/**
- * Removes the following files and directories under the specified folder path:
- *  - installed.flag
- *  -
- *  - node_modules
- */
-function _cleanInstallFolder(rushTempFolder, packageInstallFolder) {
-    try {
-        const flagFile = path.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        if (fs.existsSync(flagFile)) {
-            fs.unlinkSync(flagFile);
-        }
-        const packageLockFile = path.resolve(packageInstallFolder, 'package-lock.json');
-        if (fs.existsSync(packageLockFile)) {
-            fs.unlinkSync(packageLockFile);
-        }
-        const nodeModulesFolder = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME);
-        if (fs.existsSync(nodeModulesFolder)) {
-            const rushRecyclerFolder = _ensureAndJoinPath(rushTempFolder, 'rush-recycler', `install-run-${Date.now().toString()}`);
-            fs.renameSync(nodeModulesFolder, rushRecyclerFolder);
-        }
-    }
-    catch (e) {
-        throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`);
-    }
-}
-function _createPackageJson(packageInstallFolder, name, version) {
-    try {
-        const packageJsonContents = {
-            'name': 'ci-rush',
-            'version': '0.0.0',
-            'dependencies': {
-                [name]: version
-            },
-            'description': 'DON\'T WARN',
-            'repository': 'DON\'T WARN',
-            'license': 'MIT'
-        };
-        const packageJsonPath = path.join(packageInstallFolder, PACKAGE_JSON_FILENAME);
-        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2));
-    }
-    catch (e) {
-        throw new Error(`Unable to create package.json: ${e}`);
-    }
-}
-/**
- * Run "npm install" in the package install folder.
- */
-function _installPackage(packageInstallFolder, name, version) {
-    try {
-        console.log(`Installing ${name}...`);
-        const npmPath = getNpmPath();
-        const result = childProcess.spawnSync(npmPath, ['install'], {
-            stdio: 'inherit',
-            cwd: packageInstallFolder,
-            env: process.env
-        });
-        if (result.status !== 0) {
-            throw new Error('"npm install" encountered an error');
-        }
-        console.log(`Successfully installed ${name}@${version}`);
-    }
-    catch (e) {
-        throw new Error(`Unable to install package: ${e}`);
-    }
-}
-/**
- * Get the ".bin" path for the package.
- */
-function _getBinPath(packageInstallFolder, binName) {
-    const binFolderPath = path.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin');
-    const resolvedBinName = (os.platform() === 'win32') ? `${binName}.cmd` : binName;
-    return path.resolve(binFolderPath, resolvedBinName);
-}
-/**
- * Write a flag file to the package's install directory, signifying that the install was successful.
- */
-function _writeFlagFile(packageInstallFolder) {
-    try {
-        const flagFilePath = path.join(packageInstallFolder, INSTALLED_FLAG_FILENAME);
-        fs.writeFileSync(flagFilePath, process.version);
-    }
-    catch (e) {
-        throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`);
-    }
-}
-function installAndRun(packageName, packageVersion, packageBinName, packageBinArgs) {
-    const rushJsonFolder = findRushJsonFolder();
-    const rushCommonFolder = path.join(rushJsonFolder, 'common');
-    const rushTempFolder = _getRushTempFolder(rushCommonFolder);
-    const packageInstallFolder = _ensureAndJoinPath(rushTempFolder, 'install-run', `${packageName}@${packageVersion}`);
-    if (!_isPackageAlreadyInstalled(packageInstallFolder)) {
-        // The package isn't already installed
-        _cleanInstallFolder(rushTempFolder, packageInstallFolder);
-        const sourceNpmrcFolder = path.join(rushCommonFolder, 'config', 'rush');
-        _syncNpmrc(sourceNpmrcFolder, packageInstallFolder);
-        _createPackageJson(packageInstallFolder, packageName, packageVersion);
-        _installPackage(packageInstallFolder, packageName, packageVersion);
-        _writeFlagFile(packageInstallFolder);
-    }
-    const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`;
-    const statusMessageLine = new Array(statusMessage.length + 1).join('-');
-    console.log(os.EOL + statusMessage + os.EOL + statusMessageLine + os.EOL);
-    const binPath = _getBinPath(packageInstallFolder, packageBinName);
-    const result = childProcess.spawnSync(binPath, packageBinArgs, {
-        stdio: 'inherit',
-        cwd: process.cwd(),
-        env: process.env
-    });
-    if (result.status !== null) {
-        return result.status;
-    }
-    else {
-        throw result.error || new Error('An unknown error occurred.');
-    }
-}
-exports.installAndRun = installAndRun;
-function runWithErrorAndStatusCode(fn) {
-    process.exitCode = 1;
-    try {
-        const exitCode = fn();
-        process.exitCode = exitCode;
-    }
-    catch (e) {
-        console.error(os.EOL + os.EOL + e.toString() + os.EOL + os.EOL);
-    }
-}
-exports.runWithErrorAndStatusCode = runWithErrorAndStatusCode;
-function _run() {
-    const [nodePath, /* Ex: /bin/node */ scriptPath, /* /repo/common/scripts/install-run-rush.js */ rawPackageSpecifier, /* qrcode@^1.2.0 */ packageBinName, /* qrcode */ ...packageBinArgs /* [-f, myproject/lib] */] = process.argv;
-    if (!nodePath) {
-        throw new Error('Unexpected exception: could not detect node path');
-    }
-    if (path.basename(scriptPath).toLowerCase() !== 'install-run.js') {
-        // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control
-        // to the script that (presumably) imported this file
-        return;
-    }
-    if (process.argv.length < 4) {
-        console.log('Usage: install-run.js <package>@<version> <command> [args...]');
-        console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io');
-        process.exit(1);
-    }
-    runWithErrorAndStatusCode(() => {
-        const rushJsonFolder = findRushJsonFolder();
-        const rushCommonFolder = _ensureAndJoinPath(rushJsonFolder, 'common');
-        const packageSpecifier = _parsePackageSpecifier(rawPackageSpecifier);
-        const name = packageSpecifier.name;
-        const version = _resolvePackageVersion(rushCommonFolder, packageSpecifier);
-        if (packageSpecifier.version !== version) {
-            console.log(`Resolved to ${name}@${version}`);
-        }
-        return installAndRun(name, version, packageBinName, packageBinArgs);
-    });
-}
-_run();
-//# sourceMappingURL=install-run.js.map

configs/typescript-config/extension.tsconfig.json

@@ -1,8 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/tsconfig",
-  "extends": "./common.tsconfig.json",
-  "compilerOptions": {
-    "declaration": false,
-    "strict": true
-  }
-}

configs/typescript-config/lib.tsconfig.json

@@ -1,4 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/tsconfig",
-  "extends": "./common.tsconfig.json"
-}

configs/typescript-config/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "typescript-config",
-  "description": "TypeScript configurations",
-  "author": "GitHub",
-  "private": true,
-  "version": "0.0.1",
-  "publisher": "GitHub",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/github/vscode-codeql"
-  },
-  "scripts": {
-    "build": "",
-    "format": ""
-  },
-  "devDependencies": {},
-  "dependencies": {}
-}

extensions/ql-vscode/.eslintrc.js

@@ -1,34 +1,35 @@
 module.exports = {
-  parser: '@typescript-eslint/parser',
+  parser: "@typescript-eslint/parser",
   parserOptions: {
     ecmaVersion: 2018,
-    sourceType: 'module',
-    project: ['tsconfig.json', './src/**/tsconfig.json'],
+    sourceType: "module",
+    project: ["tsconfig.json", "./src/**/tsconfig.json", "./gulpfile.ts/tsconfig.json"],
   },
-  plugins: ['@typescript-eslint'],
+  plugins: ["@typescript-eslint"],
   env: {
     node: true,
-    es6: true
+    es6: true,
   },
-  extends: [
-    'eslint:recommended',
-    'plugin:@typescript-eslint/recommended',
-  ],
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
   rules: {
-    '@typescript-eslint/no-use-before-define': 0,
-    '@typescript-eslint/no-unused-vars': ["warn", {
-      "vars": "all",
-      "args": "none",
-      "ignoreRestSiblings": false
-    }],
+    "@typescript-eslint/no-use-before-define": 0,
+    "@typescript-eslint/no-unused-vars": [
+      "warn",
+      {
+        vars: "all",
+        args: "none",
+        ignoreRestSiblings: false,
+      },
+    ],
     "@typescript-eslint/explicit-function-return-type": "off",
     "@typescript-eslint/no-non-null-assertion": "off",
     "@typescript-eslint/no-explicit-any": "off",
-    "prefer-const": ["warn", {"destructuring": "all"}],
-    "indent": "off",
+    "prefer-const": ["warn", { destructuring: "all" }],
+    indent: "off",
     "@typescript-eslint/indent": "off",
     "@typescript-eslint/no-throw-literal": "error",
     "no-useless-escape": 0,
-    "semi": 2
+    semi: 2,
+    quotes: ["warn", "single"]
   },
 };

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,105 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.3.8 - 17 December 2020
+
+- Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting the IDE. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)
+- Fix bug when removing databases where sometimes the source folder would not also be removed from the workspace or the database files would not be deleted from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692)
+- Query results with no string representation will now be displayed with placeholder text in query results. Previously, they were omitted. [#694](https://github.com/github/vscode-codeql/pull/694)
+- Add a label for the language of a database in the databases view. This will only take effect for new databases created with the CodeQL CLI v2.4.1 or later. [#697](https://github.com/github/vscode-codeql/pull/697)
+- Add clearer error message when running a query using a missing or invalid qlpack. [#702](https://github.com/github/vscode-codeql/pull/702)
+- Add clearer error message when trying to run a command from the query history view if no item in the history is selected. [#702](https://github.com/github/vscode-codeql/pull/702)
+- Fix a bug where it is not possible to download some database archives. This fix specifically addresses large archives and archives whose central directories do not align with file headers. [#700](https://github.com/github/vscode-codeql/pull/700)
+- Avoid error dialogs when QL test discovery or database cleanup encounters a missing directory. [#706](https://github.com/github/vscode-codeql/pull/706)
+- Add descriptive text and a link in the results view. [#711](https://github.com/github/vscode-codeql/pull/711)
+- Fix the _Set Label_ command in the query history view. [#710](https://github.com/github/vscode-codeql/pull/710)
+- Add the _CodeQL: View AST_ command to the right-click context menu when a source file in a database source archive is open in the editor. [#712](https://github.com/github/vscode-codeql/pull/712)
+
+## 1.3.7 - 24 November 2020
+
+- Editors opened by navigating from the results view are no longer opened in _preview mode_. Now they are opened as a persistent editor. [#630](https://github.com/github/vscode-codeql/pull/630)
+- When comparing the results of a failed QL test run and the `.expected` file does not exist, an empty `.expected` file is created and compared against the `.actual` file. [#669](https://github.com/github/vscode-codeql/pull/669)
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
+- Add more structured output for tests. [#626](https://github.com/github/vscode-codeql/pull/626)
+- Whenever the extension restarts, orphaned databases will be cleaned up. These are databases whose files are located inside of the extension's storage area, but are not imported into the workspace.
+- After renaming a database, the database list is re-sorted. [#685](https://github.com/github/vscode-codeql/pull/685)
+- Add a `codeQl.resultsDisplay.pageSize` setting to configure the number of results displayed in a single results view page. Increase the default page size from 100 to 200. [#686](https://github.com/github/vscode-codeql/pull/686)
+- Update the AST Viewer to include edge labels (if available) in addition to the target node labels. So far, only C/C++ databases take advantage of this change. [#688](https://github.com/github/vscode-codeql/pull/688)
+
+## 1.3.6 - 4 November 2020
+
+- Fix URI encoding for databases that were created with special characters in their paths. [#648](https://github.com/github/vscode-codeql/pull/648)
+- Disable CodeQL Test commands from the command palette [#667](https://github.com/github/vscode-codeql/pull/667)
+- Fix display of booleans in results view. [#657](https://github.com/github/vscode-codeql/pull/657)
+- Avoid recursive selection changes in AST Viewer. [#668](https://github.com/github/vscode-codeql/pull/668)
+
+## 1.3.5 - 27 October 2020
+
+- Fix a bug where archived source folders for databases were not showing any contents.
+- Fix URI encoding for databases that were created with special characters in their paths.
+
+## 1.3.4 - 22 October 2020
+
+- Add friendly welcome message when the databases view is empty.
+- Add open query, open results, and remove query commands in the query history view title bar.
+- The maximum number of simultaneous queries launchable by the `CodeQL: Run Queries in Selected Files` command is now configurable by changing the `codeQL.runningQueries.maxQueries` setting.
+- Allow simultaneously run queries to be canceled in a single-click.
+- Prevent multiple upgrade dialogs from appearing when running simultaneous queries on upgradeable databases.
+- Fix sorting of results. Some pages of results would have the wrong sort order and columns.
+- Remember previous sort order when reloading query results.
+- Fix proper escaping of backslashes in SARIF message strings.
+- Allow setting `codeQL.runningQueries.numberOfThreads` and `codeQL.runningTests.numberOfThreads` to 0, (which is interpreted as 'use one thread per core on the machine').
+- Clear the problems view of all CodeQL query results when a database is removed.
+- Add a `View DIL` command on query history items. This opens a text editor containing the Datalog Intermediary Language representation of the compiled query.
+- Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/exploring-the-structure-of-your-source-code.html).
+- The `codeQL.runningTests.numberOfThreads` setting is now used correctly when running tests.
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the qlpacks.
+- Ensure output of CodeQL test runs includes compilation error messages and test failure messages.
+
+## 1.3.3 - 16 September 2020
+
+- Fix display of raw results entities with label but no url.
+- Fix bug where sort order is forgotten when changing raw results page.
+- Avoid showing a location link in results view when a result item has an empty location.
+
+## 1.3.2 - 12 August 2020
+
+- Fix error with choosing qlpack search path.
+- Fix pagination when there are no results.
+- Suppress database downloaded from URL message when action canceled.
+- Fix QL test discovery to avoid showing duplicate tests in the test explorer.
+- Enable pagination of query results
+- Add experimental AST Viewer for Go and C++. To enable, add `"codeQL.experimentalAstViewer": true` to the user settings file.
+
+## 1.3.1 - 7 July 2020
+
+- Fix unzipping of large files.
+- Ensure compare order is consistent when selecting two queries to compare. The first query selected is always the _from_ query and the query selected later is always the _to_ query.
+- Ensure added databases have zipped source locations for databases added as archives or downloaded from the internet.
+- Fix bug where it is not possible to add databases starting with `db-*`.
+- Change styling of pagination section of the results page.
+- Fix display of query text for stored quick queries.
+
+## 1.3.0 - 22 June 2020
+
+- Report error when selecting invalid database.
+- Add descriptive message for database archive import failure.
+- Respect VS Code's i18n locale setting when formatting dates and sorting strings.
+- Allow the opening of large SARIF files externally from VS Code.
+- Add new 'CodeQL: Compare Query' command that shows the differences between two queries.
+- Allow multiple items in the query history view to be removed in one operation.
+- Allow multiple items in the databases view to be removed in one operation.
+- Allow multiple items in the databases view to be upgraded in one operation.
+- Allow multiple items in the databases view to have their external folders opened.
+- Allow all selected queries to be run in one command from the file explorer.
+
+## 1.2.2 - 8 June 2020
+
+- Fix auto-indentation rules.
+- Add ability to download platform-specific releases of the CodeQL CLI if they are available.
+- Fix handling of downloading prerelease versions of the CodeQL CLI.
+- Add pagination for displaying non-interpreted results.
+
 ## 1.2.1 - 29 May 2020
 
 - Better formatting and autoindentation when adding QLDoc comments to `.ql` and `.qll` files.

extensions/ql-vscode/README.md

@@ -7,7 +7,7 @@ This project is an extension for Visual Studio Code that adds rich language supp
 - Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 - Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
-To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/master/extensions/ql-vscode/CHANGELOG.md).
+To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
 ## Quick start overview
 

extensions/ql-vscode/gulpfile.js/index.js

@@ -1,19 +0,0 @@
-'use strict';
-
-require('ts-node').register({});
-const gulp = require('gulp');
-const {
-  compileTypeScript,
-  watchTypeScript,
-  packageExtension,
-  compileTextMateGrammar,
-  copyTestData,
-  copyViewCss
-} = require('build-tasks');
-const { compileView } = require('./webpack');
-
-exports.buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
-exports.compileTextMateGrammar = compileTextMateGrammar;
-exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);
-exports.watchTypeScript = watchTypeScript;
-exports.compileTypeScript = compileTypeScript;

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -0,0 +1,72 @@
+import * as fs from 'fs-extra';
+import * as jsonc from 'jsonc-parser';
+import * as path from 'path';
+
+export interface DeployedPackage {
+  distPath: string;
+  name: string;
+  version: string;
+}
+
+const packageFiles = [
+  '.vscodeignore',
+  'CHANGELOG.md',
+  'README.md',
+  'language-configuration.json',
+  'media',
+  'node_modules',
+  'out'
+];
+
+async function copyPackage(sourcePath: string, destPath: string): Promise<void> {
+  for (const file of packageFiles) {
+    console.log(`copying ${path.resolve(sourcePath, file)} to ${path.resolve(destPath, file)}`);
+    await fs.copy(path.resolve(sourcePath, file), path.resolve(destPath, file));
+  }
+}
+
+export async function deployPackage(packageJsonPath: string): Promise<DeployedPackage> {
+  try {
+    const packageJson: any = jsonc.parse(await fs.readFile(packageJsonPath, 'utf8'));
+
+    // Default to development build; use flag --release to indicate release build.
+    const isDevBuild = !process.argv.includes('--release');
+    const distDir = path.join(__dirname, '../../../dist');
+    await fs.mkdirs(distDir);
+
+    if (isDevBuild) {
+      // NOTE: rootPackage.name had better not have any regex metacharacters
+      const oldDevBuildPattern = new RegExp('^' + packageJson.name + '[^/]+-dev[0-9.]+\\.vsix$');
+      // Dev package filenames are of the form
+      //    vscode-codeql-0.0.1-dev.2019.9.27.19.55.20.vsix
+      (await fs.readdir(distDir)).filter(name => name.match(oldDevBuildPattern)).map(build => {
+        console.log(`Deleting old dev build ${build}...`);
+        fs.unlinkSync(path.join(distDir, build));
+      });
+      const now = new Date();
+      packageJson.version = packageJson.version +
+        `-dev.${now.getUTCFullYear()}.${now.getUTCMonth() + 1}.${now.getUTCDate()}` +
+        `.${now.getUTCHours()}.${now.getUTCMinutes()}.${now.getUTCSeconds()}`;
+    }
+
+    const distPath = path.join(distDir, packageJson.name);
+    await fs.remove(distPath);
+    await fs.mkdirs(distPath);
+
+    await fs.writeFile(path.join(distPath, 'package.json'), JSON.stringify(packageJson, null, 2));
+
+    const sourcePath = path.join(__dirname, '..');
+    console.log(`Copying package '${packageJson.name}' and its dependencies to '${distPath}'...`);
+    await copyPackage(sourcePath, distPath);
+
+    return {
+      distPath: distPath,
+      name: packageJson.name,
+      version: packageJson.version
+    };
+  }
+  catch (e) {
+    console.error(e);
+    throw e;
+  }
+}

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -0,0 +1,10 @@
+import * as gulp from 'gulp';
+import { compileTypeScript, watchTypeScript, copyViewCss } from './typescript';
+import { compileTextMateGrammar } from './textmate';
+import { copyTestData } from './tests';
+import { compileView } from './webpack';
+import { packageExtension } from './package';
+
+export const buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
+export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData };
+exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);

extensions/ql-vscode/gulpfile.ts/package.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import { deployPackage } from './deploy';
-import * as child_process from 'child-process-promise';
+import * as childProcess from 'child-process-promise';
 
 export async function packageExtension(): Promise<void> {
   const deployedPackage = await deployPackage(path.resolve('package.json'));
@@ -9,7 +9,7 @@ export async function packageExtension(): Promise<void> {
     'package',
     '--out', path.resolve(deployedPackage.distPath, '..', `${deployedPackage.name}-${deployedPackage.version}.vsix`)
   ];
-  const proc = child_process.spawn('vsce', args, {
+  const proc = childProcess.spawn('./node_modules/.bin/vsce', args, {
     cwd: deployedPackage.distPath
   });
   proc.childProcess.stdout!.on('data', (data) => {

extensions/ql-vscode/gulpfile.ts/tests.ts

@@ -0,0 +1,17 @@
+import * as gulp from 'gulp';
+
+export function copyTestData() {
+  copyNoWorkspaceData();
+  copyCliIntegrationData();
+  return Promise.resolve();
+}
+
+function copyNoWorkspaceData() {
+  return gulp.src('src/vscode-tests/no-workspace/data/**/*')
+    .pipe(gulp.dest('out/vscode-tests/no-workspace/data'));
+}
+
+function copyCliIntegrationData() {
+  return gulp.src('src/vscode-tests/cli-integration/data/**/*')
+    .pipe(gulp.dest('out/vscode-tests/cli-integration/data'));
+}

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -1,5 +1,5 @@
 import * as gulp from 'gulp';
-import * as js_yaml from 'js-yaml';
+import * as jsYaml from 'js-yaml';
 import * as through from 'through2';
 import * as PluginError from 'plugin-error';
 import * as Vinyl from 'vinyl';
@@ -13,9 +13,10 @@ import * as Vinyl from 'vinyl';
  */
 function replaceReferencesWithStrings(value: string, replacements: Map<string, string>): string {
   let result = value;
+  // eslint-disable-next-line no-constant-condition
   while (true) {
     const original = result;
-    for (const key of replacements.keys()) {
+    for (const key of Array.from(replacements.keys())) {
       result = result.replace(`(?#${key})`, `(?:${replacements.get(key)})`);
     }
     if (result === original) {
@@ -32,7 +33,7 @@ function replaceReferencesWithStrings(value: string, replacements: Map<string, s
  */
 function gatherMacros(yaml: any): Map<string, string> {
   const macros = new Map<string, string>();
-  for (var key in yaml.macros) {
+  for (const key in yaml.macros) {
     macros.set(key, yaml.macros[key]);
   }
 
@@ -55,7 +56,7 @@ function getNodeMatchText(rule: any): string {
   else if (rule.patterns !== undefined) {
     const patterns: string[] = [];
     // For a list of patterns, use the disjunction of those patterns.
-    for (var patternIndex in rule.patterns) {
+    for (const patternIndex in rule.patterns) {
       const pattern = rule.patterns[patternIndex];
       if (pattern.include !== null) {
         patterns.push('(?' + pattern.include + ')');
@@ -65,7 +66,7 @@ function getNodeMatchText(rule: any): string {
     return '(?:' + patterns.join('|') + ')';
   }
   else {
-    return ''
+    return '';
   }
 }
 
@@ -78,7 +79,7 @@ function getNodeMatchText(rule: any): string {
  */
 function gatherMatchTextForRules(yaml: any): Map<string, string> {
   const replacements = new Map<string, string>();
-  for (var key in yaml.repository) {
+  for (const key in yaml.repository) {
     const node = yaml.repository[key];
     replacements.set(key, getNodeMatchText(node));
   }
@@ -106,7 +107,7 @@ function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
  * @param action Callback to invoke on each rule.
  */
 function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
-  for (var key in ruleMap) {
+  for (const key in ruleMap) {
     const rule = ruleMap[key];
     if ((typeof rule) === 'object') {
       action(rule);
@@ -124,7 +125,7 @@ function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
  * @param action The transformation to make on each match pattern.
  */
 function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
-  for (var key in rule) {
+  for (const key in rule) {
     switch (key) {
       case 'begin':
       case 'end':
@@ -184,10 +185,10 @@ function transformFile(yaml: any) {
   visitAllRulesInFile(yaml, (rule) => {
     visitAllMatchesInRule(rule, (match) => {
       if ((typeof match) === 'object') {
-        for (var key in match) {
+        for (const key in match) {
           return macros.get(key)!.replace('(?#)', `(?:${match[key]})`);
         }
-        throw new Error("No key in macro map.")
+        throw new Error('No key in macro map.');
       }
       else {
         return match;
@@ -225,7 +226,7 @@ export function transpileTextMateGrammar() {
     else if (file.isBuffer()) {
       const buf: Buffer = file.contents;
       const yamlText: string = buf.toString('utf8');
-      const jsonData: any = js_yaml.safeLoad(yamlText);
+      const jsonData: any = jsYaml.safeLoad(yamlText);
       transformFile(jsonData);
 
       file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), 'utf8');

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -1,15 +1,14 @@
 {
+  "$schema": "http://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
+    "strict": true,
     "module": "commonjs",
     "target": "es2017",
-    "outDir": "out",
-    "lib": [
-      "es6"
-    ],
+    "lib": ["es6"],
     "moduleResolution": "node",
     "sourceMap": true,
-    "rootDir": "../../src",
+    "rootDir": ".",
     "strictNullChecks": true,
     "noFallthroughCasesInSwitch": true,
     "preserveWatchOutput": true,
@@ -19,12 +18,5 @@
     "noUnusedLocals": true,
     "noUnusedParameters": true
   },
-  "include": [
-    "../../src/**/*.ts"
-  ],
-  "exclude": [
-    "../../node_modules",
-    "../../test",
-    "../../**/view"
-  ]
+  "include": ["*.ts"]
 }

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -0,0 +1,42 @@
+import * as colors from 'ansi-colors';
+import * as gulp from 'gulp';
+import * as sourcemaps from 'gulp-sourcemaps';
+import * as ts from 'gulp-typescript';
+
+function goodReporter(): ts.reporter.Reporter {
+  return {
+    error: (error, typescript) => {
+      if (error.tsFile) {
+        console.log('[' + colors.gray('gulp-typescript') + '] ' + colors.red(error.fullFilename
+          + '(' + (error.startPosition!.line + 1) + ',' + error.startPosition!.character + '): ')
+          + 'error TS' + error.diagnostic.code + ': ' + typescript.flattenDiagnosticMessageText(error.diagnostic.messageText, '\n'));
+      }
+      else {
+        console.log(error.message);
+      }
+    },
+  };
+}
+
+const tsProject = ts.createProject('tsconfig.json');
+
+export function compileTypeScript() {
+  return tsProject.src()
+    .pipe(sourcemaps.init())
+    .pipe(tsProject(goodReporter()))
+    .pipe(sourcemaps.write('.', {
+      includeContent: false,
+      sourceRoot: '.',
+    }))
+    .pipe(gulp.dest('out'));
+}
+
+export function watchTypeScript() {
+  gulp.watch('src/**/*.ts', compileTypeScript);
+}
+
+/** Copy CSS files for the results view into the output directory. */
+export function copyViewCss() {
+  return gulp.src('src/view/*.css')
+    .pipe(gulp.dest('out'));
+}

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -4,13 +4,14 @@ import * as webpack from 'webpack';
 export const config: webpack.Configuration = {
   mode: 'development',
   entry: {
-    resultsView: './src/view/results.tsx'
+    resultsView: './src/view/results.tsx',
+    compareView: './src/compare/view/Compare.tsx',
   },
   output: {
     path: path.resolve(__dirname, '..', 'out'),
-    filename: "[name].js"
+    filename: '[name].js'
   },
-  devtool: "inline-source-map",
+  devtool: 'inline-source-map',
   resolve: {
     extensions: ['.js', '.ts', '.tsx', '.json']
   },
@@ -19,6 +20,9 @@ export const config: webpack.Configuration = {
       {
         test: /\.(ts|tsx)$/,
         loader: 'ts-loader',
+        options: {
+          configFile: 'src/view/tsconfig.json',
+        }
       },
       {
         test: /\.less$/,

extensions/ql-vscode/language-configuration.json

@@ -30,9 +30,5 @@
       "end": "^\\s*//\\s*#?endregion\\b"
     }
   },
-  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)",
-  "indentationRules": {
-    "increaseIndentPattern": "^((?!.*?\\/\\*).*\\*/)?\\s*[\\}\\]].*$",
-    "decreaseIndentPattern": "^((?!\\/\\/).)*(\\{[^}\"']*|\\([^)\"']*|\\[[^\\]\"']*)$"
-  }
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
 }

extensions/ql-vscode/media/dark/clear-all.svg

@@ -0,0 +1,7 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10 12.6L10.7 13.3L12.3 11.7L13.9 13.3L14.7 12.6L13 11L14.7 9.40005L13.9 8.60005L12.3 10.3L10.7 8.60005L10 9.40005L11.6 11L10 12.6Z" fill="#C5C5C5"/>
+<path d="M1 4L15 4L15 3L1 3L1 4Z" fill="#C5C5C5"/>
+<path d="M1 7L15 7L15 6L1 6L1 7Z" fill="#C5C5C5"/>
+<path d="M9 9.5L9 9L1 9L1 10L9 10L9 9.5Z" fill="#C5C5C5"/>
+<path d="M9 13L9 12.5L9 12L1 12L1 13L9 13Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/edit.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M13.23 1H11.77L3.52002 9.25L3.35999 9.46997L1 13.59L2.41003 15L6.53003 12.64L6.75 12.48L15 4.22998V2.77002L13.23 1ZM2.41003 13.59L3.92004 10.59L5.37 12.04L2.41003 13.59ZM6.23999 11.53L4.46997 9.76001L12.47 1.76001L14.24 3.53003L6.23999 11.53Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/preview.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M2 2H14L15 3V13L14 14H2L1 13V3L2 2ZM2 13H14V3H2V13ZM13 4H3V7H13V4ZM12 6H4V5H12V6ZM9 12H13V8H9V12ZM10 9H12V11H10V9ZM7 8H3V9H7V8ZM3 11H7V12H3V11Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/dark/trash.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10 3H12H13V4H12V13L11 14H4L3 13V4H2V3H5V2C5 1.73478 5.10531 1.48038 5.29285 1.29285C5.48038 1.10531 5.73478 1 6 1H9C9.26522 1 9.51962 1.10531 9.70715 1.29285C9.89469 1.48038 10 1.73478 10 2V3ZM9 2H6V3H9V2ZM4 13H11V4H4V13ZM6 5H5V12H6V5ZM7 5H8V12H7V5ZM9 5H10V12H9V5Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/light/clear-all.svg

@@ -0,0 +1,7 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10.0001 12.6L10.7001 13.3L12.3001 11.7L13.9001 13.3L14.7001 12.6L13.0001 11L14.7001 9.40005L13.9001 8.60005L12.3001 10.3L10.7001 8.60005L10.0001 9.40005L11.6001 11L10.0001 12.6Z" fill="#424242"/>
+<path d="M1.00006 4L15.0001 4L15.0001 3L1.00006 3L1.00006 4Z" fill="#424242"/>
+<path d="M1.00006 7L15.0001 7L15.0001 6L1.00006 6L1.00006 7Z" fill="#424242"/>
+<path d="M9.00006 9.5L9.00006 9L1.00006 9L1.00006 10L9.00006 10L9.00006 9.5Z" fill="#424242"/>
+<path d="M9.00006 13L9.00006 12.5L9.00006 12L1.00006 12L1.00006 13L9.00006 13Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/edit.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M13.2302 1H11.7703L3.52026 9.25L3.36023 9.46997L1.00024 13.59L2.41028 15L6.53027 12.64L6.75024 12.48L15.0002 4.22998V2.77002L13.2302 1ZM2.41028 13.59L3.92029 10.59L5.37024 12.04L2.41028 13.59ZM6.24023 11.53L4.47021 9.76001L12.4702 1.76001L14.2402 3.53003L6.24023 11.53Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/preview.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M2.00024 2H14.0002L15.0002 3V13L14.0002 14H2.00024L1.00024 13V3L2.00024 2ZM2.00024 13H14.0002V3H2.00024V13ZM13.0002 4H3.00024V7H13.0002V4ZM12.0002 6H4.00024V5H12.0002V6ZM9.00024 12H13.0002V8H9.00024V12ZM10.0002 9H12.0002V11H10.0002V9ZM7.00024 8H3.00024V9H7.00024V8ZM3.00024 11H7.00024V12H3.00024V11Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/media/light/trash.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M10.0002 3H12.0002H13.0002V4H12.0002V13L11.0002 14H4.00024L3.00024 13V4H2.00024V3H5.00024V2C5.00024 1.73478 5.10555 1.48038 5.29309 1.29285C5.48063 1.10531 5.73503 1 6.00024 1H9.00024C9.26546 1 9.51986 1.10531 9.7074 1.29285C9.89493 1.48038 10.0002 1.73478 10.0002 2V3ZM9.00024 2H6.00024V3H9.00024V2ZM4.00024 13H11.0002V4H4.00024V13ZM6.00024 5H5.00024V12H6.00024V5ZM7.00024 5H8.00024V12H7.00024V5ZM9.00024 5H10.0002V12H9.00024V5Z" fill="#424242"/>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.2.1",
+  "version": "1.3.8",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,7 +13,7 @@
     "url": "https://github.com/github/vscode-codeql"
   },
   "engines": {
-    "vscode": "^1.39.0"
+    "vscode": "^1.43.0"
   },
   "categories": [
     "Programming Languages"
@@ -25,6 +25,7 @@
     "onLanguage:ql",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
+    "onView:codeQLAstViewer",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
     "onCommand:codeQLDatabases.chooseDatabaseFolder",
@@ -32,6 +33,7 @@
     "onCommand:codeQLDatabases.chooseDatabaseInternet",
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
+    "onCommand:codeQL.viewAst",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
@@ -117,7 +119,7 @@
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
           "default": 1,
-          "minimum": 1,
+          "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running queries."
         },
@@ -150,6 +152,16 @@
           "default": false,
           "description": "Enable automatically saving a modified query file when running a query."
         },
+        "codeQL.runningQueries.maxQueries": {
+          "type": "integer",
+          "default": 20,
+          "description": "Max number of simultaneous queries to run using the 'CodeQL: Run Queries' command."
+        },
+        "codeQL.resultsDisplay.pageSize": {
+          "type": "integer",
+          "default": 200,
+          "description": "Number of queries displayed per page of the results view."
+        },
         "codeQL.queryHistory.format": {
           "type": "string",
           "default": "[%t] %q on %d - %s",
@@ -159,7 +171,7 @@
           "scope": "window",
           "type": "integer",
           "default": 1,
-          "minimum": 1,
+          "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running CodeQL tests."
         }
@@ -170,6 +182,10 @@
         "command": "codeQL.runQuery",
         "title": "CodeQL: Run Query"
       },
+      {
+        "command": "codeQL.runQueries",
+        "title": "CodeQL: Run Queries in Selected Files"
+      },
       {
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
@@ -186,6 +202,10 @@
           "dark": "media/dark/folder-opened-plus.svg"
         }
       },
+      {
+        "command": "codeQLDatabases.removeOrphanedDatabases",
+        "title": "Delete unused databases"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseArchive",
         "title": "Choose Database from Archive",
@@ -214,6 +234,10 @@
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
       },
+      {
+        "command": "codeQL.viewAst",
+        "title": "CodeQL: View AST"
+      },
       {
         "command": "codeQL.upgradeCurrentDatabase",
         "title": "CodeQL: Upgrade Current Database"
@@ -280,15 +304,27 @@
       },
       {
         "command": "codeQLQueryHistory.openQuery",
-        "title": "Open Query"
-      },
-      {
-        "command": "codeQLQueryHistory.removeHistoryItem",
-        "title": "Remove History Item"
+        "title": "Open the query that produced these results",
+        "icon": {
+          "light": "media/light/edit.svg",
+          "dark": "media/dark/edit.svg"
+        }
       },
       {
         "command": "codeQLQueryHistory.itemClicked",
-        "title": "Query History Item"
+        "title": "Open Query Results",
+        "icon": {
+          "light": "media/light/preview.svg",
+          "dark": "media/dark/preview.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.removeHistoryItem",
+        "title": "Remove History Item(s)",
+        "icon": {
+          "light": "media/light/trash.svg",
+          "dark": "media/dark/trash.svg"
+        }
       },
       {
         "command": "codeQLQueryHistory.showQueryLog",
@@ -302,6 +338,18 @@
         "command": "codeQLQueryHistory.viewSarif",
         "title": "View SARIF"
       },
+      {
+        "command": "codeQLQueryHistory.viewDil",
+        "title": "View DIL"
+      },
+      {
+        "command": "codeQLQueryHistory.setLabel",
+        "title": "Set Label"
+      },
+      {
+        "command": "codeQLQueryHistory.compareWith",
+        "title": "Compare Results"
+      },
       {
         "command": "codeQLQueryResults.nextPathStep",
         "title": "CodeQL: Show Next Step on Path"
@@ -310,21 +358,29 @@
         "command": "codeQLQueryResults.previousPathStep",
         "title": "CodeQL: Show Previous Step on Path"
       },
-      {
-        "command": "codeQLQueryHistory.setLabel",
-        "title": "Set Label"
-      },
       {
         "command": "codeQL.restartQueryServer",
         "title": "CodeQL: Restart Query Server"
       },
       {
         "command": "codeQLTests.showOutputDifferences",
-        "title": "CodeQL: Show Test Output Differences"
+        "title": "Show Test Output Differences"
       },
       {
         "command": "codeQLTests.acceptOutput",
-        "title": "CodeQL: Accept Test Output"
+        "title": "Accept Test Output"
+      },
+      {
+        "command": "codeQLAstViewer.gotoCode",
+        "title": "Go To Code"
+      },
+      {
+        "command": "codeQLAstViewer.clear",
+        "title": "Clear AST",
+        "icon": {
+          "light": "media/light/clear-all.svg",
+          "dark": "media/dark/clear-all.svg"
+        }
       }
     ],
     "menus": {
@@ -358,6 +414,26 @@
           "command": "codeQLDatabases.chooseDatabaseLgtm",
           "when": "view == codeQLDatabases",
           "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.openQuery",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.itemClicked",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.removeHistoryItem",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLAstViewer.clear",
+          "when": "view == codeQLAstViewer",
+          "group": "navigation"
         }
       ],
       "view/item/context": [
@@ -401,6 +477,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
         },
+        {
+          "command": "codeQLQueryHistory.compareWith",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
         {
           "command": "codeQLQueryHistory.showQueryLog",
           "group": "9_qlCommands",
@@ -416,6 +497,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
         },
+        {
+          "command": "codeQLQueryHistory.viewDil",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -434,9 +520,14 @@
           "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
         },
         {
-          "command": "codeQL.runQuery",
+          "command": "codeQL.viewAst",
           "group": "9_qlCommands",
-          "when": "resourceLangId == ql && resourceExtname == .ql"
+          "when": "resourceScheme == codeql-zip-archive"
+        },
+        {
+          "command": "codeQL.runQueries",
+          "group": "9_qlCommands",
+          "when": "resourceScheme != codeql-zip-archive"
         }
       ],
       "commandPalette": [
@@ -444,6 +535,10 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueries",
+          "when": "false"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -452,6 +547,10 @@
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -484,6 +583,10 @@
           "command": "codeQLDatabases.chooseDatabaseArchive",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.removeOrphanedDatabases",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.chooseDatabaseInternet",
           "when": "false"
@@ -520,9 +623,33 @@
           "command": "codeQLQueryHistory.viewSarif",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.viewDil",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.setLabel",
           "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.compareWith",
+          "when": "false"
+        },
+        {
+          "command": "codeQLAstViewer.gotoCode",
+          "when": "false"
+        },
+        {
+          "command": "codeQLAstViewer.clear",
+          "when": "false"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "when": "false"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -530,6 +657,10 @@
           "command": "codeQL.runQuery",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -554,9 +685,27 @@
         {
           "id": "codeQLQueryHistory",
           "name": "Query History"
+        },
+        {
+          "id": "codeQLAstViewer",
+          "name": "AST Viewer"
         }
       ]
-    }
+    },
+    "viewsWelcome": [
+      {
+        "view": "codeQLAstViewer",
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+      },
+      {
+        "view": "codeQLQueryHistory",
+        "contents": "Run the 'CodeQL: Run Query' command on a QL query.\n[Run Query](command:codeQL.runQuery)"
+      },
+      {
+        "view": "codeQLDatabases",
+        "contents": "Add a Code QL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+      }
+    ]
   },
   "scripts": {
     "build": "gulp",
@@ -564,92 +713,95 @@
     "watch:extension": "tsc --watch",
     "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
-    "integration": "node ./out/vscode-tests/run-integration-tests.js",
+    "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
+    "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
-    "postinstall": "npm rebuild && node ./node_modules/vscode/bin/install",
-    "format": "tsfmt -r",
-    "lint": "eslint src test --ext .ts,.tsx",
+    "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
+    "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
     "format-staged": "lint-staged"
   },
   "dependencies": {
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.0.1",
     "glob-promise": "^3.4.0",
-    "js-yaml": "^3.12.0",
+    "js-yaml": "^3.14.0",
+    "minimist": "~1.2.5",
     "node-fetch": "~2.6.0",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
-    "semmle-bqrs": "^0.0.1",
-    "semmle-io-node": "^0.0.1",
-    "semmle-vscode-utils": "^0.0.1",
+    "semver": "~7.3.2",
     "tmp": "^0.1.0",
+    "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
     "vscode-jsonrpc": "^5.0.1",
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
     "vscode-test-adapter-util": "~0.7.0",
-    "minimist": "~1.2.5"
+    "zip-a-folder": "~0.0.12"
   },
   "devDependencies": {
     "@types/chai": "^4.1.7",
+    "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^8.0.0",
+    "@types/fs-extra": "^9.0.3",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
-    "@types/js-yaml": "~3.12.1",
+    "@types/gulp-sourcemaps": "0.0.32",
+    "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
-    "@types/mocha": "~5.2.7",
-    "@types/node": "^12.0.8",
+    "@types/mocha": "^8.0.4",
+    "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
+    "@types/proxyquire": "~1.3.28",
     "@types/react": "^16.8.17",
     "@types/react-dom": "^16.8.4",
     "@types/sarif": "~2.1.2",
+    "@types/semver": "~7.2.0",
+    "@types/sinon": "~7.5.2",
+    "@types/sinon-chai": "~3.2.3",
+    "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.39.0",
+    "@types/vscode": "^1.43.0",
     "@types/webpack": "^4.32.1",
     "@types/xml2js": "~0.4.4",
-    "build-tasks": "^0.0.1",
+    "@typescript-eslint/eslint-plugin": "~2.23.0",
+    "@typescript-eslint/parser": "~2.23.0",
+    "ansi-colors": "^4.1.1",
     "chai": "^4.2.0",
+    "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
+    "eslint": "~6.8.0",
+    "eslint-plugin-react": "~7.19.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
-    "mocha": "~6.2.1",
+    "husky": "~4.2.5",
+    "jsonc-parser": "^2.3.0",
+    "lint-staged": "~10.2.2",
+    "mocha": "^8.2.1",
     "mocha-sinon": "~2.1.0",
     "npm-run-all": "^4.1.5",
+    "prettier": "~2.0.5",
+    "proxyquire": "~2.1.3",
     "sinon": "~9.0.0",
+    "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
     "ts-loader": "^5.4.5",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "^3.7.2",
-    "typescript-config": "^0.0.1",
+    "typescript": "~3.8.3",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
     "vscode-test": "^1.4.0",
     "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2",
-    "eslint": "~6.8.0",
-    "@typescript-eslint/eslint-plugin": "~2.23.0",
-    "@typescript-eslint/parser": "~2.23.0",
-    "chai-as-promised": "~7.1.1",
-    "@types/chai-as-promised": "~7.1.2",
-    "@types/sinon": "~7.5.2",
-    "sinon-chai": "~3.5.0",
-    "@types/sinon-chai": "~3.2.3",
-    "proxyquire": "~2.1.3",
-    "@types/proxyquire": "~1.3.28",
-    "eslint-plugin-react": "~7.19.0",
-    "husky": "~4.2.5",
-    "lint-staged": "~10.2.2",
-    "prettier": "~2.0.5"
+    "webpack-cli": "^3.3.2"
   },
   "husky": {
     "hooks": {
@@ -662,7 +814,8 @@
       "prettier --write"
     ],
     "./**/*.{ts,tsx}": [
-      "tsfmt -r"
+      "tsfmt -r",
+      "eslint --fix"
     ]
   }
 }

extensions/ql-vscode/src/adapt.ts

@@ -1,103 +0,0 @@
-import { DecodedBqrsChunk, ResultSetSchema, ColumnKind, Column, ColumnValue } from "./bqrs-cli-types";
-import { LocationValue, ResultSetSchema as AdaptedSchema, ColumnSchema, ColumnType, LocationStyle } from 'semmle-bqrs';
-
-// FIXME: This is a temporary bit of impedance matching to convert
-// from the types provided by ./bqrs-cli-types, to the types used by
-// the view layer.
-//
-// The reason that it is benign for now is that it is only used by
-// feature-flag-guarded codepaths that won't be encountered by normal
-// users. It is not yet guaranteed to produce correct output for raw
-// results.
-//
-// Eventually, the view layer should be refactored to directly accept data
-// of types coming from bqrs-cli-types, and this file can be deleted.
-
-export type ResultRow = ResultValue[];
-
-export interface ResultElement {
-  label: string;
-  location?: LocationValue;
-}
-
-export interface ResultUri {
-  uri: string;
-}
-
-export type ResultValue = ResultElement | ResultUri | string;
-
-export interface RawResultSet {
-  readonly schema: AdaptedSchema;
-  readonly rows: readonly ResultRow[];
-}
-
-function adaptKind(kind: ColumnKind): ColumnType {
-  // XXX what about 'u'?
-  if (kind === 'e') {
-    return { type: 'e', primitiveType: 's', locationStyle: LocationStyle.FivePart, hasLabel: true };
-  }
-  else {
-    return { type: kind };
-  }
-}
-
-function adaptColumn(col: Column): ColumnSchema {
-  return { name: col.name!, type: adaptKind(col.kind) };
-}
-
-export function adaptSchema(schema: ResultSetSchema): AdaptedSchema {
-  return {
-    columns: schema.columns.map(adaptColumn),
-    name: schema.name,
-    tupleCount: schema.rows,
-    version: 0,
-  };
-}
-
-export function adaptValue(val: ColumnValue): ResultValue {
-  // XXX taking a lot of incorrect shortcuts here
-
-  if (typeof val === 'string') {
-    return val;
-  }
-
-  if (typeof val === 'number' || typeof val === 'boolean') {
-    return val + '';
-  }
-
-  const url = val.url;
-
-  if (typeof url === 'string') {
-    return url;
-  }
-
-  if (url === undefined) {
-    return 'none';
-  }
-
-  return {
-    label: val.label || '',
-    location: {
-      t: LocationStyle.FivePart,
-      lineStart: url.startLine,
-      lineEnd: url.endLine,
-      colStart: url.startColumn,
-      colEnd: url.endColumn,
-      // FIXME: This seems definitely wrong. Should we be using
-      // something like the code in sarif-utils.ts?
-      file: url.uri.replace(/file:/, ''),
-    }
-  };
-
-}
-
-export function adaptRow(row: ColumnValue[]): ResultRow {
-  return row.map(adaptValue);
-}
-
-export function adaptBqrs(schema: AdaptedSchema, page: DecodedBqrsChunk): RawResultSet {
-  return {
-    schema,
-    rows: page.tuples.map(adaptRow),
-  };
-}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -84,12 +84,25 @@ export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
   // This lets us separate the paths, ignoring the leading slash if we added one.
   const sourceArchiveZipPathEndIndex = sourceArchiveZipPathStartIndex + sourceArchiveZipPath.length;
   const authority = `${sourceArchiveZipPathStartIndex}-${sourceArchiveZipPathEndIndex}`;
-  return vscode.Uri.parse(zipArchiveScheme + ':/').with({
+  return vscode.Uri.parse(zipArchiveScheme + ':/', true).with({
     path: encodedPath,
     authority,
   });
 }
 
+/**
+ * Convenience method to create a codeql-zip-archive with a path to the root
+ * archive
+ *
+ * @param pathToArchive the filesystem path to the root of the archive
+ */
+export function encodeArchiveBasePath(sourceArchiveZipPath: string) {
+  return encodeSourceArchiveUri({
+    sourceArchiveZipPath,
+    pathWithinSourceArchive: ''
+  });
+}
+
 const sourceArchiveUriAuthorityPattern = /^(\d+)-(\d+)$/;
 
 class InvalidSourceArchiveUriError extends Error {
@@ -100,6 +113,14 @@ class InvalidSourceArchiveUriError extends Error {
 
 /** Decodes an encoded source archive URI into its corresponding paths. Inverse of `encodeSourceArchiveUri`. */
 export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
+  if (!uri.authority) {
+    // Uri is malformed, but this is recoverable
+    logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
+    return {
+      pathWithinSourceArchive: '/',
+      sourceArchiveZipPath: uri.path
+    };
+  }
   const match = sourceArchiveUriAuthorityPattern.exec(uri.authority);
   if (match === null)
     throw new InvalidSourceArchiveUriError(uri);
@@ -108,7 +129,7 @@ export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
   if (isNaN(zipPathStartIndex) || isNaN(zipPathEndIndex))
     throw new InvalidSourceArchiveUriError(uri);
   return {
-    pathWithinSourceArchive: uri.path.substring(zipPathEndIndex),
+    pathWithinSourceArchive: uri.path.substring(zipPathEndIndex) || '/',
     sourceArchiveZipPath: uri.path.substring(zipPathStartIndex, zipPathEndIndex),
   };
 }

extensions/ql-vscode/src/astViewer.ts

@@ -0,0 +1,193 @@
+import {
+  window,
+  TreeDataProvider,
+  EventEmitter,
+  Event,
+  ProviderResult,
+  TreeItemCollapsibleState,
+  TreeItem,
+  TreeView,
+  TextEditorSelectionChangeEvent,
+  TextEditorSelectionChangeKind,
+  Location,
+  Range
+} from 'vscode';
+import * as path from 'path';
+
+import { DatabaseItem } from './databases';
+import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
+import { showLocation } from './interface-utils';
+import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
+import { commandRunner } from './helpers';
+import { DisposableObject } from './vscode-utils/disposable-object';
+
+export interface AstItem {
+  id: BqrsId;
+  label?: string;
+  location?: UrlValue;
+  fileLocation?: Location;
+  children: ChildAstItem[];
+  order: number;
+}
+
+export interface ChildAstItem extends AstItem {
+  parent: ChildAstItem | AstItem;
+}
+
+class AstViewerDataProvider extends DisposableObject implements TreeDataProvider<AstItem> {
+
+  public roots: AstItem[] = [];
+  public db: DatabaseItem | undefined;
+
+  private _onDidChangeTreeData =
+    this.push(new EventEmitter<AstItem | undefined>());
+  readonly onDidChangeTreeData: Event<AstItem | undefined> =
+    this._onDidChangeTreeData.event;
+
+  constructor() {
+    super();
+    this.push(
+      commandRunner('codeQLAstViewer.gotoCode',
+        async (item: AstItem) => {
+          await showLocation(item.fileLocation);
+        })
+    );
+  }
+
+  refresh(): void {
+    this._onDidChangeTreeData.fire();
+  }
+  getChildren(item?: AstItem): ProviderResult<AstItem[]> {
+    const children = item ? item.children : this.roots;
+    return children.sort((c1, c2) => (c1.order - c2.order));
+  }
+
+  getParent(item: ChildAstItem): ProviderResult<AstItem> {
+    return item.parent;
+  }
+
+  getTreeItem(item: AstItem): TreeItem {
+    const line = this.extractLineInfo(item?.location);
+
+    const state = item.children.length
+      ? TreeItemCollapsibleState.Collapsed
+      : TreeItemCollapsibleState.None;
+    const treeItem = new TreeItem(item.label || '', state);
+    treeItem.description = line ? `Line ${line}` : '';
+    treeItem.id = String(item.id);
+    treeItem.tooltip = `${treeItem.description} ${treeItem.label}`;
+    treeItem.command = {
+      command: 'codeQLAstViewer.gotoCode',
+      title: 'Go To Code',
+      tooltip: `Go To ${item.location}`,
+      arguments: [item]
+    };
+    return treeItem;
+  }
+
+  private extractLineInfo(loc?: UrlValue) {
+    if (!loc) {
+      return '';
+    } else if (isStringLoc(loc)) {
+      return loc;
+    } else if (isWholeFileLoc(loc)) {
+      return loc.uri;
+    } else if (isLineColumnLoc(loc)) {
+      return loc.startLine;
+    } else {
+      return '';
+    }
+  }
+}
+
+export class AstViewer extends DisposableObject {
+  private treeView: TreeView<AstItem>;
+  private treeDataProvider: AstViewerDataProvider;
+  private currentFile: string | undefined;
+
+  constructor() {
+    super();
+
+    this.treeDataProvider = new AstViewerDataProvider();
+    this.treeView = window.createTreeView('codeQLAstViewer', {
+      treeDataProvider: this.treeDataProvider,
+      showCollapseAll: true
+    });
+
+    this.push(this.treeView);
+    this.push(this.treeDataProvider);
+    this.push(
+      commandRunner('codeQLAstViewer.clear', async () => {
+        this.clear();
+      })
+    );
+    this.push(window.onDidChangeTextEditorSelection(this.updateTreeSelection, this));
+  }
+
+  updateRoots(roots: AstItem[], db: DatabaseItem, fileName: string) {
+    this.treeDataProvider.roots = roots;
+    this.treeDataProvider.db = db;
+    this.treeDataProvider.refresh();
+    this.treeView.message = `AST for ${path.basename(fileName)}`;
+    this.treeView.reveal(roots[0], { focus: false });
+    this.currentFile = fileName;
+  }
+
+  private updateTreeSelection(e: TextEditorSelectionChangeEvent) {
+    function isInside(selectedRange: Range, astRange?: Range): boolean {
+      return !!astRange?.contains(selectedRange);
+    }
+
+    // Recursively iterate all children until we find the node with the smallest
+    // range that contains the selection.
+    // Some nodes do not have a location, but their children might, so must
+    // recurse though location-less AST nodes to see if children are correct.
+    function findBest(selectedRange: Range, items?: AstItem[]): AstItem | undefined {
+      if (!items || !items.length) {
+        return;
+      }
+      for (const item of items) {
+        let candidate: AstItem | undefined = undefined;
+        if (isInside(selectedRange, item.fileLocation?.range)) {
+          candidate = item;
+        }
+        // always iterate through children since the location of an AST node in code QL does not
+        // always cover the complete text of the node.
+        candidate = findBest(selectedRange, item.children) || candidate;
+        if (candidate) {
+          return candidate;
+        }
+      }
+      return;
+    }
+
+    // Avoid recursive tree-source code updates.
+    if (e.kind === TextEditorSelectionChangeKind.Command) {
+      return;
+    }
+
+    if (
+      this.treeView.visible &&
+      e.textEditor.document.uri.fsPath === this.currentFile &&
+      e.selections.length === 1
+    ) {
+      const selection = e.selections[0];
+      const range = selection.anchor.isBefore(selection.active)
+        ? new Range(selection.anchor, selection.active)
+        : new Range(selection.active, selection.anchor);
+
+      const targetItem = findBest(range, this.treeDataProvider.roots);
+      if (targetItem) {
+        this.treeView.reveal(targetItem);
+      }
+    }
+  }
+
+  private clear() {
+    this.treeDataProvider.roots = [];
+    this.treeDataProvider.db = undefined;
+    this.treeDataProvider.refresh();
+    this.treeView.message = undefined;
+    this.currentFile = undefined;
+  }
+}

extensions/ql-vscode/src/cli-version.ts

@@ -1,96 +1,24 @@
-import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "./logging";
+import * as semver from 'semver';
+import { runCodeQlCliCommand } from './cli';
+import { Logger } from './logging';
 
 /**
  * Get the version of a CodeQL CLI.
  */
-export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<Version | undefined> {
-  const output: string = await runCodeQlCliCommand(
-    codeQlPath,
-    ["version"],
-    ["--format=terse"],
-    "Checking CodeQL version",
-    logger
-  );
-  return tryParseVersionString(output.trim());
-}
-
-/**
- * Try to parse a version string, returning undefined if we can't parse it.
- * 
- * Version strings must contain a major, minor, and patch version.  They may optionally
- * start with "v" and may optionally contain some "tail" string after the major, minor, and
- * patch versions, for example as in `v2.1.0+baf5bff`.
- */
-export function tryParseVersionString(versionString: string): Version | undefined {
-  const match = versionString.match(versionRegex);
-  if (match === null) {
+export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): Promise<semver.SemVer | undefined> {
+  try {
+    const output: string = await runCodeQlCliCommand(
+      codeQlPath,
+      ['version'],
+      ['--format=terse'],
+      'Checking CodeQL version',
+      logger
+    );
+    return semver.parse(output.trim()) || undefined;
+  } catch (e) {
+    // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
+    // Either way, we can't determine compatibility.
+    logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
     return undefined;
   }
-  return {
-    buildMetadata: match[5],
-    majorVersion: Number.parseInt(match[1], 10),
-    minorVersion: Number.parseInt(match[2], 10),
-    patchVersion: Number.parseInt(match[3], 10),
-    prereleaseVersion: match[4],
-    rawString: versionString,
-  };
-}
-
-/**
- * Regex for parsing semantic versions
- * 
- * From the semver spec https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
- */
-const versionRegex = new RegExp(String.raw`^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)` +
-  String.raw`(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?` +
-  String.raw`(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`);
-
-/**
- * A version of the CodeQL CLI.
- */
-export interface Version {
-  /**
-   * Build metadata
-   * 
-   * For example, this will be `abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   * Build metadata must be ignored when comparing versions. 
-   */
-  buildMetadata: string | undefined;
-
-  /**
-   * Major version number
-   * 
-   * For example, this will be `2` for version 2.1.0-alpha.1+abcdef0.
-   */
-  majorVersion: number;
-
-  /**
-   * Minor version number
-   * 
-   * For example, this will be `1` for version 2.1.0-alpha.1+abcdef0.
-   */
-  minorVersion: number;
-
-  /**
-   * Patch version number
-   * 
-   * For example, this will be `0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  patchVersion: number;
-
-  /**
-   * Prerelease version
-   * 
-   * For example, this will be `alpha.1` for version 2.1.0-alpha.1+abcdef0.
-   * The prerelease version must be considered when comparing versions.
-   */
-  prereleaseVersion: string | undefined;
-
-  /**
-   * Raw version string
-   * 
-   * For example, this will be `2.1.0-alpha.1+abcdef0` for version 2.1.0-alpha.1+abcdef0.
-   */
-  rawString: string;
 }

extensions/ql-vscode/src/cli.ts

@@ -4,21 +4,25 @@ import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
+import { SemVer } from 'semver';
 import { Readable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as tk from 'tree-kill';
-import * as util from 'util';
+import { promisify } from 'util';
 import { CancellationToken, Disposable } from 'vscode';
-import { BQRSInfo, DecodedBqrsChunk } from "./bqrs-cli-types";
-import { DistributionProvider } from "./distribution";
-import { assertNever } from "./helpers-pure";
-import { QueryMetadata, SortDirection } from "./interface-types";
-import { Logger, ProgressReporter } from "./logging";
+
+import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
+import { CliConfig } from './config';
+import { DistributionProvider, FindDistributionResultKind } from './distribution';
+import { assertNever } from './pure/helpers-pure';
+import { QueryMetadata, SortDirection } from './pure/interface-types';
+import { Logger, ProgressReporter } from './logging';
+import { CompilationMessage } from './pure/messages';
 
 /**
  * The version of the SARIF format that we are using.
  */
-const SARIF_FORMAT = "sarifv2.1.0";
+const SARIF_FORMAT = 'sarifv2.1.0';
 
 /**
  * Flags to pass to all cli commands.
@@ -46,6 +50,7 @@ export interface DbInfo {
   sourceArchiveRoot: string;
   datasetFolder: string;
   logsFolder: string;
+  languages: string[];
 }
 
 /**
@@ -87,10 +92,23 @@ export interface TestRunOptions {
 export interface TestCompleted {
   test: string;
   pass: boolean;
-  messages: string[];
+  messages: CompilationMessage[];
   compilationMs: number;
   evaluationMs: number;
   expected: string;
+  diff: string[] | undefined;
+}
+
+/**
+ * Optional arguments for the `bqrsDecode` function
+ */
+interface BqrsDecodeOptions {
+  /** How many results to get. */
+  pageSize?: number;
+  /** The 0-based index of the first result to get. */
+  offset?: number;
+  /** The entity names to retrieve from the bqrs file. Default is url, string */
+  entities?: string[];
 }
 
 /**
@@ -101,6 +119,16 @@ export interface TestCompleted {
  */
 export class CodeQLCliServer implements Disposable {
 
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  private static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
+
+  /**
+   * CLI version where languages are exposed during a `codeql resolve database` command.
+   */
+  private static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
+
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
   /** Queue of future commands*/
@@ -110,18 +138,38 @@ export class CodeQLCliServer implements Disposable {
   /**  A buffer with a single null byte. */
   nullBuffer: Buffer;
 
-  constructor(private config: DistributionProvider, private logger: Logger) {
+  /** Version of current cli, lazily computed by the `getVersion()` method */
+  private _version: SemVer | undefined;
+
+  /** Path to current codeQL executable, or undefined if not running yet. */
+  codeQlPath: string | undefined;
+
+  /**
+   * When set to true, ignore some modal popups and assume user has clicked "yes".
+   */
+  public quiet = false;
+
+  constructor(
+    private distributionProvider: DistributionProvider,
+    private cliConfig: CliConfig,
+    private logger: Logger
+  ) {
     this.commandQueue = [];
     this.commandInProcess = false;
     this.nullBuffer = Buffer.alloc(1);
-    if (this.config.onDidChangeDistribution) {
-      this.config.onDidChangeDistribution(() => {
+    if (this.distributionProvider.onDidChangeDistribution) {
+      this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
       });
     }
+    if (this.cliConfig.onDidChangeConfiguration) {
+      this.cliConfig.onDidChangeConfiguration(() => {
+        this.restartCliServer();
+        this._version = undefined;
+      });
+    }
   }
 
-
   dispose(): void {
     this.killProcessIfRunning();
   }
@@ -131,7 +179,7 @@ export class CodeQLCliServer implements Disposable {
       // Tell the Java CLI server process to shut down.
       this.logger.log('Sending shutdown request');
       try {
-        this.process.stdin.write(JSON.stringify(["shutdown"]), "utf8");
+        this.process.stdin.write(JSON.stringify(['shutdown']), 'utf8');
         this.process.stdin.write(this.nullBuffer);
         this.logger.log('Sent shutdown request');
       } catch (e) {
@@ -176,7 +224,7 @@ export class CodeQLCliServer implements Disposable {
    * Get the path to the CodeQL CLI distribution, or throw an exception if not found.
    */
   private async getCodeQlPath(): Promise<string> {
-    const codeqlPath = await this.config.getCodeQlPathWithoutVersionCheck();
+    const codeqlPath = await this.distributionProvider.getCodeQlPathWithoutVersionCheck();
     if (!codeqlPath) {
       throw new Error('Failed to find CodeQL distribution.');
     }
@@ -187,14 +235,21 @@ export class CodeQLCliServer implements Disposable {
    * Launch the cli server
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
-    const config = await this.getCodeQlPath();
-    return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, _data => { /**/ });
+    const codeQlPath = await this.getCodeQlPath();
+    return await spawnServer(
+      codeQlPath,
+      'CodeQL CLI Server',
+      ['execute', 'cli-server'],
+      [],
+      this.logger,
+      _data => { /**/ }
+    );
   }
 
   private async runCodeQlCliInternal(command: string[], commandArgs: string[], description: string): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
-      throw new Error("runCodeQlCliInternal called while cli was running");
+      throw new Error('runCodeQlCliInternal called while cli was running');
     }
     this.commandInProcess = true;
     try {
@@ -209,7 +264,7 @@ export class CodeQLCliServer implements Disposable {
 
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
-      const argsString = args.join(" ");
+      const argsString = args.join(' ');
       this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
         await new Promise((resolve, reject) => {
@@ -228,16 +283,16 @@ export class CodeQLCliServer implements Disposable {
             stderrBuffers.push(newData);
           });
           // Listen for process exit.
-          process.addListener("close", (code) => reject(code));
+          process.addListener('close', (code) => reject(code));
           // Write the command followed by a null terminator.
-          process.stdin.write(JSON.stringify(args), "utf8");
+          process.stdin.write(JSON.stringify(args), 'utf8');
           process.stdin.write(this.nullBuffer);
         });
         // Join all the data together
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
-        const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        this.logger.log(`CLI command succeeded.`);
+        const data = fullBuffer.toString('utf8', 0, fullBuffer.length - 1);
+        this.logger.log('CLI command succeeded.');
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
@@ -246,15 +301,15 @@ export class CodeQLCliServer implements Disposable {
         const newError =
           stderrBuffers.length == 0
             ? new Error(`${description} failed: ${err}`)
-            : new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString("utf8")}`);
+            : new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString('utf8')}`);
         newError.stack += (err.stack || '');
         throw newError;
       } finally {
-        this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
         // Remove the listeners we set up.
         process.stdout.removeAllListeners('data');
         process.stderr.removeAllListeners('data');
-        process.removeAllListeners("close");
+        process.removeAllListeners('close');
       }
     } finally {
       this.commandInProcess = false;
@@ -413,10 +468,10 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(workspaces: string[], queryPath: string): Promise<QuerySetup> {
     const subcommandArgs = [
       '--query', queryPath,
-      "--additional-packs",
+      '--additional-packs',
       workspaces.join(path.delimiter)
     ];
-    return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, "Resolving library paths");
+    return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, 'Resolving library paths');
   }
 
   /**
@@ -428,7 +483,11 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       testPath
     ];
-    return await this.runJsonCodeQlCliCommand<ResolvedTests>(['resolve', 'tests'], subcommandArgs, 'Resolving tests');
+    return await this.runJsonCodeQlCliCommand<ResolvedTests>(
+      ['resolve', 'tests', '--strict-test-discovery'],
+      subcommandArgs,
+      'Resolving tests'
+    );
   }
 
   /**
@@ -443,7 +502,8 @@ export class CodeQLCliServer implements Disposable {
 
     const subcommandArgs = [
       '--additional-packs', workspaces.join(path.delimiter),
-      '--threads', '8',
+      '--threads',
+      this.cliConfig.numberTestThreads.toString(),
       ...testPaths
     ];
 
@@ -458,7 +518,7 @@ export class CodeQLCliServer implements Disposable {
    * @param queryPath The path to the query.
    */
   async resolveMetadata(queryPath: string): Promise<QueryMetadata> {
-    return await this.runJsonCodeQlCliCommand<QueryMetadata>(['resolve', 'metadata'], [queryPath], "Resolving query metadata");
+    return await this.runJsonCodeQlCliCommand<QueryMetadata>(['resolve', 'metadata'], [queryPath], 'Resolving query metadata');
   }
 
   /**
@@ -474,7 +534,7 @@ export class CodeQLCliServer implements Disposable {
     if (queryMemoryMb !== undefined) {
       args.push('--ram', queryMemoryMb.toString());
     }
-    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, "Resolving RAM settings", progressReporter);
+    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', progressReporter);
   }
   /**
    * Gets the headers (and optionally pagination info) of a bqrs.
@@ -483,53 +543,56 @@ export class CodeQLCliServer implements Disposable {
    */
   async bqrsInfo(bqrsPath: string, pageSize?: number): Promise<BQRSInfo> {
     const subcommandArgs = (
-      pageSize ? ["--paginate-rows", pageSize.toString()] : []
+      pageSize ? ['--paginate-rows', pageSize.toString()] : []
     ).concat(
       bqrsPath
     );
-    return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, "Reading bqrs header");
+    return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, 'Reading bqrs header');
   }
 
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
   * @param resultSet The result set to get.
-  * @param pageSize How many results to get.
-  * @param offset The 0-based index of the first result to get.
+  * @param options Optional BqrsDecodeOptions arguments
   */
-  async bqrsDecode(bqrsPath: string, resultSet: string, pageSize?: number, offset?: number): Promise<DecodedBqrsChunk> {
-    const subcommandArgs = [
-      "--entities=url,string",
-      "--result-set", resultSet,
-    ].concat(
-      pageSize ? ["--rows", pageSize.toString()] : []
-    ).concat(
-      offset ? ["--start-at", offset.toString()] : []
-    ).concat([bqrsPath]);
-    return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, "Reading bqrs data");
-  }
+  async bqrsDecode(
+    bqrsPath: string,
+    resultSet: string,
+    { pageSize, offset, entities = ['url', 'string'] }: BqrsDecodeOptions = {}
+  ): Promise<DecodedBqrsChunk> {
 
+    const subcommandArgs = [
+      `--entities=${entities.join(',')}`,
+      '--result-set', resultSet,
+    ].concat(
+      pageSize ? ['--rows', pageSize.toString()] : []
+    ).concat(
+      offset ? ['--start-at', offset.toString()] : []
+    ).concat([bqrsPath]);
+    return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
+  }
 
   async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     const args = [
       `-t=kind=${metadata.kind}`,
       `-t=id=${metadata.id}`,
-      "--output", interpretedResultsPath,
-      "--format", SARIF_FORMAT,
+      '--output', interpretedResultsPath,
+      '--format', SARIF_FORMAT,
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      "--no-group-results",
+      '--no-group-results',
     ];
     if (sourceInfo !== undefined) {
       args.push(
-        "--source-archive", sourceInfo.sourceArchive,
-        "--source-location-prefix", sourceInfo.sourceLocationPrefix
+        '--source-archive', sourceInfo.sourceArchive,
+        '--source-location-prefix', sourceInfo.sourceLocationPrefix
       );
     }
     args.push(resultsPath);
-    await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, "Interpreting query results");
+    await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
 
     let output: string;
     try {
@@ -549,9 +612,9 @@ export class CodeQLCliServer implements Disposable {
     const sortDirectionStrings = sortDirections.map(direction => {
       switch (direction) {
         case SortDirection.asc:
-          return "asc";
+          return 'asc';
         case SortDirection.desc:
-          return "desc";
+          return 'desc';
         default:
           return assertNever(direction);
       }
@@ -559,14 +622,14 @@ export class CodeQLCliServer implements Disposable {
 
     await this.runCodeQlCliCommand(['bqrs', 'decode'],
       [
-        "--format=bqrs",
+        '--format=bqrs',
         `--result-set=${resultSet}`,
         `--output=${sortedResultsPath}`,
-        `--sort-key=${sortKeys.join(",")}`,
-        `--sort-direction=${sortDirectionStrings.join(",")}`,
+        `--sort-key=${sortKeys.join(',')}`,
+        `--sort-direction=${sortDirectionStrings.join(',')}`,
         resultsPath
       ],
-      "Sorting query results");
+      'Sorting query results');
   }
 
 
@@ -576,7 +639,7 @@ export class CodeQLCliServer implements Disposable {
    */
   resolveDatabase(databasePath: string): Promise<DbInfo> {
     return this.runJsonCodeQlCliCommand(['resolve', 'database'], [databasePath],
-      "Resolving database");
+      'Resolving database');
   }
 
   /**
@@ -591,7 +654,7 @@ export class CodeQLCliServer implements Disposable {
     return this.runJsonCodeQlCliCommand<UpgradesInfo>(
       ['resolve', 'upgrades'],
       args,
-      "Resolving database upgrade scripts",
+      'Resolving database upgrade scripts',
     );
   }
 
@@ -604,14 +667,14 @@ export class CodeQLCliServer implements Disposable {
    */
   resolveQlpacks(additionalPacks: string[], searchPath?: string[]): Promise<QlpacksInfo> {
     const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
-    if (searchPath !== undefined) {
+    if (searchPath?.length) {
       args.push('--search-path', path.join(...searchPath));
     }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(
       ['resolve', 'qlpacks'],
       args,
-      "Resolving qlpack information",
+      'Resolving qlpack information',
     );
   }
 
@@ -632,9 +695,50 @@ export class CodeQLCliServer implements Disposable {
     return this.runJsonCodeQlCliCommand<string[]>(
       ['resolve', 'queries'],
       args,
-      "Resolving queries",
+      'Resolving queries',
     );
   }
+
+  async generateDil(qloFile: string, outFile: string): Promise<void> {
+    const extraArgs = await this.supportsDecompileDil()
+      ? ['--kind', 'dil', '-o', outFile, qloFile]
+      : ['-o', outFile, qloFile];
+    await this.runCodeQlCliCommand(
+      ['query', 'decompile'],
+      extraArgs,
+      'Generating DIL',
+    );
+  }
+
+  public async getVersion() {
+    if (!this._version) {
+      this._version = await this.refreshVersion();
+    }
+    return this._version;
+  }
+
+  private async supportsDecompileDil() {
+    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0;
+  }
+
+  public async supportsLanguageName() {
+    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_LANGUAGE) >= 0;
+  }
+
+  private async refreshVersion() {
+    const distribution = await this.distributionProvider.getDistribution();
+    switch (distribution.kind) {
+      case FindDistributionResultKind.CompatibleDistribution:
+      // eslint-disable-next-line no-fallthrough
+      case FindDistributionResultKind.IncompatibleDistribution:
+        return distribution.version;
+
+      default:
+        // We should not get here because if no distributions are available, then
+        // the cli class is never instantiated.
+        throw new Error('No distribution found');
+    }
+  }
 }
 
 /**
@@ -666,7 +770,7 @@ export function spawnServer(
 
   // Start the server process.
   const base = codeqlPath;
-  const argsString = args.join(" ");
+  const argsString = args.join(' ');
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Starting ${name}` });
   }
@@ -692,7 +796,7 @@ export function spawnServer(
 
 /**
  * Runs a CodeQL CLI command without invoking the CLI server, returning the output as a string.
- * @param config The configuration containing the path to the CLI.
+ * @param codeQlPath The path to the CLI.
  * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
  * @param commandArgs The arguments to pass to the `codeql` command.
  * @param description Description of the action being run, to be shown in log and error messages.
@@ -700,18 +804,25 @@ export function spawnServer(
  * @param progressReporter Used to output progress messages, e.g. to the status bar.
  * @returns The contents of the command's stdout, if the command succeeded.
  */
-export async function runCodeQlCliCommand(codeQlPath: string, command: string[], commandArgs: string[], description: string, logger: Logger, progressReporter?: ProgressReporter): Promise<string> {
+export async function runCodeQlCliCommand(
+  codeQlPath: string,
+  command: string[],
+  commandArgs: string[],
+  description: string,
+  logger: Logger,
+  progressReporter?: ProgressReporter
+): Promise<string> {
   // Add logging arguments first, in case commandArgs contains positional parameters.
   const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
-  const argsString = args.join(" ");
+  const argsString = args.join(' ');
   try {
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
     }
     logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
-    const result = await util.promisify(child_process.execFile)(codeQlPath, args);
+    const result = await promisify(child_process.execFile)(codeQlPath, args);
     logger.log(result.stderr);
-    logger.log(`CLI command succeeded.`);
+    logger.log('CLI command succeeded.');
     return result.stdout;
   } catch (err) {
     throw new Error(`${description} failed: ${err.stderr || err}`);
@@ -815,3 +926,16 @@ async function logStream(stream: Readable, logger: Logger): Promise<void> {
     logger.log(line);
   }
 }
+
+
+export function shouldDebugIdeServer() {
+  return 'IDE_SERVER_JAVA_DEBUG' in process.env
+    && process.env.IDE_SERVER_JAVA_DEBUG !== '0'
+    && process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}
+
+export function shouldDebugQueryServer() {
+  return 'QUERY_SERVER_JAVA_DEBUG' in process.env
+    && process.env.QUERY_SERVER_JAVA_DEBUG !== '0'
+    && process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}

extensions/ql-vscode/src/compare/compare-interface.ts

@@ -0,0 +1,277 @@
+import { DisposableObject } from '../vscode-utils/disposable-object';
+import {
+  WebviewPanel,
+  ExtensionContext,
+  window as Window,
+  ViewColumn,
+  Uri,
+} from 'vscode';
+import * as path from 'path';
+
+import { tmpDir } from '../run-queries';
+import { CompletedQuery } from '../query-results';
+import {
+  FromCompareViewMessage,
+  ToCompareViewMessage,
+  QueryCompareResult,
+} from '../pure/interface-types';
+import { Logger } from '../logging';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseManager } from '../databases';
+import { getHtmlForWebview, jumpToLocation } from '../interface-utils';
+import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../pure/bqrs-cli-types';
+import resultsDiff from './resultsDiff';
+
+interface ComparePair {
+  from: CompletedQuery;
+  to: CompletedQuery;
+}
+
+export class CompareInterfaceManager extends DisposableObject {
+  private comparePair: ComparePair | undefined;
+  private panel: WebviewPanel | undefined;
+  private panelLoaded = false;
+  private panelLoadedCallBacks: (() => void)[] = [];
+
+  constructor(
+    private ctx: ExtensionContext,
+    private databaseManager: DatabaseManager,
+    private cliServer: CodeQLCliServer,
+    private logger: Logger,
+    private showQueryResultsCallback: (
+      item: CompletedQuery
+    ) => Promise<void>
+  ) {
+    super();
+  }
+
+  async showResults(
+    from: CompletedQuery,
+    to: CompletedQuery,
+    selectedResultSetName?: string
+  ) {
+    this.comparePair = { from, to };
+    this.getPanel().reveal(undefined, true);
+
+    await this.waitForPanelLoaded();
+    const [
+      commonResultSetNames,
+      currentResultSetName,
+      fromResultSet,
+      toResultSet,
+    ] = await this.findCommonResultSetNames(
+      from,
+      to,
+      selectedResultSetName
+    );
+    if (currentResultSetName) {
+      let rows: QueryCompareResult | undefined;
+      let message: string | undefined;
+      try {
+        rows = this.compareResults(fromResultSet, toResultSet);
+      } catch (e) {
+        message = e.message;
+      }
+
+      await this.postMessage({
+        t: 'setComparisons',
+        stats: {
+          fromQuery: {
+            // since we split the description into several rows
+            // only run interpolation if the label is user-defined
+            // otherwise we will wind up with duplicated rows
+            name: from.options.label
+              ? from.interpolate(from.getLabel())
+              : from.queryName,
+            status: from.statusString,
+            time: from.time,
+          },
+          toQuery: {
+            name: to.options.label
+              ? to.interpolate(to.getLabel())
+              : to.queryName,
+            status: to.statusString,
+            time: to.time,
+          },
+        },
+        columns: fromResultSet.schema.columns,
+        commonResultSetNames,
+        currentResultSetName: currentResultSetName,
+        rows,
+        message,
+        datebaseUri: to.database.databaseUri,
+      });
+    }
+  }
+
+  getPanel(): WebviewPanel {
+    if (this.panel == undefined) {
+      const { ctx } = this;
+      const panel = (this.panel = Window.createWebviewPanel(
+        'compareView',
+        'Compare CodeQL Query Results',
+        { viewColumn: ViewColumn.Active, preserveFocus: true },
+        {
+          enableScripts: true,
+          enableFindWidget: true,
+          retainContextWhenHidden: true,
+          localResourceRoots: [
+            Uri.file(tmpDir.name),
+            Uri.file(path.join(this.ctx.extensionPath, 'out')),
+          ],
+        }
+      ));
+      this.panel.onDidDispose(
+        () => {
+          this.panel = undefined;
+          this.comparePair = undefined;
+        },
+        null,
+        ctx.subscriptions
+      );
+
+      const scriptPathOnDisk = Uri.file(
+        ctx.asAbsolutePath('out/compareView.js')
+      );
+
+      const stylesheetPathOnDisk = Uri.file(
+        ctx.asAbsolutePath('out/resultsView.css')
+      );
+
+      panel.webview.html = getHtmlForWebview(
+        panel.webview,
+        scriptPathOnDisk,
+        stylesheetPathOnDisk
+      );
+      panel.webview.onDidReceiveMessage(
+        async (e) => this.handleMsgFromView(e),
+        undefined,
+        ctx.subscriptions
+      );
+    }
+    return this.panel;
+  }
+
+  private waitForPanelLoaded(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.panelLoaded) {
+        resolve();
+      } else {
+        this.panelLoadedCallBacks.push(resolve);
+      }
+    });
+  }
+
+  private async handleMsgFromView(
+    msg: FromCompareViewMessage
+  ): Promise<void> {
+    switch (msg.t) {
+      case 'compareViewLoaded':
+        this.panelLoaded = true;
+        this.panelLoadedCallBacks.forEach((cb) => cb());
+        this.panelLoadedCallBacks = [];
+        break;
+
+      case 'changeCompare':
+        this.changeTable(msg.newResultSetName);
+        break;
+
+      case 'viewSourceFile':
+        await jumpToLocation(msg, this.databaseManager, this.logger);
+        break;
+
+      case 'openQuery':
+        await this.openQuery(msg.kind);
+        break;
+    }
+  }
+
+  private postMessage(msg: ToCompareViewMessage): Thenable<boolean> {
+    return this.getPanel().webview.postMessage(msg);
+  }
+
+  private async findCommonResultSetNames(
+    from: CompletedQuery,
+    to: CompletedQuery,
+    selectedResultSetName: string | undefined
+  ): Promise<[string[], string, RawResultSet, RawResultSet]> {
+    const fromSchemas = await this.cliServer.bqrsInfo(
+      from.query.resultsPaths.resultsPath
+    );
+    const toSchemas = await this.cliServer.bqrsInfo(
+      to.query.resultsPaths.resultsPath
+    );
+    const fromSchemaNames = fromSchemas['result-sets'].map(
+      (schema) => schema.name
+    );
+    const toSchemaNames = toSchemas['result-sets'].map(
+      (schema) => schema.name
+    );
+    const commonResultSetNames = fromSchemaNames.filter((name) =>
+      toSchemaNames.includes(name)
+    );
+    const currentResultSetName =
+      selectedResultSetName || commonResultSetNames[0];
+    const fromResultSet = await this.getResultSet(
+      fromSchemas,
+      currentResultSetName,
+      from.query.resultsPaths.resultsPath
+    );
+    const toResultSet = await this.getResultSet(
+      toSchemas,
+      currentResultSetName,
+      to.query.resultsPaths.resultsPath
+    );
+    return [
+      commonResultSetNames,
+      currentResultSetName,
+      fromResultSet,
+      toResultSet,
+    ];
+  }
+
+  private async changeTable(newResultSetName: string) {
+    if (!this.comparePair?.from || !this.comparePair.to) {
+      return;
+    }
+    await this.showResults(
+      this.comparePair.from,
+      this.comparePair.to,
+      newResultSetName
+    );
+  }
+
+  private async getResultSet(
+    bqrsInfo: BQRSInfo,
+    resultSetName: string,
+    resultsPath: string
+  ): Promise<RawResultSet> {
+    const schema = bqrsInfo['result-sets'].find(
+      (schema) => schema.name === resultSetName
+    );
+    if (!schema) {
+      throw new Error(`Schema ${resultSetName} not found.`);
+    }
+    const chunk = await this.cliServer.bqrsDecode(
+      resultsPath,
+      resultSetName
+    );
+    return transformBqrsResultSet(schema, chunk);
+  }
+
+  private compareResults(
+    fromResults: RawResultSet,
+    toResults: RawResultSet
+  ): QueryCompareResult {
+    // Only compare columns that have the same name
+    return resultsDiff(fromResults, toResults);
+  }
+
+  private openQuery(kind: 'from' | 'to') {
+    const toOpen =
+      kind === 'from' ? this.comparePair?.from : this.comparePair?.to;
+    if (toOpen) {
+      this.showQueryResultsCallback(toOpen);
+    }
+  }
+}

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -0,0 +1,58 @@
+import { RawResultSet } from '../pure/bqrs-cli-types';
+import { QueryCompareResult } from '../pure/interface-types';
+
+/**
+ * Compare the rows of two queries. Use deep equality to determine if
+ * rows have been added or removed across two invocations of a query.
+ *
+ * Assumptions:
+ *
+ * 1. Queries have the same sort order
+ * 2. Queries have same number and order of columns
+ * 3. Rows are not changed or re-ordered, they are only added or removed
+ *
+ * @param fromResults the source query
+ * @param toResults the target query
+ *
+ * @throws Error when:
+ *  1. number of columns do not match
+ *  2. If either query is empty
+ *  3. If the queries are 100% disjoint
+ */
+export default function resultsDiff(
+  fromResults: RawResultSet,
+  toResults: RawResultSet
+): QueryCompareResult {
+
+  if (fromResults.schema.columns.length !== toResults.schema.columns.length) {
+    throw new Error('CodeQL Compare: Columns do not match.');
+  }
+
+  if (!fromResults.rows.length) {
+    throw new Error('CodeQL Compare: Source query has no results.');
+  }
+
+  if (!toResults.rows.length) {
+    throw new Error('CodeQL Compare: Target query has no results.');
+  }
+
+  const results = {
+    from: arrayDiff(fromResults.rows, toResults.rows),
+    to: arrayDiff(toResults.rows, fromResults.rows),
+  };
+
+  if (
+    fromResults.rows.length === results.from.length &&
+    toResults.rows.length === results.to.length
+  ) {
+    throw new Error('CodeQL Compare: No overlap between the selected queries.');
+  }
+
+  return results;
+}
+
+function arrayDiff<T>(source: readonly T[], toRemove: readonly T[]): T[] {
+  // Stringify the object so that we can compare hashes in the set
+  const rest = new Set(toRemove.map((item) => JSON.stringify(item)));
+  return source.filter((element) => !rest.has(JSON.stringify(element)));
+}

extensions/ql-vscode/src/compare/view/.eslintrc.js

@@ -0,0 +1,13 @@
+module.exports = {
+  env: {
+    browser: true
+  },
+  extends: [
+    "plugin:react/recommended"
+  ],
+  settings: {
+    react: {
+      version: 'detect'
+    }
+  }
+}

extensions/ql-vscode/src/compare/view/Compare.tsx

@@ -0,0 +1,82 @@
+import * as React from 'react';
+import { useState, useEffect } from 'react';
+import * as Rdom from 'react-dom';
+
+import {
+  ToCompareViewMessage,
+  SetComparisonsMessage,
+} from '../../pure/interface-types';
+import CompareSelector from './CompareSelector';
+import { vscode } from '../../view/vscode-api';
+import CompareTable from './CompareTable';
+
+const emptyComparison: SetComparisonsMessage = {
+  t: 'setComparisons',
+  stats: {},
+  rows: undefined,
+  columns: [],
+  commonResultSetNames: [],
+  currentResultSetName: '',
+  datebaseUri: '',
+  message: 'Empty comparison'
+};
+
+export function Compare(_: {}): JSX.Element {
+  const [comparison, setComparison] = useState<SetComparisonsMessage>(
+    emptyComparison
+  );
+
+  const message = comparison.message || 'Empty comparison';
+  const hasRows = comparison.rows && (comparison.rows.to.length || comparison.rows.from.length);
+
+  useEffect(() => {
+    window.addEventListener('message', (evt: MessageEvent) => {
+      if (evt.origin === window.origin) {
+        const msg: ToCompareViewMessage = evt.data;
+        switch (msg.t) {
+          case 'setComparisons':
+            setComparison(msg);
+        }
+      } else {
+        console.error(`Invalid event origin ${evt.origin}`);
+      }
+    });
+  });
+  if (!comparison) {
+    return <div>Waiting for results to load.</div>;
+  }
+
+  try {
+    return (
+      <>
+        <div className="vscode-codeql__compare-header">
+          <div className="vscode-codeql__compare-header-item">
+            Table to compare:
+          </div>
+          <CompareSelector
+            availableResultSets={comparison.commonResultSetNames}
+            currentResultSetName={comparison.currentResultSetName}
+            updateResultSet={(newResultSetName: string) =>
+              vscode.postMessage({ t: 'changeCompare', newResultSetName })
+            }
+          />
+        </div>
+        {hasRows ? (
+          <CompareTable comparison={comparison}></CompareTable>
+        ) : (
+            <div className="vscode-codeql__compare-message">{message}</div>
+          )}
+      </>
+    );
+  } catch (err) {
+    console.error(err);
+    return <div>Error!</div>;
+  }
+}
+
+Rdom.render(
+  <Compare />,
+  document.getElementById('root'),
+  // Post a message to the extension when fully loaded.
+  () => vscode.postMessage({ t: 'compareViewLoaded' })
+);

extensions/ql-vscode/src/compare/view/CompareSelector.tsx

@@ -0,0 +1,22 @@
+import * as React from 'react';
+
+interface Props {
+  availableResultSets: string[];
+  currentResultSetName: string;
+  updateResultSet: (newResultSet: string) => void;
+}
+
+export default function CompareSelector(props: Props) {
+  return (
+    <select
+      value={props.currentResultSetName}
+      onChange={(e) => props.updateResultSet(e.target.value)}
+    >
+      {props.availableResultSets.map((resultSet) => (
+        <option key={resultSet} value={resultSet}>
+          {resultSet}
+        </option>
+      ))}
+    </select>
+  );
+}

extensions/ql-vscode/src/compare/view/CompareTable.tsx

@@ -0,0 +1,96 @@
+import * as React from 'react';
+
+import { SetComparisonsMessage } from '../../pure/interface-types';
+import RawTableHeader from '../../view/RawTableHeader';
+import { className } from '../../view/result-table-utils';
+import { ResultRow } from '../../pure/bqrs-cli-types';
+import RawTableRow from '../../view/RawTableRow';
+import { vscode } from '../../view/vscode-api';
+
+interface Props {
+  comparison: SetComparisonsMessage;
+}
+
+export default function CompareTable(props: Props) {
+  const comparison = props.comparison;
+  const rows = props.comparison.rows!;
+
+  async function openQuery(kind: 'from' | 'to') {
+    vscode.postMessage({
+      t: 'openQuery',
+      kind,
+    });
+  }
+
+  function createRows(rows: ResultRow[], databaseUri: string) {
+    return (
+      <tbody>
+        {rows.map((row, rowIndex) => (
+          <RawTableRow
+            key={rowIndex}
+            rowIndex={rowIndex}
+            row={row}
+            databaseUri={databaseUri}
+          />
+        ))}
+      </tbody>
+    );
+  }
+
+  return (
+    <table className='vscode-codeql__compare-body'>
+      <thead>
+        <tr>
+          <td>
+            <a
+              onClick={() => openQuery('from')}
+              className='vscode-codeql__compare-open'
+            >
+              {comparison.stats.fromQuery?.name}
+            </a>
+          </td>
+          <td>
+            <a
+              onClick={() => openQuery('to')}
+              className='vscode-codeql__compare-open'
+            >
+              {comparison.stats.toQuery?.name}
+            </a>
+          </td>
+        </tr>
+        <tr>
+          <td>{comparison.stats.fromQuery?.time}</td>
+          <td>{comparison.stats.toQuery?.time}</td>
+        </tr>
+        <tr>
+          <th>{rows.from.length} rows removed</th>
+          <th>{rows.to.length} rows added</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>
+            <table className={className}>
+              <RawTableHeader
+                columns={comparison.columns}
+                schemaName={comparison.currentResultSetName}
+                preventSort={true}
+              />
+              {createRows(rows.from, comparison.datebaseUri)}
+            </table>
+          </td>
+          <td>
+            <table className={className}>
+              <RawTableHeader
+                columns={comparison.columns}
+                schemaName={comparison.currentResultSetName}
+                preventSort={true}
+              />
+              {createRows(rows.to, comparison.datebaseUri)}
+            </table>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  );
+}

extensions/ql-vscode/src/compare/view/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "module": "esnext",
+    "moduleResolution": "node",
+    "target": "es6",
+    "outDir": "out",
+    "lib": [
+      "es6",
+      "dom"
+    ],
+    "jsx": "react",
+    "sourceMap": true,
+    "rootDir": "..",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "experimentalDecorators": true
+  },
+  "exclude": [
+    "node_modules"
+  ]
+}

extensions/ql-vscode/src/config.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from 'semmle-vscode-utils';
+import { DisposableObject } from './vscode-utils/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
@@ -39,18 +39,6 @@ class Setting {
 
 const ROOT_SETTING = new Setting('codeQL');
 
-// Enable experimental features
-
-/**
- * Any settings below are deliberately not in package.json so that
- * they do not appear in the settings ui in vscode itself. If users
- * want to enable experimental features, they can add them directly in
- * their vscode settings json file.
- */
-
-/* Advanced setting: used to enable bqrs parsing in the cli instead of in the webview. */
-export const EXPERIMENTAL_BQRS_SETTING = new Setting('experimentalBqrsParsing', ROOT_SETTING);
-
 // Distribution configuration
 
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
@@ -64,12 +52,13 @@ const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
 
 export interface DistributionConfig {
-  customCodeQlPath?: string;
+  readonly customCodeQlPath?: string;
+  updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
   ownerName?: string;
   repositoryName?: string;
-  onDidChangeDistributionConfiguration?: Event<void>;
+  onDidChangeConfiguration?: Event<void>;
 }
 
 // Query server configuration
@@ -79,7 +68,13 @@ const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
+const RUNNING_TESTS_SETTING = new Setting('runningTests', ROOT_SETTING);
+const RESULTS_DISPLAY_SETTING = new Setting('resultsDisplay', ROOT_SETTING);
+
+export const NUMBER_OF_TEST_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_TESTS_SETTING);
+export const MAX_QUERIES = new Setting('maxQueries', RUNNING_QUERIES_SETTING);
 export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
+export const PAGE_SIZE = new Setting('pageSize', RESULTS_DISPLAY_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
 const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
@@ -90,7 +85,7 @@ export interface QueryServerConfig {
   numThreads: number;
   queryMemoryMb?: number;
   timeoutSecs: number;
-  onDidChangeQueryServerConfiguration?: Event<void>;
+  onDidChangeConfiguration?: Event<void>;
 }
 
 /** When these settings change, the query history should be refreshed. */
@@ -98,9 +93,17 @@ const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING];
 
 export interface QueryHistoryConfig {
   format: string;
-  onDidChangeQueryHistoryConfiguration: Event<void>;
+  onDidChangeConfiguration: Event<void>;
 }
 
+const CLI_SETTINGS = [NUMBER_OF_TEST_THREADS_SETTING];
+
+export interface CliConfig {
+  numberTestThreads: number;
+  onDidChangeConfiguration?: Event<void>;
+}
+
+
 abstract class ConfigListener extends DisposableObject {
   protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());
 
@@ -128,6 +131,10 @@ abstract class ConfigListener extends DisposableObject {
   private updateConfiguration(): void {
     this._onDidChangeConfiguration.fire();
   }
+
+  public get onDidChangeConfiguration(): Event<void> {
+    return this._onDidChangeConfiguration.event;
+  }
 }
 
 export class DistributionConfigListener extends ConfigListener implements DistributionConfig {
@@ -143,8 +150,8 @@ export class DistributionConfigListener extends ConfigListener implements Distri
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
-  public get onDidChangeDistributionConfiguration(): Event<void> {
-    return this._onDidChangeConfiguration.event;
+  public async updateCustomCodeQlPath(newPath: string | undefined) {
+    await CUSTOM_CODEQL_PATH_SETTING.updateValue(newPath, ConfigurationTarget.Global);
   }
 
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
@@ -153,7 +160,7 @@ export class DistributionConfigListener extends ConfigListener implements Distri
 }
 
 export class QueryServerConfigListener extends ConfigListener implements QueryServerConfig {
-  private constructor(private _codeQlPath: string) {
+  public constructor(private _codeQlPath = '') {
     super();
   }
 
@@ -199,10 +206,6 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     return DEBUG_SETTING.getValue<boolean>();
   }
 
-  public get onDidChangeQueryServerConfiguration(): Event<void> {
-    return this._onDidChangeConfiguration.event;
-  }
-
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(QUERY_SERVER_RESTARTING_SETTINGS, e);
   }
@@ -213,11 +216,27 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
     this.handleDidChangeConfigurationForRelevantSettings(QUERY_HISTORY_SETTINGS, e);
   }
 
-  public get onDidChangeQueryHistoryConfiguration(): Event<void> {
-    return this._onDidChangeConfiguration.event;
-  }
-
   public get format(): string {
     return QUERY_HISTORY_FORMAT_SETTING.getValue<string>();
   }
 }
+
+export class CliConfigListener extends ConfigListener implements CliConfig {
+
+  public get numberTestThreads(): number {
+    return NUMBER_OF_TEST_THREADS_SETTING.getValue();
+  }
+
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
+  }
+}
+
+// Enable experimental features
+
+/**
+ * Any settings below are deliberately not in package.json so that
+ * they do not appear in the settings ui in vscode itself. If users
+ * want to enable experimental features, they can add them directly in
+ * their vscode settings json file.
+ */

extensions/ql-vscode/src/contextual/astBuilder.ts

@@ -0,0 +1,147 @@
+import { QueryWithResults } from '../run-queries';
+import { CodeQLCliServer } from '../cli';
+import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
+import { DatabaseItem } from '../databases';
+import { ChildAstItem, AstItem } from '../astViewer';
+import fileRangeFromURI from './fileRangeFromURI';
+
+/**
+ * A class that wraps a tree of QL results from a query that
+ * has an @kind of graph
+ */
+export default class AstBuilder {
+
+  private roots: AstItem[] | undefined;
+  private bqrsPath: string;
+  constructor(
+    queryResults: QueryWithResults,
+    private cli: CodeQLCliServer,
+    public db: DatabaseItem,
+    public fileName: string
+  ) {
+    this.bqrsPath = queryResults.query.resultsPaths.resultsPath;
+  }
+
+  async getRoots(): Promise<AstItem[]> {
+    if (!this.roots) {
+      this.roots = await this.parseRoots();
+    }
+    return this.roots;
+  }
+
+  private async parseRoots(): Promise<AstItem[]> {
+    const options = { entities: ['id', 'url', 'string'] };
+    const [nodeTuples, edgeTuples, graphProperties] = await Promise.all([
+      await this.cli.bqrsDecode(this.bqrsPath, 'nodes', options),
+      await this.cli.bqrsDecode(this.bqrsPath, 'edges', options),
+      await this.cli.bqrsDecode(this.bqrsPath, 'graphProperties', options),
+    ]);
+
+    if (!this.isValidGraph(graphProperties)) {
+      throw new Error('AST is invalid');
+    }
+
+    const idToItem = new Map<BqrsId, AstItem>();
+    const parentToChildren = new Map<BqrsId, BqrsId[]>();
+    const childToParent = new Map<BqrsId, BqrsId>();
+    const astOrder = new Map<BqrsId, number>();
+    const edgeLabels = new Map<BqrsId, string>();
+    const roots = [];
+
+    // Build up the parent-child relationships
+    edgeTuples.tuples.forEach(tuple => {
+      const [source, target, tupleType, value] = tuple as [EntityValue, EntityValue, string, string];
+      const sourceId = source.id!;
+      const targetId = target.id!;
+
+      switch (tupleType) {
+        case 'semmle.order':
+          astOrder.set(targetId, Number(value));
+          break;
+
+        case 'semmle.label': {
+          childToParent.set(targetId, sourceId);
+          let children = parentToChildren.get(sourceId);
+          if (!children) {
+            parentToChildren.set(sourceId, children = []);
+          }
+          children.push(targetId);
+
+          // ignore values that indicate a numeric order.
+          if (!Number.isFinite(Number(value))) {
+            edgeLabels.set(targetId, value);
+          }
+          break;
+        }
+
+        default:
+        // ignore other tupleTypes since they are not needed by the ast viewer
+      }
+    });
+
+    // populate parents and children
+    nodeTuples.tuples.forEach(tuple => {
+      const [entity, tupleType, value] = tuple as [EntityValue, string, string];
+      const id = entity.id!;
+
+      switch (tupleType) {
+        case 'semmle.order':
+          astOrder.set(id, Number(value));
+          break;
+
+        case 'semmle.label': {
+          // If an edge label exists, include it and separate from the node label using ':'
+          const nodeLabel = value ?? entity.label;
+          const edgeLabel = edgeLabels.get(id);
+          const label = [edgeLabel, nodeLabel].filter(e => e).join(': ');
+          const item = {
+            id,
+            label,
+            location: entity.url,
+            fileLocation: fileRangeFromURI(entity.url, this.db),
+            children: [] as ChildAstItem[],
+            order: Number.MAX_SAFE_INTEGER
+          };
+
+          idToItem.set(id, item);
+          const parent = idToItem.get(childToParent.has(id) ? childToParent.get(id)! : -1);
+
+          if (parent) {
+            const astItem = item as ChildAstItem;
+            astItem.parent = parent;
+            parent.children.push(astItem);
+          }
+          const children = parentToChildren.has(id) ? parentToChildren.get(id)! : [];
+          children.forEach(childId => {
+            const child = idToItem.get(childId) as ChildAstItem | undefined;
+            if (child) {
+              child.parent = item;
+              item.children.push(child);
+            }
+          });
+          break;
+        }
+
+        default:
+        // ignore other tupleTypes since they are not needed by the ast viewer
+      }
+    });
+
+    // find the roots and add the order
+    for (const [, item] of idToItem) {
+      item.order = astOrder.has(item.id)
+        ? astOrder.get(item.id)!
+        : Number.MAX_SAFE_INTEGER;
+
+      if (!('parent' in item)) {
+        roots.push(item);
+      }
+    }
+    return roots;
+  }
+
+  private isValidGraph(graphProperties: DecodedBqrsChunk) {
+    const tuple = graphProperties?.tuples?.find(t => t[0] === 'semmle.graphKind');
+    return tuple?.[1] === 'tree';
+  }
+}

extensions/ql-vscode/src/contextual/fileRangeFromURI.ts

@@ -0,0 +1,31 @@
+import * as vscode from 'vscode';
+
+import { UrlValue, LineColumnLocation } from '../pure/bqrs-cli-types';
+import { isEmptyPath } from '../pure/bqrs-utils';
+import { DatabaseItem } from '../databases';
+
+
+export default function fileRangeFromURI(uri: UrlValue | undefined, db: DatabaseItem): vscode.Location | undefined {
+  if (!uri || typeof uri === 'string') {
+    return undefined;
+  } else if ('startOffset' in uri) {
+    return undefined;
+  } else {
+    const loc = uri as LineColumnLocation;
+    if (isEmptyPath(loc.uri)) {
+      return undefined;
+    }
+    const range = new vscode.Range(Math.max(0, (loc.startLine || 0) - 1),
+      Math.max(0, (loc.startColumn || 0) - 1),
+      Math.max(0, (loc.endLine || 0) - 1),
+      Math.max(0, (loc.endColumn || 0)));
+    try {
+      if (uri.uri.startsWith('file:')) {
+        return new vscode.Location(db.resolveSourceFile(uri.uri), range);
+      }
+      return undefined;
+    } catch (e) {
+      return undefined;
+    }
+  }
+}

extensions/ql-vscode/src/contextual/keyType.ts

@@ -0,0 +1,37 @@
+export enum KeyType {
+  DefinitionQuery = 'DefinitionQuery',
+  ReferenceQuery = 'ReferenceQuery',
+  PrintAstQuery = 'PrintAstQuery',
+}
+
+export function tagOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery:
+      return 'ide-contextual-queries/local-definitions';
+    case KeyType.ReferenceQuery:
+      return 'ide-contextual-queries/local-references';
+    case KeyType.PrintAstQuery:
+      return 'ide-contextual-queries/print-ast';
+  }
+}
+
+export function nameOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery:
+      return 'definitions';
+    case KeyType.ReferenceQuery:
+      return 'references';
+    case KeyType.PrintAstQuery:
+      return 'print AST';
+  }
+}
+
+export function kindOfKeyType(keyType: KeyType): string {
+  switch (keyType) {
+    case KeyType.DefinitionQuery:
+    case KeyType.ReferenceQuery:
+      return 'definitions';
+    case KeyType.PrintAstQuery:
+      return 'graph';
+  }
+}

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -0,0 +1,124 @@
+import * as vscode from 'vscode';
+
+import { decodeSourceArchiveUri, encodeArchiveBasePath } from '../archive-filesystem-provider';
+import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from '../pure/bqrs-cli-types';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseManager, DatabaseItem } from '../databases';
+import fileRangeFromURI from './fileRangeFromURI';
+import * as messages from '../pure/messages';
+import { QueryServerClient } from '../queryserver-client';
+import { QueryWithResults, compileAndRunQueryAgainstDatabase } from '../run-queries';
+import { ProgressCallback } from '../helpers';
+import { KeyType } from './keyType';
+import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+
+const SELECT_QUERY_NAME = '#select';
+export const TEMPLATE_NAME = 'selectedSourceFile';
+
+export interface FullLocationLink extends vscode.LocationLink {
+  originUri: vscode.Uri;
+}
+
+/**
+ * This function executes a contextual query inside a given database, filters, and converts
+ * the results into source locations. This function is the workhorse for all search-based
+ * contextual queries like find references and find definitions.
+ *
+ * @param cli The cli server
+ * @param qs The query server client
+ * @param dbm The database manager
+ * @param uriString The selected source file and location
+ * @param keyType The contextual query type to run
+ * @param progress A progress callback
+ * @param token A CancellationToken
+ * @param filter A function that will filter extraneous results
+ */
+export async function getLocationsForUriString(
+  cli: CodeQLCliServer,
+  qs: QueryServerClient,
+  dbm: DatabaseManager,
+  uriString: string,
+  keyType: KeyType,
+  progress: ProgressCallback,
+  token: vscode.CancellationToken,
+  filter: (src: string, dest: string) => boolean
+): Promise<FullLocationLink[]> {
+  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString, true));
+  const sourceArchiveUri = encodeArchiveBasePath(uri.sourceArchiveZipPath);
+
+  const db = dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+  if (!db) {
+    return [];
+  }
+
+  const qlpack = await qlpackOfDatabase(cli, db);
+  const templates = createTemplates(uri.pathWithinSourceArchive);
+
+  const links: FullLocationLink[] = [];
+  for (const query of await resolveQueries(cli, qlpack, keyType)) {
+    const results = await compileAndRunQueryAgainstDatabase(
+      cli,
+      qs,
+      db,
+      false,
+      vscode.Uri.file(query),
+      progress,
+      token,
+      templates
+    );
+
+    if (results.result.resultType == messages.QueryResultType.SUCCESS) {
+      links.push(...await getLinksFromResults(results, cli, db, filter));
+    }
+  }
+  return links;
+}
+
+async function getLinksFromResults(
+  results: QueryWithResults,
+  cli: CodeQLCliServer,
+  db: DatabaseItem,
+  filter: (srcFile: string, destFile: string) => boolean
+): Promise<FullLocationLink[]> {
+  const localLinks: FullLocationLink[] = [];
+  const bqrsPath = results.query.resultsPaths.resultsPath;
+  const info = await cli.bqrsInfo(bqrsPath);
+  const selectInfo = getResultSetSchema(SELECT_QUERY_NAME, info);
+  if (isValidSelect(selectInfo)) {
+    // TODO: Page this
+    const allTuples = await cli.bqrsDecode(bqrsPath, SELECT_QUERY_NAME);
+    for (const tuple of allTuples.tuples) {
+      const [src, dest] = tuple as [EntityValue, EntityValue];
+      const srcFile = src.url && fileRangeFromURI(src.url, db);
+      const destFile = dest.url && fileRangeFromURI(dest.url, db);
+      if (srcFile && destFile && filter(srcFile.uri.toString(), destFile.uri.toString())) {
+        localLinks.push({
+          targetRange: destFile.range,
+          targetUri: destFile.uri,
+          originSelectionRange: srcFile.range,
+          originUri: srcFile.uri
+        });
+      }
+    }
+  }
+  return localLinks;
+}
+
+function createTemplates(path: string): messages.TemplateDefinitions {
+  return {
+    [TEMPLATE_NAME]: {
+      values: {
+        tuples: [[{
+          stringValue: path
+        }]]
+      }
+    }
+  };
+}
+
+function isValidSelect(selectInfo: ResultSetSchema | undefined) {
+  return selectInfo && selectInfo.columns.length == 3
+    && selectInfo.columns[0].kind == ColumnKindCode.ENTITY
+    && selectInfo.columns[1].kind == ColumnKindCode.ENTITY
+    && selectInfo.columns[2].kind == ColumnKindCode.STRING;
+}

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -0,0 +1,48 @@
+import * as fs from 'fs-extra';
+import * as yaml from 'js-yaml';
+import * as tmp from 'tmp-promise';
+
+import * as helpers from '../helpers';
+import {
+  KeyType,
+  kindOfKeyType,
+  nameOfKeyType,
+  tagOfKeyType
+} from './keyType';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseItem } from '../databases';
+
+export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string> {
+  if (db.contents === undefined) {
+    throw new Error('Database is invalid and cannot infer QLPack.');
+  }
+  const datasetPath = db.contents.datasetUri.fsPath;
+  const dbscheme = await helpers.getPrimaryDbscheme(datasetPath);
+  return await helpers.getQlPackForDbscheme(cli, dbscheme);
+}
+
+
+export async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
+  const suiteFile = (await tmp.file({
+    postfix: '.qls'
+  })).path;
+  const suiteYaml = {
+    qlpack,
+    include: {
+      kind: kindOfKeyType(keyType),
+      'tags contain': tagOfKeyType(keyType)
+    }
+  };
+  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
+
+  const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
+  if (queries.length === 0) {
+    helpers.showAndLogErrorMessage(
+      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
+      Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
+      for this language.`
+    );
+    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
+  }
+  return queries;
+}

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -0,0 +1,189 @@
+import * as vscode from 'vscode';
+
+import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
+import { CodeQLCliServer } from '../cli';
+import { DatabaseManager } from '../databases';
+import { CachedOperation, ProgressCallback, withProgress } from '../helpers';
+import * as messages from '../pure/messages';
+import { QueryServerClient } from '../queryserver-client';
+import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';
+import AstBuilder from './astBuilder';
+import {
+  KeyType,
+} from './keyType';
+import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
+import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+
+/**
+ * Run templated CodeQL queries to find definitions and references in
+ * source-language files. We may eventually want to find a way to
+ * generalize this to other custom queries, e.g. showing dataflow to
+ * or from a selected identifier.
+ */
+
+export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
+  private cache: CachedOperation<vscode.LocationLink[]>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+  }
+
+  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+    const fileLinks = await this.cache.get(document.uri.toString());
+    const locLinks: vscode.LocationLink[] = [];
+    for (const link of fileLinks) {
+      if (link.originSelectionRange!.contains(position)) {
+        locLinks.push(link);
+      }
+    }
+    return locLinks;
+  }
+
+  private async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+    return withProgress({
+      location: vscode.ProgressLocation.Notification,
+      cancellable: true,
+      title: 'Finding definitions'
+    }, async (progress, token) => {
+      return getLocationsForUriString(
+        this.cli,
+        this.qs,
+        this.dbm,
+        uriString,
+        KeyType.DefinitionQuery,
+        progress,
+        token,
+        (src, _dest) => src === uriString
+      );
+    });
+  }
+}
+
+export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+  private cache: CachedOperation<FullLocationLink[]>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<FullLocationLink[]>(this.getReferences.bind(this));
+  }
+
+  async provideReferences(
+    document: vscode.TextDocument,
+    position: vscode.Position,
+    _context: vscode.ReferenceContext,
+    _token: vscode.CancellationToken
+  ): Promise<vscode.Location[]> {
+    const fileLinks = await this.cache.get(document.uri.toString());
+    const locLinks: vscode.Location[] = [];
+    for (const link of fileLinks) {
+      if (link.targetRange!.contains(position)) {
+        locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
+      }
+    }
+    return locLinks;
+  }
+
+  private async getReferences(uriString: string): Promise<FullLocationLink[]> {
+    return withProgress({
+      location: vscode.ProgressLocation.Notification,
+      cancellable: true,
+      title: 'Finding references'
+    }, async (progress, token) => {
+      return getLocationsForUriString(
+        this.cli,
+        this.qs,
+        this.dbm,
+        uriString,
+        KeyType.DefinitionQuery,
+        progress,
+        token,
+        (src, _dest) => src === uriString
+      );
+    });
+  }
+}
+
+export class TemplatePrintAstProvider {
+  private cache: CachedOperation<QueryWithResults | undefined>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private qs: QueryServerClient,
+    private dbm: DatabaseManager,
+
+    // Note: progress and token are only used if a cached value is not available
+    private progress: ProgressCallback,
+    private token: vscode.CancellationToken
+  ) {
+    this.cache = new CachedOperation<QueryWithResults | undefined>(this.getAst.bind(this));
+  }
+
+  async provideAst(document?: vscode.TextDocument): Promise<AstBuilder | undefined> {
+    if (!document) {
+      return;
+    }
+    const queryResults = await this.cache.get(document.uri.toString());
+    if (!queryResults) {
+      return;
+    }
+
+    return new AstBuilder(
+      queryResults, this.cli,
+      this.dbm.findDatabaseItem(vscode.Uri.parse(queryResults.database.databaseUri!, true))!,
+      document.fileName
+    );
+  }
+
+  private async getAst(uriString: string): Promise<QueryWithResults> {
+    const uri = vscode.Uri.parse(uriString, true);
+    if (uri.scheme !== zipArchiveScheme) {
+      throw new Error('AST Viewing is only available for databases with zipped source archives.');
+    }
+
+    const zippedArchive = decodeSourceArchiveUri(uri);
+    const sourceArchiveUri = encodeArchiveBasePath(zippedArchive.sourceArchiveZipPath);
+    const db = this.dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+
+    if (!db) {
+      throw new Error('Can\'t infer database from the provided source.');
+    }
+
+    const qlpack = await qlpackOfDatabase(this.cli, db);
+    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintAstQuery);
+    if (queries.length > 1) {
+      throw new Error('Found multiple Print AST queries. Can\'t continue');
+    }
+    if (queries.length === 0) {
+      throw new Error('Did not find any Print AST queries. Can\'t continue');
+    }
+
+    const query = queries[0];
+    const templates: messages.TemplateDefinitions = {
+      [TEMPLATE_NAME]: {
+        values: {
+          tuples: [[{
+            stringValue: zippedArchive.pathWithinSourceArchive
+          }]]
+        }
+      }
+    };
+
+    return await compileAndRunQueryAgainstDatabase(
+      this.cli,
+      this.qs,
+      db,
+      false,
+      vscode.Uri.file(query),
+      this.progress,
+      this.token,
+      templates
+    );
+  }
+}

extensions/ql-vscode/src/databaseFetcher.ts

@@ -1,67 +1,59 @@
-import fetch, { Response } from "node-fetch";
-import * as unzipper from "unzipper";
+import fetch, { Response } from 'node-fetch';
+import * as unzipper from 'unzipper';
+import { zip } from 'zip-a-folder';
 import {
   Uri,
-  ProgressOptions,
-  ProgressLocation,
+  CancellationToken,
   commands,
   window,
-} from "vscode";
-import * as fs from "fs-extra";
-import * as path from "path";
-import { DatabaseManager, DatabaseItem } from "./databases";
+} from 'vscode';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { DatabaseManager, DatabaseItem } from './databases';
 import {
+  reportStreamProgress,
   ProgressCallback,
-  showAndLogErrorMessage,
-  withProgress,
   showAndLogInformationMessage,
-} from "./helpers";
-import { logger } from "./logging";
+} from './helpers';
+import { logger } from './logging';
+import { tmpDir } from './run-queries';
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
  *
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportInternetDatabase(
-  databasesManager: DatabaseManager,
-  storagePath: string
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  progress: ProgressCallback,
+  token: CancellationToken,
 ): Promise<DatabaseItem | undefined> {
-  let item: DatabaseItem | undefined = undefined;
-
-  try {
-    const databaseUrl = await window.showInputBox({
-      prompt: "Enter URL of zipfile of database to download",
-    });
-    if (databaseUrl) {
-      validateHttpsUrl(databaseUrl);
-
-      const progressOptions: ProgressOptions = {
-        location: ProgressLocation.Notification,
-        title: "Adding database from URL",
-        cancellable: false,
-      };
-      await withProgress(
-        progressOptions,
-        async (progress) =>
-          (item = await databaseArchiveFetcher(
-            databaseUrl,
-            databasesManager,
-            storagePath,
-            progress
-          ))
-      );
-      commands.executeCommand("codeQLDatabases.focus");
-    }
-    showAndLogInformationMessage(
-      "Database downloaded and imported successfully."
-    );
-  } catch (e) {
-    showAndLogErrorMessage(e.message);
+  const databaseUrl = await window.showInputBox({
+    prompt: 'Enter URL of zipfile of database to download',
+  });
+  if (!databaseUrl) {
+    return;
   }
 
+  validateHttpsUrl(databaseUrl);
+
+  const item = await databaseArchiveFetcher(
+    databaseUrl,
+    databaseManager,
+    storagePath,
+    progress,
+    token
+  );
+
+  if (item) {
+    commands.executeCommand('codeQLDatabases.focus');
+    showAndLogInformationMessage('Database downloaded and imported successfully.');
+  }
   return item;
+
 }
 
 /**
@@ -69,98 +61,80 @@ export async function promptImportInternetDatabase(
  * User enters a project url and then the user is asked which language
  * to download (if there is more than one)
  *
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportLgtmDatabase(
-  databasesManager: DatabaseManager,
-  storagePath: string
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  progress: ProgressCallback,
+  token: CancellationToken
 ): Promise<DatabaseItem | undefined> {
-  let item: DatabaseItem | undefined = undefined;
-
-  try {
-    const lgtmUrl = await window.showInputBox({
-      prompt:
-        "Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)",
-    });
-    if (!lgtmUrl) {
-      return;
-    }
-    if (looksLikeLgtmUrl(lgtmUrl)) {
-      const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
-      if (databaseUrl) {
-        const progressOptions: ProgressOptions = {
-          location: ProgressLocation.Notification,
-          title: "Adding database from LGTM",
-          cancellable: false,
-        };
-        await withProgress(
-          progressOptions,
-          async (progress) =>
-            (item = await databaseArchiveFetcher(
-              databaseUrl,
-              databasesManager,
-              storagePath,
-              progress
-            ))
-        );
-        commands.executeCommand("codeQLDatabases.focus");
-      }
-    } else {
-      throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
-    }
-    if (item) {
-      showAndLogInformationMessage(
-        "Database downloaded and imported successfully."
-      );
-    }
-  } catch (e) {
-    showAndLogErrorMessage(e.message);
+  const lgtmUrl = await window.showInputBox({
+    prompt:
+      'Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)',
+  });
+  if (!lgtmUrl) {
+    return;
   }
 
-  return item;
+  if (looksLikeLgtmUrl(lgtmUrl)) {
+    const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
+    if (databaseUrl) {
+      const item = await databaseArchiveFetcher(
+        databaseUrl,
+        databaseManager,
+        storagePath,
+        progress,
+        token
+      );
+      if (item) {
+        commands.executeCommand('codeQLDatabases.focus');
+        showAndLogInformationMessage('Database downloaded and imported successfully.');
+      }
+      return item;
+    }
+  } else {
+    throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
+  }
+  return;
 }
 
 /**
  * Imports a database from a local archive.
  *
  * @param databaseUrl the file url of the archive to import
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function importArchiveDatabase(
   databaseUrl: string,
-  databasesManager: DatabaseManager,
-  storagePath: string
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  progress: ProgressCallback,
+  token: CancellationToken,
 ): Promise<DatabaseItem | undefined> {
-  let item: DatabaseItem | undefined = undefined;
   try {
-    const progressOptions: ProgressOptions = {
-      location: ProgressLocation.Notification,
-      title: "Importing database from archive",
-      cancellable: false,
-    };
-    await withProgress(
-      progressOptions,
-      async (progress) =>
-        (item = await databaseArchiveFetcher(
-          databaseUrl,
-          databasesManager,
-          storagePath,
-          progress
-        ))
+    const item = await databaseArchiveFetcher(
+      databaseUrl,
+      databaseManager,
+      storagePath,
+      progress,
+      token
     );
-    commands.executeCommand("codeQLDatabases.focus");
-
     if (item) {
-      showAndLogInformationMessage(
-        "Database unzipped and imported successfully."
-      );
+      commands.executeCommand('codeQLDatabases.focus');
+      showAndLogInformationMessage('Database unzipped and imported successfully.');
     }
+    return item;
   } catch (e) {
-    showAndLogErrorMessage(e.message);
+    if (e.message.includes('unexpected end of file')) {
+      throw new Error('Database is corrupt or too large. Try unzipping outside of VS Code and importing the unzipped folder instead.');
+    } else {
+      // delegate
+      throw e;
+    }
   }
-  return item;
 }
 
 /**
@@ -168,51 +142,60 @@ export async function importArchiveDatabase(
  * or in the local filesystem.
  *
  * @param databaseUrl URL from which to grab the database
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
- * @param progressCallback optional callback to send progress messages to
+ * @param progress callback to send progress messages to
+ * @param token cancellation token
  */
 async function databaseArchiveFetcher(
   databaseUrl: string,
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
-  progressCallback?: ProgressCallback
+  progress: ProgressCallback,
+  token: CancellationToken
 ): Promise<DatabaseItem> {
-  progressCallback?.({
-    maxStep: 3,
-    message: "Getting database",
+  progress({
+    message: 'Getting database',
     step: 1,
+    maxStep: 4,
   });
   if (!storagePath) {
-    throw new Error("No storage path specified.");
+    throw new Error('No storage path specified.');
   }
   await fs.ensureDir(storagePath);
   const unzipPath = await getStorageFolder(storagePath, databaseUrl);
 
   if (isFile(databaseUrl)) {
-    await readAndUnzip(databaseUrl, unzipPath);
+    await readAndUnzip(databaseUrl, unzipPath, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, progressCallback);
+    await fetchAndUnzip(databaseUrl, unzipPath, progress);
   }
 
-  progressCallback?.({
-    maxStep: 3,
-    message: "Opening database",
+  progress({
+    message: 'Opening database',
     step: 3,
+    maxStep: 4,
   });
 
   // find the path to the database. The actual database might be in a sub-folder
   const dbPath = await findDirWithFile(
     unzipPath,
-    ".dbinfo",
-    "codeql-database.yml"
+    '.dbinfo',
+    'codeql-database.yml'
   );
   if (dbPath) {
-    const item = await databasesManager.openDatabase(Uri.file(dbPath));
-    databasesManager.setCurrentDatabaseItem(item);
+    progress({
+      message: 'Validating and fixing source location',
+      step: 4,
+      maxStep: 4,
+    });
+    await ensureZippedSourceLocation(dbPath);
+
+    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath));
+    await databaseManager.setCurrentDatabaseItem(item);
     return item;
   } else {
-    throw new Error("Database not found in archive.");
+    throw new Error('Database not found in archive.');
   }
 }
 
@@ -224,7 +207,7 @@ async function getStorageFolder(storagePath: string, urlStr: string) {
   // MacOS has a max filename length of 255
   // and remove a few extra chars in case we need to add a counter at the end.
   let lastName = path.basename(url.path).substring(0, 250);
-  if (lastName.endsWith(".zip")) {
+  if (lastName.endsWith('.zip')) {
     lastName = lastName.substring(0, lastName.length - 4);
   }
 
@@ -237,7 +220,7 @@ async function getStorageFolder(storagePath: string, urlStr: string) {
     counter++;
     folderName = path.join(realpath, `${lastName}-${counter}`);
     if (counter > 100) {
-      throw new Error("Could not find a unique name for downloaded database.");
+      throw new Error('Could not find a unique name for downloaded database.');
     }
   }
   return folderName;
@@ -251,62 +234,72 @@ function validateHttpsUrl(databaseUrl: string) {
     throw new Error(`Invalid url: ${databaseUrl}`);
   }
 
-  if (uri.scheme !== "https") {
-    throw new Error("Must use https for downloading a database.");
+  if (uri.scheme !== 'https') {
+    throw new Error('Must use https for downloading a database.');
   }
 }
 
-async function readAndUnzip(databaseUrl: string, unzipPath: string) {
-  const unzipStream = unzipper.Extract({
-    path: unzipPath,
-  });
-
-  await new Promise((resolve, reject) => {
-    // we already know this is a file scheme
-    const databaseFile = Uri.parse(databaseUrl).fsPath;
-    const stream = fs.createReadStream(databaseFile);
-    stream.on("error", reject);
-    unzipStream.on("error", reject);
-    unzipStream.on("close", resolve);
-    stream.pipe(unzipStream);
+async function readAndUnzip(
+  zipUrl: string,
+  unzipPath: string,
+  progress?: ProgressCallback
+) {
+  // TODO: Providing progress as the file is unzipped is currently blocked
+  // on https://github.com/ZJONSSON/node-unzipper/issues/222
+  const zipFile = Uri.parse(zipUrl).fsPath;
+  progress?.({
+    maxStep: 10,
+    step: 9,
+    message: `Unzipping into ${path.basename(unzipPath)}`
   });
+  // Must get the zip central directory since streaming the
+  // zip contents may not have correct local file headers.
+  // Instead, we can only rely on the central directory.
+  const directory = await unzipper.Open.file(zipFile);
+  await directory.extract({ path: unzipPath });
 }
 
 async function fetchAndUnzip(
   databaseUrl: string,
   unzipPath: string,
-  progressCallback?: ProgressCallback
+  progress?: ProgressCallback
 ) {
-  const response = await fetch(databaseUrl);
+  // Although it is possible to download and stream directly to an unzipped directory,
+  // we need to avoid this for two reasons. The central directory is located at the
+  // end of the zip file. It is the source of truth of the content locations. Individual
+  // file headers may be incorrect. Additionally, saving to file first will reduce memory
+  // pressure compared with unzipping while downloading the archive.
 
-  await checkForFailingResponse(response);
+  const archivePath = path.join(tmpDir.name, `archive-${Date.now()}.zip`);
 
-  const unzipStream = unzipper.Extract({
-    path: unzipPath,
-  });
-  progressCallback?.({
+  progress?.({
     maxStep: 3,
-    message: "Unzipping database",
-    step: 2,
-  });
-  await new Promise((resolve, reject) => {
-    const handler = (err: Error) => {
-      if (err.message.startsWith('invalid signature')) {
-        reject(new Error('Not a valid archive.'));
-      } else {
-        reject(err);
-      }
-    };
-    response.body.on("error", handler);
-    unzipStream.on("error", handler);
-    unzipStream.on("close", resolve);
-    response.body.pipe(unzipStream);
+    message: 'Downloading database',
+    step: 1,
   });
+
+  const response = await checkForFailingResponse(await fetch(databaseUrl));
+  const archiveFileStream = fs.createWriteStream(archivePath);
+
+  const contentLength = response.headers.get('content-length');
+  const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
+  reportStreamProgress(response.body, 'Downloading database', totalNumBytes, progress);
+
+  await new Promise((resolve, reject) =>
+    response.body.pipe(archiveFileStream)
+      .on('finish', resolve)
+      .on('error', reject)
+  );
+
+  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, progress);
+
+  // remove archivePath eagerly since these archives can be large.
+  await fs.remove(archivePath);
 }
 
-async function checkForFailingResponse(response: Response): Promise<void | never> {
+async function checkForFailingResponse(response: Response): Promise<Response | never> {
   if (response.ok) {
-    return;
+    return response;
   }
 
   // An error downloading the database. Attempt to extract the resaon behind it.
@@ -322,7 +315,7 @@ async function checkForFailingResponse(response: Response): Promise<void | never
 }
 
 function isFile(databaseUrl: string) {
-  return Uri.parse(databaseUrl).scheme === "file";
+  return Uri.parse(databaseUrl).scheme === 'file';
 }
 
 /**
@@ -377,16 +370,16 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
 
   try {
     const uri = Uri.parse(lgtmUrl, true);
-    if (uri.scheme !== "https") {
+    if (uri.scheme !== 'https') {
       return false;
     }
 
-    if (uri.authority !== "lgtm.com" && uri.authority !== "www.lgtm.com") {
+    if (uri.authority !== 'lgtm.com' && uri.authority !== 'www.lgtm.com') {
       return false;
     }
 
-    const paths = uri.path.split("/").filter((segment) => segment);
-    return paths.length >= 4 && paths[0] === "projects";
+    const paths = uri.path.split('/').filter((segment) => segment);
+    return paths.length >= 4 && paths[0] === 'projects';
   } catch (e) {
     return false;
   }
@@ -396,10 +389,10 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
 export async function convertToDatabaseUrl(lgtmUrl: string) {
   try {
     const uri = Uri.parse(lgtmUrl, true);
-    const paths = ["api", "v1.0"].concat(
-      uri.path.split("/").filter((segment) => segment)
+    const paths = ['api', 'v1.0'].concat(
+      uri.path.split('/').filter((segment) => segment)
     ).slice(0, 6);
-    const projectUrl = `https://lgtm.com/${paths.join("/")}`;
+    const projectUrl = `https://lgtm.com/${paths.join('/')}`;
     const projectResponse = await fetch(projectUrl);
     const projectJson = await projectResponse.json();
 
@@ -412,12 +405,12 @@ export async function convertToDatabaseUrl(lgtmUrl: string) {
       return;
     }
     return `https://lgtm.com/${[
-      "api",
-      "v1.0",
-      "snapshots",
+      'api',
+      'v1.0',
+      'snapshots',
       projectJson.id,
       language,
-    ].join("/")}`;
+    ].join('/')}`;
   } catch (e) {
     logger.log(`Error: ${e.message}`);
     throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
@@ -436,7 +429,28 @@ async function promptForLanguage(
 
   return await window.showQuickPick(
     projectJson.languages.map((lang: { language: string }) => lang.language), {
-    placeHolder: "Select the database language to download:"
+    placeHolder: 'Select the database language to download:'
   }
   );
 }
+
+/**
+ * Databases created by the old odasa tool will not have a zipped
+ * source location. However, this extension works better if sources
+ * are zipped.
+ *
+ * This function ensures that the source location is zipped. If the
+ * `src` folder exists and the `src.zip` file does not, the `src`
+ * folder will be zipped and then deleted.
+ *
+ * @param databasePath The full path to the unzipped database
+ */
+async function ensureZippedSourceLocation(databasePath: string): Promise<void> {
+  const srcFolderPath = path.join(databasePath, 'src');
+  const srcZipPath = srcFolderPath + '.zip';
+
+  if ((await fs.pathExists(srcFolderPath)) && !(await fs.pathExists(srcZipPath))) {
+    await zip(srcFolderPath, srcZipPath);
+    await fs.remove(srcFolderPath);
+  }
+}

extensions/ql-vscode/src/databases-ui.ts

@@ -1,15 +1,44 @@
 import * as path from 'path';
-import { DisposableObject } from 'semmle-vscode-utils';
-import { commands, Event, EventEmitter, ExtensionContext, ProviderResult, TreeDataProvider, TreeItem, Uri, window, env } from 'vscode';
+import { DisposableObject } from './vscode-utils/disposable-object';
+import {
+  Event,
+  EventEmitter,
+  ProviderResult,
+  TreeDataProvider,
+  TreeItem,
+  Uri,
+  window,
+  env,
+} from 'vscode';
+import * as fs from 'fs-extra';
+
 import * as cli from './cli';
-import { DatabaseItem, DatabaseManager, getUpgradesDirectories } from './databases';
-import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
+import {
+  DatabaseChangedEvent,
+  DatabaseItem,
+  DatabaseManager,
+  getUpgradesDirectories,
+} from './databases';
+import {
+  commandRunner,
+  commandRunnerWithProgress,
+  getOnDiskWorkspaceFolders,
+  ProgressCallback,
+  showAndLogErrorMessage,
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder
+} from './helpers';
 import { logger } from './logging';
-import { clearCacheInDatabase, UserCancellationException } from './run-queries';
+import { clearCacheInDatabase } from './run-queries';
 import * as qsClient from './queryserver-client';
 import { upgradeDatabase } from './upgrades';
-import { importArchiveDatabase, promptImportInternetDatabase, promptImportLgtmDatabase } from './databaseFetcher';
-import * as fs from 'fs-extra';
+import {
+  importArchiveDatabase,
+  promptImportInternetDatabase,
+  promptImportLgtmDatabase,
+} from './databaseFetcher';
+import { CancellationToken } from 'vscode';
+import { asyncFilter } from './pure/helpers-pure';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -26,21 +55,23 @@ const SELECTED_DATABASE_ICON: ThemableIconPath = {
  */
 const INVALID_DATABASE_ICON: ThemableIconPath = 'media/red-x.svg';
 
-function joinThemableIconPath(base: string, iconPath: ThemableIconPath): ThemableIconPath {
+function joinThemableIconPath(
+  base: string,
+  iconPath: ThemableIconPath
+): ThemableIconPath {
   if (typeof iconPath == 'object')
     return {
       light: path.join(base, iconPath.light),
-      dark: path.join(base, iconPath.dark)
+      dark: path.join(base, iconPath.dark),
     };
-  else
-    return path.join(base, iconPath);
+  else return path.join(base, iconPath);
 }
 
 enum SortOrder {
   NameAsc = 'NameAsc',
   NameDesc = 'NameDesc',
   DateAddedAsc = 'DateAddedAsc',
-  DateAddedDesc = 'DateAddedDesc'
+  DateAddedDesc = 'DateAddedDesc',
 }
 
 /**
@@ -48,48 +79,71 @@ enum SortOrder {
  */
 class DatabaseTreeDataProvider extends DisposableObject
   implements TreeDataProvider<DatabaseItem> {
-
   private _sortOrder = SortOrder.NameAsc;
 
-  private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
+  private readonly _onDidChangeTreeData = this.push(new EventEmitter<DatabaseItem | undefined>());
   private currentDatabaseItem: DatabaseItem | undefined;
 
-  constructor(private ctx: ExtensionContext, private databaseManager: DatabaseManager) {
+  constructor(
+    private databaseManager: DatabaseManager,
+    private readonly extensionPath: string
+  ) {
     super();
 
     this.currentDatabaseItem = databaseManager.currentDatabaseItem;
 
-    this.push(this.databaseManager.onDidChangeDatabaseItem(this.handleDidChangeDatabaseItem));
-    this.push(this.databaseManager.onDidChangeCurrentDatabaseItem(
-      this.handleDidChangeCurrentDatabaseItem));
+    this.push(
+      this.databaseManager.onDidChangeDatabaseItem(
+        this.handleDidChangeDatabaseItem
+      )
+    );
+    this.push(
+      this.databaseManager.onDidChangeCurrentDatabaseItem(
+        this.handleDidChangeCurrentDatabaseItem
+      )
+    );
   }
 
   public get onDidChangeTreeData(): Event<DatabaseItem | undefined> {
     return this._onDidChangeTreeData.event;
   }
 
-  private handleDidChangeDatabaseItem = (databaseItem: DatabaseItem | undefined): void => {
-    this._onDidChangeTreeData.fire(databaseItem);
-  }
+  private handleDidChangeDatabaseItem = (event: DatabaseChangedEvent): void => {
+    // Note that events from the databse manager are instances of DatabaseChangedEvent
+    // and events fired by the UI are instances of DatabaseItem
 
-  private handleDidChangeCurrentDatabaseItem = (databaseItem: DatabaseItem | undefined): void => {
+    // When event.item is undefined, then the entire tree is refreshed.
+    // When event.item is a db item, then only that item is refreshed.
+    this._onDidChangeTreeData.fire(event.item);
+  };
+
+  private handleDidChangeCurrentDatabaseItem = (
+    event: DatabaseChangedEvent
+  ): void => {
     if (this.currentDatabaseItem) {
       this._onDidChangeTreeData.fire(this.currentDatabaseItem);
     }
-    this.currentDatabaseItem = databaseItem;
+    this.currentDatabaseItem = event.item;
     if (this.currentDatabaseItem) {
       this._onDidChangeTreeData.fire(this.currentDatabaseItem);
     }
-  }
+  };
 
   public getTreeItem(element: DatabaseItem): TreeItem {
     const item = new TreeItem(element.name);
     if (element === this.currentDatabaseItem) {
-      item.iconPath = joinThemableIconPath(this.ctx.extensionPath, SELECTED_DATABASE_ICON);
+      item.iconPath = joinThemableIconPath(
+        this.extensionPath,
+        SELECTED_DATABASE_ICON
+      );
     } else if (element.error !== undefined) {
-      item.iconPath = joinThemableIconPath(this.ctx.extensionPath, INVALID_DATABASE_ICON);
+      item.iconPath = joinThemableIconPath(
+        this.extensionPath,
+        INVALID_DATABASE_ICON
+      );
     }
     item.tooltip = element.databaseUri.fsPath;
+    item.description = element.language;
     return item;
   }
 
@@ -98,17 +152,16 @@ class DatabaseTreeDataProvider extends DisposableObject
       return this.databaseManager.databaseItems.slice(0).sort((db1, db2) => {
         switch (this.sortOrder) {
           case SortOrder.NameAsc:
-            return db1.name.localeCompare(db2.name);
+            return db1.name.localeCompare(db2.name, env.language);
           case SortOrder.NameDesc:
-            return db2.name.localeCompare(db1.name);
+            return db2.name.localeCompare(db1.name, env.language);
           case SortOrder.DateAddedAsc:
             return (db1.dateAdded || 0) - (db2.dateAdded || 0);
           case SortOrder.DateAddedDesc:
             return (db2.dateAdded || 0) - (db1.dateAdded || 0);
         }
       });
-    }
-    else {
+    } else {
       return [];
     }
   }
@@ -135,8 +188,7 @@ class DatabaseTreeDataProvider extends DisposableObject
 function getFirst(list: Uri[] | undefined): Uri | undefined {
   if (list === undefined || list.length === 0) {
     return undefined;
-  }
-  else {
+  } else {
     return list[0];
   }
 }
@@ -155,7 +207,7 @@ async function chooseDatabaseDir(byFolder: boolean): Promise<Uri | undefined> {
     canSelectFiles: !byFolder,
     canSelectFolders: byFolder,
     canSelectMany: false,
-    filters: byFolder ? {} : { Archives: ['zip'] }
+    filters: byFolder ? {} : { Archives: ['zip'] },
   });
   return getFirst(chosen);
 }
@@ -164,62 +216,266 @@ export class DatabaseUI extends DisposableObject {
   private treeDataProvider: DatabaseTreeDataProvider;
 
   public constructor(
-    ctx: ExtensionContext,
     private cliserver: cli.CodeQLCliServer,
     private databaseManager: DatabaseManager,
     private readonly queryServer: qsClient.QueryServerClient | undefined,
-    private readonly storagePath: string
+    private readonly storagePath: string,
+    readonly extensionPath: string
   ) {
     super();
 
-    this.treeDataProvider = this.push(new DatabaseTreeDataProvider(ctx, databaseManager));
-    this.push(window.createTreeView('codeQLDatabases', { treeDataProvider: this.treeDataProvider }));
+    this.treeDataProvider = this.push(
+      new DatabaseTreeDataProvider(databaseManager, extensionPath)
+    );
+    this.push(
+      window.createTreeView('codeQLDatabases', {
+        treeDataProvider: this.treeDataProvider,
+        canSelectMany: true,
+      })
+    );
+  }
 
+  init() {
     logger.log('Registering database panel commands.');
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseFolder', this.handleChooseDatabaseFolder));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseArchive', this.handleChooseDatabaseArchive));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseInternet', this.handleChooseDatabaseInternet));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseLgtm', this.handleChooseDatabaseLgtm));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.setCurrentDatabase', this.handleSetCurrentDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.upgradeCurrentDatabase', this.handleUpgradeCurrentDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.clearCache', this.handleClearCache));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.setCurrentDatabase', this.handleMakeCurrentDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.sortByName', this.handleSortByName));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.sortByDateAdded', this.handleSortByDateAdded));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.removeDatabase', this.handleRemoveDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.upgradeDatabase', this.handleUpgradeDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.renameDatabase', this.handleRenameDatabase));
-    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.openDatabaseFolder', this.handleOpenFolder));
+    this.push(
+      commandRunnerWithProgress(
+        'codeQL.setCurrentDatabase',
+        this.handleSetCurrentDatabase,
+        {
+          title: 'Importing database from archive',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQL.upgradeCurrentDatabase',
+        this.handleUpgradeCurrentDatabase,
+        {
+          title: 'Upgrading current database',
+          cancellable: true,
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQL.clearCache',
+        this.handleClearCache,
+        {
+          title: 'Clearing Cache',
+        })
+    );
+
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseFolder',
+        this.handleChooseDatabaseFolder,
+        {
+          title: 'Adding database from folder',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseArchive',
+        this.handleChooseDatabaseArchive,
+        {
+          title: 'Adding database from archive',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseInternet',
+        this.handleChooseDatabaseInternet,
+        {
+          title: 'Adding database from URL',
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseLgtm',
+        this.handleChooseDatabaseLgtm,
+        {
+          title: 'Adding database from LGTM',
+        })
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.setCurrentDatabase',
+        this.handleMakeCurrentDatabase
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.sortByName',
+        this.handleSortByName
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.sortByDateAdded',
+        this.handleSortByDateAdded
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.removeDatabase',
+        this.handleRemoveDatabase,
+        {
+          title: 'Removing database',
+          cancellable: false
+        }
+      )
+    );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.upgradeDatabase',
+        this.handleUpgradeDatabase,
+        {
+          title: 'Upgrading database',
+          cancellable: true,
+        }
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.renameDatabase',
+        this.handleRenameDatabase
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.openDatabaseFolder',
+        this.handleOpenFolder
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.removeOrphanedDatabases',
+        this.handleRemoveOrphanedDatabases
+      )
+    );
   }
 
-  private handleMakeCurrentDatabase = async (databaseItem: DatabaseItem): Promise<void> => {
+  private handleMakeCurrentDatabase = async (
+    databaseItem: DatabaseItem
+  ): Promise<void> => {
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
-  }
+  };
 
-  handleChooseDatabaseFolder = async (): Promise<DatabaseItem | undefined> => {
+  handleChooseDatabaseFolder = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
     try {
-      return await this.chooseAndSetDatabase(true);
+      return await this.chooseAndSetDatabase(true, progress, token);
     } catch (e) {
       showAndLogErrorMessage(e.message);
       return undefined;
     }
-  }
+  };
 
-  handleChooseDatabaseArchive = async (): Promise<DatabaseItem | undefined> => {
+  handleRemoveOrphanedDatabases = async (): Promise<void> => {
+    logger.log('Removing orphaned databases from workspace storage.');
+    let dbDirs = undefined;
+
+    if (
+      !(await fs.pathExists(this.storagePath) ||
+        !(await fs.stat(this.storagePath)).isDirectory())
+    ) {
+      logger.log('Missing or invalid storage directory. Not trying to remove orphaned databases.');
+      return;
+    }
+
+    dbDirs =
+      // read directory
+      (await fs.readdir(this.storagePath, { withFileTypes: true }))
+        // remove non-directories
+        .filter(dirent => dirent.isDirectory())
+        // get the full path
+        .map(dirent => path.join(this.storagePath, dirent.name))
+        // remove databases still in workspace
+        .filter(dbDir => {
+          const dbUri = Uri.file(dbDir);
+          return this.databaseManager.databaseItems.every(item => item.databaseUri.fsPath !== dbUri.fsPath);
+        });
+
+    // remove non-databases
+    dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
+
+    if (!dbDirs.length) {
+      logger.log('No orphaned databases found.');
+      return;
+    }
+
+    // delete
+    const failures = [] as string[];
+    await Promise.all(
+      dbDirs.map(async dbDir => {
+        try {
+          logger.log(`Deleting orphaned database '${dbDir}'.`);
+          await fs.rmdir(dbDir, { recursive: true } as any);  // typings doesn't recognize the options argument
+        } catch (e) {
+          failures.push(`${path.basename(dbDir)}`);
+        }
+      })
+    );
+
+    if (failures.length) {
+      const dirname = path.dirname(failures[0]);
+      showAndLogErrorMessage(
+        `Failed to delete unused databases:\n  ${
+        failures.join('\n  ')
+        }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.`
+      );
+    }
+  };
+
+
+  handleChooseDatabaseArchive = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
     try {
-      return await this.chooseAndSetDatabase(false);
+      return await this.chooseAndSetDatabase(false, progress, token);
     } catch (e) {
       showAndLogErrorMessage(e.message);
       return undefined;
     }
-  }
+  };
 
-  handleChooseDatabaseInternet = async (): Promise<DatabaseItem | undefined> => {
-    return await promptImportInternetDatabase(this.databaseManager, this.storagePath);
-  }
+  handleChooseDatabaseInternet = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<
+    DatabaseItem | undefined
+  > => {
+    return await promptImportInternetDatabase(
+      this.databaseManager,
+      this.storagePath,
+      progress,
+      token
+    );
+  };
 
-  handleChooseDatabaseLgtm = async (): Promise<DatabaseItem | undefined> => {
-    return await promptImportLgtmDatabase(this.databaseManager, this.storagePath);
+  handleChooseDatabaseLgtm = async (
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
+    return await promptImportLgtmDatabase(
+      this.databaseManager,
+      this.storagePath,
+      progress,
+      token
+    );
+  };
+
+  async tryUpgradeCurrentDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ) {
+    await this.handleUpgradeCurrentDatabase(progress, token);
   }
 
   private handleSortByName = async () => {
@@ -228,7 +484,7 @@ export class DatabaseUI extends DisposableObject {
     } else {
       this.treeDataProvider.sortOrder = SortOrder.NameAsc;
     }
-  }
+  };
 
   private handleSortByDateAdded = async () => {
     if (this.treeDataProvider.sortOrder === SortOrder.DateAddedAsc) {
@@ -236,28 +492,49 @@ export class DatabaseUI extends DisposableObject {
     } else {
       this.treeDataProvider.sortOrder = SortOrder.DateAddedAsc;
     }
-  }
+  };
 
-  private handleUpgradeCurrentDatabase = async (): Promise<void> => {
-    await this.handleUpgradeDatabase(this.databaseManager.currentDatabaseItem);
-  }
+  private handleUpgradeCurrentDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> => {
+    await this.handleUpgradeDatabase(
+      progress, token,
+      this.databaseManager.currentDatabaseItem,
+      []
+    );
+  };
 
-  private handleUpgradeDatabase = async (databaseItem: DatabaseItem | undefined): Promise<void> => {
+  private handleUpgradeDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+    databaseItem: DatabaseItem | undefined,
+    multiSelect: DatabaseItem[] | undefined,
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      await Promise.all(
+        multiSelect.map((dbItem) => this.handleUpgradeDatabase(progress, token, dbItem, []))
+      );
+    }
     if (this.queryServer === undefined) {
-      logger.log('Received request to upgrade database, but there is no running query server.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but there is no running query server.'
+      );
     }
     if (databaseItem === undefined) {
-      logger.log('Received request to upgrade database, but no database was provided.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but no database was provided.'
+      );
     }
     if (databaseItem.contents === undefined) {
-      logger.log('Received request to upgrade database, but database contents could not be found.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but database contents could not be found.'
+      );
     }
     if (databaseItem.contents.dbSchemeUri === undefined) {
-      logger.log('Received request to upgrade database, but database has no schema.');
-      return;
+      throw new Error(
+        'Received request to upgrade database, but database has no schema.'
+      );
     }
 
     // Search for upgrade scripts in any workspace folders available
@@ -265,90 +542,139 @@ export class DatabaseUI extends DisposableObject {
 
     const upgradeInfo = await this.cliserver.resolveUpgrades(
       databaseItem.contents.dbSchemeUri.fsPath,
-      searchPath,
+      searchPath
     );
 
     const { scripts, finalDbscheme } = upgradeInfo;
 
     if (finalDbscheme === undefined) {
-      logger.log('Could not determine target dbscheme to upgrade to.');
-      return;
+      throw new Error('Could not determine target dbscheme to upgrade to.');
     }
     const targetDbSchemeUri = Uri.file(finalDbscheme);
 
+    await upgradeDatabase(
+      this.queryServer,
+      databaseItem,
+      targetDbSchemeUri,
+      getUpgradesDirectories(scripts),
+      progress,
+      token
+    );
+  };
+
+  private handleClearCache = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> => {
+    if (
+      this.queryServer !== undefined &&
+      this.databaseManager.currentDatabaseItem !== undefined
+    ) {
+      await clearCacheInDatabase(
+        this.queryServer,
+        this.databaseManager.currentDatabaseItem,
+        progress,
+        token
+      );
+    }
+  };
+
+  private handleSetCurrentDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+    uri: Uri,
+  ): Promise<void> => {
     try {
-      await upgradeDatabase(this.queryServer, databaseItem, targetDbSchemeUri, getUpgradesDirectories(scripts));
-    }
-    catch (e) {
-      if (e instanceof UserCancellationException) {
-        logger.log(e.message);
-      }
-      else
-        throw e;
-    }
-  }
-
-  private handleClearCache = async (): Promise<void> => {
-    if ((this.queryServer !== undefined) &&
-      (this.databaseManager.currentDatabaseItem !== undefined)) {
-
-      await clearCacheInDatabase(this.queryServer, this.databaseManager.currentDatabaseItem);
-    }
-  }
-
-  private handleSetCurrentDatabase = async (uri: Uri): Promise<DatabaseItem | undefined> => {
-    // Assume user has selected an archive if the file has a .zip extension
-    if (uri.path.endsWith('.zip')) {
-      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
-    }
-
-    return await this.setCurrentDatabase(uri);
-  }
-
-  private handleRemoveDatabase = (databaseItem: DatabaseItem): void => {
-    this.databaseManager.removeDatabaseItem(databaseItem);
-  }
-
-  private handleRenameDatabase = async (databaseItem: DatabaseItem): Promise<void> => {
-    try {
-      const newName = await window.showInputBox({
-        prompt: 'Choose new database name',
-        value: databaseItem.name
-      });
-
-      if (newName) {
-        this.databaseManager.renameDatabaseItem(databaseItem, newName);
+      // Assume user has selected an archive if the file has a .zip extension
+      if (uri.path.endsWith('.zip')) {
+        await importArchiveDatabase(
+          uri.toString(true),
+          this.databaseManager,
+          this.storagePath,
+          progress,
+          token
+        );
+      } else {
+        await this.setCurrentDatabase(progress, token, uri);
       }
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      // rethrow and let this be handled by default error handling.
+      throw new Error(
+        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${
+        e.message
+        }`
+      );
     }
-  }
+  };
 
-  private handleOpenFolder = async (databaseItem: DatabaseItem): Promise<void> => {
-    try {
+  private handleRemoveDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      await Promise.all(multiSelect.map((dbItem) =>
+        this.databaseManager.removeDatabaseItem(progress, token, dbItem)
+      ));
+    } else {
+      await this.databaseManager.removeDatabaseItem(progress, token, databaseItem);
+    }
+  };
+
+  private handleRenameDatabase = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    this.assertSingleDatabase(multiSelect);
+
+    const newName = await window.showInputBox({
+      prompt: 'Choose new database name',
+      value: databaseItem.name,
+    });
+
+    if (newName) {
+      this.databaseManager.renameDatabaseItem(databaseItem, newName);
+    }
+  };
+
+  private handleOpenFolder = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      await Promise.all(
+        multiSelect.map((dbItem) => env.openExternal(dbItem.databaseUri))
+      );
+    } else {
       await env.openExternal(databaseItem.databaseUri);
-    } catch (e) {
-      showAndLogErrorMessage(e.message);
     }
-  }
+  };
 
   /**
    * Return the current database directory. If we don't already have a
    * current database, ask the user for one, and return that, or
    * undefined if they cancel.
    */
-  public async getDatabaseItem(): Promise<DatabaseItem | undefined> {
+  public async getDatabaseItem(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> {
     if (this.databaseManager.currentDatabaseItem === undefined) {
-      await this.chooseAndSetDatabase(false);
+      await this.chooseAndSetDatabase(false, progress, token);
     }
 
     return this.databaseManager.currentDatabaseItem;
   }
 
-  private async setCurrentDatabase(uri: Uri): Promise<DatabaseItem | undefined> {
+  private async setCurrentDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    uri: Uri
+  ): Promise<DatabaseItem | undefined> {
     let databaseItem = this.databaseManager.findDatabaseItem(uri);
     if (databaseItem === undefined) {
-      databaseItem = await this.databaseManager.openDatabase(uri);
+      databaseItem = await this.databaseManager.openDatabase(progress, token, uri);
     }
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
 
@@ -359,7 +685,11 @@ export class DatabaseUI extends DisposableObject {
    * Ask the user for a database directory. Returns the chosen database, or `undefined` if the
    * operation was canceled.
    */
-  private async chooseAndSetDatabase(byFolder: boolean): Promise<DatabaseItem | undefined> {
+  private async chooseAndSetDatabase(
+    byFolder: boolean,
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<DatabaseItem | undefined> {
     const uri = await chooseDatabaseDir(byFolder);
 
     if (!uri) {
@@ -369,12 +699,17 @@ export class DatabaseUI extends DisposableObject {
     if (byFolder) {
       const fixedUri = await this.fixDbUri(uri);
       // we are selecting a database folder
-      return await this.setCurrentDatabase(fixedUri);
-    }
-    else {
+      return await this.setCurrentDatabase(progress, token, fixedUri);
+    } else {
       // we are selecting a database archive. Must unzip into a workspace-controlled area
       // before importing.
-      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
+      return await importArchiveDatabase(
+        uri.toString(true),
+        this.databaseManager,
+        this.storagePath,
+        progress,
+        token
+      );
     }
   }
 
@@ -394,9 +729,19 @@ export class DatabaseUI extends DisposableObject {
     if ((await fs.stat(dbPath)).isFile()) {
       dbPath = path.dirname(dbPath);
     }
-    if (path.basename(dbPath).startsWith('db-')) {
+
+    if (isLikelyDbLanguageFolder(dbPath)) {
       dbPath = path.dirname(dbPath);
     }
     return Uri.file(dbPath);
   }
+
+  private assertSingleDatabase(
+    multiSelect: DatabaseItem[] = [],
+    message = 'Please select a single database.'
+  ) {
+    if (multiSelect.length > 1) {
+      throw new Error(message);
+    }
+  }
 }

extensions/ql-vscode/src/databases.ts

@@ -4,11 +4,19 @@ import * as path from 'path';
 import * as vscode from 'vscode';
 import * as cli from './cli';
 import { ExtensionContext } from 'vscode';
-import { showAndLogErrorMessage, showAndLogWarningMessage, showAndLogInformationMessage } from './helpers';
-import { zipArchiveScheme, encodeSourceArchiveUri, decodeSourceArchiveUri } from './archive-filesystem-provider';
-import { DisposableObject } from 'semmle-vscode-utils';
-import { QueryServerConfig } from './config';
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  showAndLogInformationMessage,
+  isLikelyDatabaseRoot,
+  ProgressCallback,
+  withProgress
+} from './helpers';
+import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
+import { DisposableObject } from './vscode-utils/disposable-object';
 import { Logger, logger } from './logging';
+import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
+import { QueryServerClient } from './queryserver-client';
 
 /**
  * databases.ts
@@ -36,11 +44,13 @@ export interface DatabaseOptions {
   displayName?: string;
   ignoreSourceArchive?: boolean;
   dateAdded?: number | undefined;
+  language?: string;
 }
 
-interface FullDatabaseOptions extends DatabaseOptions {
+export interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
   dateAdded: number | undefined;
+  language: string | undefined;
 }
 
 interface PersistedDatabaseItem {
@@ -121,19 +131,21 @@ async function findSourceArchive(
 
     if (await fs.pathExists(basePath)) {
       return vscode.Uri.file(basePath);
-    }
-    else if (await fs.pathExists(zipPath)) {
-      return vscode.Uri.file(zipPath).with({ scheme: zipArchiveScheme });
+    } else if (await fs.pathExists(zipPath)) {
+      return encodeArchiveBasePath(zipPath);
     }
   }
-  if (!silent)
-    showAndLogInformationMessage(`Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`);
+  if (!silent) {
+    showAndLogInformationMessage(
+      `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`
+    );
+  }
   return undefined;
 }
 
 async function resolveDatabase(
   databasePath: string
-): Promise<DatabaseContents | undefined> {
+): Promise<DatabaseContents> {
 
   const name = path.basename(databasePath);
 
@@ -155,20 +167,6 @@ async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
   return await glob('*.dbscheme', { cwd: dbDirectory });
 }
 
-async function resolveRawDataset(datasetPath: string): Promise<DatabaseContents | undefined> {
-  if ((await getDbSchemeFiles(datasetPath)).length > 0) {
-    return {
-      kind: DatabaseKind.RawDataset,
-      name: path.basename(datasetPath),
-      datasetUri: vscode.Uri.file(datasetPath),
-      sourceArchiveUri: undefined
-    };
-  }
-  else {
-    return undefined;
-  }
-}
-
 async function resolveDatabaseContents(uri: vscode.Uri): Promise<DatabaseContents> {
   if (uri.scheme !== 'file') {
     throw new Error(`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`);
@@ -178,7 +176,7 @@ async function resolveDatabaseContents(uri: vscode.Uri): Promise<DatabaseContent
     throw new InvalidDatabaseError(`Database '${databasePath}' does not exist.`);
   }
 
-  const contents = await resolveDatabase(databasePath) || await resolveRawDataset(databasePath);
+  const contents = await resolveDatabase(databasePath);
 
   if (contents === undefined) {
     throw new InvalidDatabaseError(`'${databasePath}' is not a valid database.`);
@@ -205,6 +203,9 @@ export interface DatabaseItem {
   readonly databaseUri: vscode.Uri;
   /** The name of the database to be displayed in the UI */
   name: string;
+
+  /** The primary language of the database or empty string if unknown */
+  readonly language: string;
   /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
   readonly sourceArchive: vscode.Uri | undefined;
   /**
@@ -261,18 +262,44 @@ export interface DatabaseItem {
    * Holds if `uri` belongs to this database's source archive.
    */
   belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  getPersistedState(): PersistedDatabaseItem;
 }
 
-class DatabaseItemImpl implements DatabaseItem {
+export enum DatabaseEventKind {
+  Add = 'Add',
+  Remove = 'Remove',
+
+  // Fired when databases are refreshed from persisted state
+  Refresh = 'Refresh',
+
+  // Fired when the current database changes
+  Change = 'Change',
+
+  Rename = 'Rename'
+}
+
+export interface DatabaseChangedEvent {
+  kind: DatabaseEventKind;
+  item: DatabaseItem | undefined;
+}
+
+// Exported for testing
+export class DatabaseItemImpl implements DatabaseItem {
   private _error: Error | undefined = undefined;
   private _contents: DatabaseContents | undefined;
   /** A cache of database info */
   private _dbinfo: cli.DbInfo | undefined;
 
-  public constructor(public readonly databaseUri: vscode.Uri,
-    contents: DatabaseContents | undefined, private options: FullDatabaseOptions,
-    private readonly onChanged: (item: DatabaseItemImpl) => void) {
-
+  public constructor(
+    public readonly databaseUri: vscode.Uri,
+    contents: DatabaseContents | undefined,
+    private options: FullDatabaseOptions,
+    private readonly onChanged: (event: DatabaseChangedEvent) => void
+  ) {
     this._contents = contents;
   }
 
@@ -295,8 +322,7 @@ class DatabaseItemImpl implements DatabaseItem {
   public get sourceArchive(): vscode.Uri | undefined {
     if (this.options.ignoreSourceArchive || (this._contents === undefined)) {
       return undefined;
-    }
-    else {
+    } else {
       return this._contents.sourceArchiveUri;
     }
   }
@@ -326,46 +352,52 @@ class DatabaseItemImpl implements DatabaseItem {
       }
     }
     finally {
-      this.onChanged(this);
+      this.onChanged({
+        kind: DatabaseEventKind.Refresh,
+        item: this
+      });
     }
   }
 
-  public resolveSourceFile(file: string | undefined): vscode.Uri {
+  public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
-    if (sourceArchive === undefined) {
-      if (file !== undefined) {
-        // Treat it as an absolute path.
-        return vscode.Uri.file(file);
-      }
-      else {
+    const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
+    if (uri && uri.scheme !== 'file') {
+      throw new Error(`Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`);
+    }
+    if (!sourceArchive) {
+      if (uri) {
+        return uri;
+      } else {
         return this.databaseUri;
       }
     }
-    else {
-      if (file !== undefined) {
-        const absoluteFilePath = file.replace(':', '_');
-        // Strip any leading slashes from the file path, and replace `:` with `_`.
-        const relativeFilePath = absoluteFilePath.replace(/^\/*/, '').replace(':', '_');
-        if (sourceArchive.scheme == zipArchiveScheme) {
-          return encodeSourceArchiveUri({
-            pathWithinSourceArchive: absoluteFilePath,
-            sourceArchiveZipPath: sourceArchive.fsPath,
-          });
-        }
-        else {
-          let newPath = sourceArchive.path;
-          if (!newPath.endsWith('/')) {
-            // Ensure a trailing slash.
-            newPath += '/';
-          }
-          newPath += relativeFilePath;
 
-          return sourceArchive.with({ path: newPath });
+    if (uri) {
+      const relativeFilePath = decodeURI(uri.path).replace(':', '_').replace(/^\/*/, '');
+      if (sourceArchive.scheme === zipArchiveScheme) {
+        const zipRef = decodeSourceArchiveUri(sourceArchive);
+        const pathWithinSourceArchive = zipRef.pathWithinSourceArchive === '/'
+          ? relativeFilePath
+          : zipRef.pathWithinSourceArchive + '/' + relativeFilePath;
+        return encodeSourceArchiveUri({
+          pathWithinSourceArchive,
+          sourceArchiveZipPath: zipRef.sourceArchiveZipPath,
+        });
+
+      } else {
+        let newPath = sourceArchive.path;
+        if (!newPath.endsWith('/')) {
+          // Ensure a trailing slash.
+          newPath += '/';
         }
+        newPath += relativeFilePath;
+
+        return sourceArchive.with({ path: newPath });
       }
-      else {
-        return sourceArchive;
-      }
+
+    } else {
+      return sourceArchive;
     }
   }
 
@@ -383,10 +415,7 @@ class DatabaseItemImpl implements DatabaseItem {
    * Holds if the database item refers to an exported snapshot
    */
   public async hasMetadataFile(): Promise<boolean> {
-    return (await Promise.all([
-      fs.pathExists(path.join(this.databaseUri.fsPath, '.dbinfo')),
-      fs.pathExists(path.join(this.databaseUri.fsPath, 'codeql-database.yml'))
-    ])).some(x => x);
+    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
   }
 
   /**
@@ -416,6 +445,10 @@ class DatabaseItemImpl implements DatabaseItem {
     return dbInfo.datasetFolder;
   }
 
+  public get language() {
+    return this.options.language || '';
+  }
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive.
    */
@@ -423,10 +456,7 @@ class DatabaseItemImpl implements DatabaseItem {
     const sourceArchive = this.sourceArchive;
     if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith('.zip'))
       return undefined;
-    return encodeSourceArchiveUri({
-      pathWithinSourceArchive: '/',
-      sourceArchiveZipPath: sourceArchive.fsPath,
-    });
+    return encodeArchiveBasePath(sourceArchive.fsPath);
   }
 
   /**
@@ -464,42 +494,47 @@ function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | un
 }
 
 export class DatabaseManager extends DisposableObject {
-  private readonly _onDidChangeDatabaseItem = this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  private readonly _onDidChangeDatabaseItem = this.push(new vscode.EventEmitter<DatabaseChangedEvent>());
 
   readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
 
-  private readonly _onDidChangeCurrentDatabaseItem = this.push(new vscode.EventEmitter<DatabaseItem | undefined>());
+  private readonly _onDidChangeCurrentDatabaseItem = this.push(new vscode.EventEmitter<DatabaseChangedEvent>());
   readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event;
 
-  private readonly _databaseItems: DatabaseItemImpl[] = [];
+  private readonly _databaseItems: DatabaseItem[] = [];
   private _currentDatabaseItem: DatabaseItem | undefined = undefined;
 
-  constructor(private ctx: ExtensionContext,
-    public config: QueryServerConfig,
-    public logger: Logger) {
+  constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly qs: QueryServerClient,
+    private readonly cli: cli.CodeQLCliServer,
+    public logger: Logger
+  ) {
     super();
 
     this.loadPersistedState();  // Let this run async.
   }
 
   public async openDatabase(
-    uri: vscode.Uri, options?: DatabaseOptions
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    uri: vscode.Uri,
   ): Promise<DatabaseItem> {
-
     const contents = await resolveDatabaseContents(uri);
-    const realOptions = options || {};
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj';
     const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ?
-        realOptions.ignoreSourceArchive : isQLTestDatabase,
-      displayName: realOptions.displayName,
-      dateAdded: realOptions.dateAdded || Date.now()
+      ignoreSourceArchive: isQLTestDatabase,
+      // displayName is only set if a user explicitly renames a database
+      displayName: undefined,
+      dateAdded: Date.now(),
+      language: await this.getPrimaryLanguage(uri.fsPath)
     };
-    const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (item) => {
-      this._onDidChangeDatabaseItem.fire(item);
+    const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => {
+      this._onDidChangeDatabaseItem.fire(event);
     });
-    await this.addDatabaseItem(databaseItem);
+
+    await this.addDatabaseItem(progress, token, databaseItem);
     await this.addDatabaseSourceArchiveFolder(databaseItem);
 
     return databaseItem;
@@ -545,12 +580,15 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async createDatabaseItemFromPersistedState(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
     state: PersistedDatabaseItem
   ): Promise<DatabaseItem> {
 
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
     let dateAdded = undefined;
+    let language = undefined;
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -561,43 +599,69 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.dateAdded === 'number') {
         dateAdded = state.options.dateAdded;
       }
+      language = state.options.language;
     }
+
+    const dbBaseUri = vscode.Uri.parse(state.uri, true);
+    if (language === undefined) {
+      // we haven't been successful yet at getting the language. try again
+      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
+    }
+
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive,
       displayName,
-      dateAdded
+      dateAdded,
+      language
     };
-    const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri), undefined, fullOptions,
-      (item) => {
-        this._onDidChangeDatabaseItem.fire(item);
+    const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions,
+      (event) => {
+        this._onDidChangeDatabaseItem.fire(event);
       });
-    await this.addDatabaseItem(item);
 
+    await this.addDatabaseItem(progress, token, item);
     return item;
   }
 
   private async loadPersistedState(): Promise<void> {
-    const currentDatabaseUri = this.ctx.workspaceState.get<string>(CURRENT_DB);
-    const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(DB_LIST, []);
-
-    try {
-      for (const database of databases) {
-        const databaseItem = await this.createDatabaseItemFromPersistedState(database);
+    return withProgress({
+      location: vscode.ProgressLocation.Notification
+    },
+      async (progress, token) => {
+        const currentDatabaseUri = this.ctx.workspaceState.get<string>(CURRENT_DB);
+        const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(DB_LIST, []);
+        let step = 0;
+        progress({
+          maxStep: databases.length,
+          message: 'Loading persisted databases',
+          step
+        });
         try {
-          await databaseItem.refresh();
-          if (currentDatabaseUri === database.uri) {
-            this.setCurrentDatabaseItem(databaseItem, true);
+          for (const database of databases) {
+            progress({
+              maxStep: databases.length,
+              message: `Loading ${database.options?.displayName || 'databases'}`,
+              step: ++step
+            });
+
+            const databaseItem = await this.createDatabaseItemFromPersistedState(progress, token, database);
+            try {
+              await databaseItem.refresh();
+              await this.registerDatabase(progress, token, databaseItem);
+              if (currentDatabaseUri === database.uri) {
+                this.setCurrentDatabaseItem(databaseItem, true);
+              }
+            }
+            catch (e) {
+              // When loading from persisted state, leave invalid databases in the list. They will be
+              // marked as invalid, and cannot be set as the current database.
+            }
           }
+        } catch (e) {
+          // database list had an unexpected type - nothing to be done?
+          showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
         }
-        catch (e) {
-          // When loading from persisted state, leave invalid databases in the list. They will be
-          // marked as invalid, and cannot be set as the current database.
-        }
-      }
-    } catch (e) {
-      // database list had an unexpected type - nothing to be done?
-      showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
-    }
+      });
   }
 
   public get databaseItems(): readonly DatabaseItem[] {
@@ -608,8 +672,10 @@ export class DatabaseManager extends DisposableObject {
     return this._currentDatabaseItem;
   }
 
-  public async setCurrentDatabaseItem(item: DatabaseItem | undefined,
-    skipRefresh = false): Promise<void> {
+  public async setCurrentDatabaseItem(
+    item: DatabaseItem | undefined,
+    skipRefresh = false
+  ): Promise<void> {
 
     if (!skipRefresh && (item !== undefined)) {
       await item.refresh();  // Will throw on invalid database.
@@ -617,7 +683,11 @@ export class DatabaseManager extends DisposableObject {
     if (this._currentDatabaseItem !== item) {
       this._currentDatabaseItem = item;
       this.updatePersistedCurrentDatabaseItem();
-      this._onDidChangeCurrentDatabaseItem.fire(item);
+
+      this._onDidChangeCurrentDatabaseItem.fire({
+        item,
+        kind: DatabaseEventKind.Change
+      });
     }
   }
 
@@ -640,21 +710,45 @@ export class DatabaseManager extends DisposableObject {
     return this._databaseItems.find(item => item.sourceArchive && item.sourceArchive.toString(true) === uriString);
   }
 
-  private async addDatabaseItem(item: DatabaseItemImpl) {
+  private async addDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem
+  ) {
     this._databaseItems.push(item);
     this.updatePersistedDatabaseList();
-    this._onDidChangeDatabaseItem.fire(undefined);
+
+    // Add this database item to the allow-list
+    // Database items reconstituted from persisted state
+    // will not have their contents yet.
+    if (item.contents?.datasetUri) {
+      await this.registerDatabase(progress, token, item);
+    }
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Add
+    });
   }
 
   public async renameDatabaseItem(item: DatabaseItem, newName: string) {
     item.name = newName;
     this.updatePersistedDatabaseList();
-    this._onDidChangeDatabaseItem.fire(item);
+    this._onDidChangeDatabaseItem.fire({
+      // pass undefined so that the entire tree is rebuilt in order to re-sort
+      item: undefined,
+      kind: DatabaseEventKind.Rename
+    });
   }
 
-  public removeDatabaseItem(item: DatabaseItem) {
-    if (this._currentDatabaseItem == item)
+  public async removeDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem
+  ) {
+    if (this._currentDatabaseItem == item) {
       this._currentDatabaseItem = undefined;
+    }
     const index = this.databaseItems.findIndex(searchItem => searchItem === item);
     if (index >= 0) {
       this._databaseItems.splice(index, 1);
@@ -662,21 +756,58 @@ export class DatabaseManager extends DisposableObject {
     this.updatePersistedDatabaseList();
 
     // Delete folder from workspace, if it is still there
-    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(folder => item.belongsToSourceArchiveExplorerUri(folder.uri));
-    if (index >= 0) {
+    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
+      folder => item.belongsToSourceArchiveExplorerUri(folder.uri)
+    );
+    if (folderIndex >= 0) {
       logger.log(`Removing workspace folder at index ${folderIndex}`);
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
     // Delete folder from file system only if it is controlled by the extension
     if (this.isExtensionControlledLocation(item.databaseUri)) {
-      logger.log(`Deleting database from filesystem.`);
-      fs.remove(item.databaseUri.path).then(
-        () => logger.log(`Deleted '${item.databaseUri.path}'`),
-        e => logger.log(`Failed to delete '${item.databaseUri.path}'. Reason: ${e.message}`));
+      logger.log('Deleting database from filesystem.');
+      fs.remove(item.databaseUri.fsPath).then(
+        () => logger.log(`Deleted '${item.databaseUri.fsPath}'`),
+        e => logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
     }
 
-    this._onDidChangeDatabaseItem.fire(undefined);
+    // Remove this database item from the allow-list
+    await this.deregisterDatabase(progress, token, item);
+
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Remove
+    });
+  }
+
+  private async deregisterDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    if (dbItem.contents && (await this.qs.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
+    }
+  }
+
+  private async registerDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    if (dbItem.contents && (await this.qs.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
+    }
   }
 
   private updatePersistedCurrentDatabaseItem(): void {
@@ -690,7 +821,24 @@ export class DatabaseManager extends DisposableObject {
 
   private isExtensionControlledLocation(uri: vscode.Uri) {
     const storagePath = this.ctx.storagePath || this.ctx.globalStoragePath;
-    return uri.path.startsWith(storagePath);
+    // the uri.fsPath function on windows returns a lowercase drive letter,
+    // but storagePath will have an uppercase drive letter. Be sure to compare
+    // URIs to URIs only
+    if (storagePath) {
+      return uri.fsPath.startsWith(vscode.Uri.file(storagePath).fsPath);
+    }
+    return false;
+  }
+
+  private async getPrimaryLanguage(dbPath: string) {
+    if (!(await this.cli.supportsLanguageName())) {
+      // return undefined so that we recalculate on restart until the cli is at a version that
+      // supports this feature. This recalculation is cheap since we avoid calling into the cli
+      // unless we know it can return the langauges property.
+      return undefined;
+    }
+    const dbInfo = await this.cli.resolveDatabase(dbPath);
+    return dbInfo.languages?.[0] || '';
   }
 }
 

extensions/ql-vscode/src/definitions.ts

@@ -1,214 +0,0 @@
-import * as fs from 'fs-extra';
-import * as yaml from 'js-yaml';
-import * as tmp from 'tmp';
-import * as vscode from "vscode";
-import { decodeSourceArchiveUri, zipArchiveScheme } from "./archive-filesystem-provider";
-import { ColumnKindCode, EntityValue, getResultSetSchema, LineColumnLocation, UrlValue } from "./bqrs-cli-types";
-import { CodeQLCliServer } from "./cli";
-import { DatabaseItem, DatabaseManager } from "./databases";
-import * as helpers from './helpers';
-import { CachedOperation } from './helpers';
-import * as messages from "./messages";
-import { QueryServerClient } from "./queryserver-client";
-import { compileAndRunQueryAgainstDatabase, QueryWithResults } from "./run-queries";
-
-/**
- * Run templated CodeQL queries to find definitions and references in
- * source-language files. We may eventually want to find a way to
- * generalize this to other custom queries, e.g. showing dataflow to
- * or from a selected identifier.
- */
-
-const TEMPLATE_NAME = "selectedSourceFile";
-const SELECT_QUERY_NAME = "#select";
-
-enum KeyType {
-  DefinitionQuery = 'DefinitionQuery',
-  ReferenceQuery = 'ReferenceQuery',
-}
-
-function tagOfKeyType(keyType: KeyType): string {
-  switch (keyType) {
-    case KeyType.DefinitionQuery: return "ide-contextual-queries/local-definitions";
-    case KeyType.ReferenceQuery: return "ide-contextual-queries/local-references";
-  }
-}
-
-function nameOfKeyType(keyType: KeyType): string {
-  switch (keyType) {
-    case KeyType.DefinitionQuery: return "definitions";
-    case KeyType.ReferenceQuery: return "references";
-  }
-}
-
-async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
-  const suiteFile = tmp.fileSync({ postfix: '.qls' }).name;
-  const suiteYaml = { qlpack, include: { kind: 'definitions', 'tags contain': tagOfKeyType(keyType) } };
-  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
-
-  const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
-  if (queries.length === 0) {
-    vscode.window.showErrorMessage(
-      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. It might be necessary to upgrade the CodeQL libraries.`
-    );
-    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
-  }
-  return queries;
-}
-
-async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string | undefined> {
-  if (db.contents === undefined)
-    return undefined;
-  const datasetPath = db.contents.datasetUri.fsPath;
-  const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath);
-  return qlpack;
-}
-
-interface FullLocationLink extends vscode.LocationLink {
-  originUri: vscode.Uri;
-}
-
-export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
-  private cache: CachedOperation<vscode.LocationLink[]>;
-
-  constructor(
-    private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
-    private dbm: DatabaseManager,
-  ) {
-    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
-  }
-
-  async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
-    return getLinksForUriString(this.cli, this.qs, this.dbm, uriString, KeyType.DefinitionQuery, (src, _dest) => src === uriString);
-  }
-
-  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
-    const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.LocationLink[] = [];
-    for (const link of fileLinks) {
-      if (link.originSelectionRange!.contains(position)) {
-        locLinks.push(link);
-      }
-    }
-    return locLinks;
-  }
-}
-
-export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
-  private cache: CachedOperation<FullLocationLink[]>;
-
-  constructor(
-    private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
-    private dbm: DatabaseManager,
-  ) {
-    this.cache = new CachedOperation<FullLocationLink[]>(this.getReferences.bind(this));
-  }
-
-  async getReferences(uriString: string): Promise<FullLocationLink[]> {
-    return getLinksForUriString(this.cli, this.qs, this.dbm, uriString, KeyType.ReferenceQuery, (_src, dest) => dest === uriString);
-  }
-
-  async provideReferences(document: vscode.TextDocument, position: vscode.Position, _context: vscode.ReferenceContext, _token: vscode.CancellationToken): Promise<vscode.Location[]> {
-    const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.Location[] = [];
-    for (const link of fileLinks) {
-      if (link.targetRange!.contains(position)) {
-        locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
-      }
-    }
-    return locLinks;
-  }
-}
-
-interface FileRange {
-  file: vscode.Uri;
-  range: vscode.Range;
-}
-
-async function getLinksFromResults(results: QueryWithResults, cli: CodeQLCliServer, db: DatabaseItem, filter: (srcFile: string, destFile: string) => boolean): Promise<FullLocationLink[]> {
-  const localLinks: FullLocationLink[] = [];
-  const bqrsPath = results.query.resultsPaths.resultsPath;
-  const info = await cli.bqrsInfo(bqrsPath);
-  const selectInfo = getResultSetSchema(SELECT_QUERY_NAME, info);
-  if (selectInfo && selectInfo.columns.length == 3
-    && selectInfo.columns[0].kind == ColumnKindCode.ENTITY
-    && selectInfo.columns[1].kind == ColumnKindCode.ENTITY
-    && selectInfo.columns[2].kind == ColumnKindCode.STRING) {
-    // TODO: Page this
-    const allTuples = await cli.bqrsDecode(bqrsPath, SELECT_QUERY_NAME);
-    for (const tuple of allTuples.tuples) {
-      const src = tuple[0] as EntityValue;
-      const dest = tuple[1] as EntityValue;
-      const srcFile = src.url && fileRangeFromURI(src.url, db);
-      const destFile = dest.url && fileRangeFromURI(dest.url, db);
-      if (srcFile && destFile && filter(srcFile.file.toString(), destFile.file.toString())) {
-        localLinks.push({ targetRange: destFile.range, targetUri: destFile.file, originSelectionRange: srcFile.range, originUri: srcFile.file });
-      }
-    }
-  }
-  return localLinks;
-}
-
-async function getLinksForUriString(
-  cli: CodeQLCliServer,
-  qs: QueryServerClient,
-  dbm: DatabaseManager,
-  uriString: string,
-  keyType: KeyType,
-  filter: (src: string, dest: string) => boolean
-) {
-  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString));
-  const sourceArchiveUri = vscode.Uri.file(uri.sourceArchiveZipPath).with({ scheme: zipArchiveScheme });
-
-  const db = dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
-  if (db) {
-    const qlpack = await qlpackOfDatabase(cli, db);
-    if (qlpack === undefined) {
-      throw new Error("Can't infer qlpack from database source archive");
-    }
-    const links: FullLocationLink[] = [];
-    for (const query of await resolveQueries(cli, qlpack, keyType)) {
-      const templates: messages.TemplateDefinitions = {
-        [TEMPLATE_NAME]: {
-          values: {
-            tuples: [[{
-              stringValue: uri.pathWithinSourceArchive
-            }]]
-          }
-        }
-      };
-      const results = await compileAndRunQueryAgainstDatabase(cli, qs, db, false, vscode.Uri.file(query), templates);
-      if (results.result.resultType == messages.QueryResultType.SUCCESS) {
-        links.push(...await getLinksFromResults(results, cli, db, filter));
-      }
-    }
-    return links;
-  } else {
-    return [];
-  }
-}
-
-function fileRangeFromURI(uri: UrlValue, db: DatabaseItem): FileRange | undefined {
-  if (typeof uri === "string") {
-    return undefined;
-  } else if ('startOffset' in uri) {
-    return undefined;
-  } else {
-    const loc = uri as LineColumnLocation;
-    const range = new vscode.Range(Math.max(0, loc.startLine - 1),
-      Math.max(0, loc.startColumn - 1),
-      Math.max(0, loc.endLine - 1),
-      Math.max(0, loc.endColumn));
-    try {
-      const parsed = vscode.Uri.parse(uri.uri, true);
-      if (parsed.scheme === "file") {
-        return { file: db.resolveSourceFile(parsed.fsPath), range };
-      }
-      return undefined;
-    } catch (e) {
-      return undefined;
-    }
-  }
-}

extensions/ql-vscode/src/discovery.ts

@@ -1,4 +1,5 @@
-import { DisposableObject } from 'semmle-vscode-utils';
+import { DisposableObject } from './vscode-utils/disposable-object';
+import { logger } from './logging';
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
@@ -9,7 +10,7 @@ export abstract class Discovery<T> extends DisposableObject {
   private retry = false;
   private discoveryInProgress = false;
 
-  constructor() {
+  constructor(private readonly name: string) {
     super();
   }
 
@@ -59,6 +60,11 @@ export abstract class Discovery<T> extends DisposableObject {
         this.update(results);
       }
     });
+
+    discoveryPromise.catch(err => {
+      logger.log(`${this.name} failed. Reason: ${err.message}`);
+    });
+
     discoveryPromise.finally(() => {
       if (this.retry) {
         // Another refresh request came in while we were still running a previous discovery

extensions/ql-vscode/src/distribution.ts

@@ -1,15 +1,16 @@
-import * as fetch from "node-fetch";
-import * as fs from "fs-extra";
-import * as os from "os";
-import * as path from "path";
-import * as unzipper from "unzipper";
-import * as url from "url";
-import { ExtensionContext, Event } from "vscode";
-import { DistributionConfig } from "./config";
-import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from "./helpers";
-import { logger } from "./logging";
-import * as helpers from "./helpers";
-import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-version";
+import * as fetch from 'node-fetch';
+import * as fs from 'fs-extra';
+import * as os from 'os';
+import * as path from 'path';
+import * as semver from 'semver';
+import * as unzipper from 'unzipper';
+import * as url from 'url';
+import { ExtensionContext, Event } from 'vscode';
+import { DistributionConfig } from './config';
+import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from './helpers';
+import { logger } from './logging';
+import * as helpers from './helpers';
+import { getCodeQlCliVersion } from './cli-version';
 
 /**
  * distribution.ts
@@ -24,7 +25,7 @@ import { getCodeQlCliVersion, tryParseVersionString, Version } from "./cli-versi
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
-const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";
+const DEFAULT_DISTRIBUTION_OWNER_NAME = 'github';
 
 /**
  * Default value for the repository name of the extension-managed distribution on GitHub.
@@ -32,64 +33,99 @@ const DEFAULT_DISTRIBUTION_OWNER_NAME = "github";
  * We set the default here rather than as a default config value so that this default is invoked
  * upon blanking the setting.
  */
-const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = "codeql-cli-binaries";
+const DEFAULT_DISTRIBUTION_REPOSITORY_NAME = 'codeql-cli-binaries';
 
 /**
- * Version constraint for the CLI.
+ * Range of versions of the CLI that are compatible with the extension.
  *
  * This applies to both extension-managed and CLI distributions.
  */
-export const DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT: VersionConstraint = {
-  description: "2.*.*",
-  isVersionCompatible: (v: Version) => {
-    return v.majorVersion === 2 && v.minorVersion >= 0;
-  }
-};
+export const DEFAULT_DISTRIBUTION_VERSION_RANGE: semver.Range = new semver.Range('2.x');
 
 export interface DistributionProvider {
   getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
   onDidChangeDistribution?: Event<void>;
+  getDistribution(): Promise<FindDistributionResult>;
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
-    this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionConstraint);
-    this._onDidChangeDistribution = config.onDidChangeDistributionConfiguration;
-    this._updateCheckRateLimiter = new InvocationRateLimiter(
+
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  public static getRequiredAssetName(): string {
+    switch (os.platform()) {
+      case 'linux':
+        return 'codeql-linux64.zip';
+      case 'darwin':
+        return 'codeql-osx64.zip';
+      case 'win32':
+        return 'codeql-win64.zip';
+      default:
+        return 'codeql.zip';
+    }
+  }
+
+  constructor(
+    public readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    extensionContext: ExtensionContext
+  ) {
+    this._onDidChangeDistribution = config.onDidChangeConfiguration;
+    this.extensionSpecificDistributionManager =
+      new ExtensionSpecificDistributionManager(config, versionRange, extensionContext);
+    this.updateCheckRateLimiter = new InvocationRateLimiter(
       extensionContext,
-      "extensionSpecificDistributionUpdateCheck",
-      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+      'extensionSpecificDistributionUpdateCheck',
+      () => this.extensionSpecificDistributionManager.checkForUpdatesToDistribution()
     );
-    this._versionConstraint = versionConstraint;
   }
 
   /**
    * Look up a CodeQL launcher binary.
    */
   public async getDistribution(): Promise<FindDistributionResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath === undefined) {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    if (distribution === undefined) {
       return {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(codeQlPath, logger);
-    if (version !== undefined && !this._versionConstraint.isVersionCompatible(version)) {
+    const version = await getCodeQlCliVersion(distribution.codeQlPath, logger);
+    if (version === undefined) {
       return {
-        codeQlPath,
+        distribution,
+        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
+      };
+    }
+
+    /**
+     * Specifies whether prerelease versions of the CodeQL CLI should be accepted.
+     *
+     * Suppose a user sets the includePrerelease config option, obtains a prerelease, then decides
+     * they no longer want a prerelease, so unsets the includePrerelease config option.
+     * Unsetting the includePrerelease config option should trigger an update check, and this
+     * update check should present them an update that returns them back to a non-prerelease
+     * version.
+     *
+     * Therefore, we adopt the following:
+     *
+     * - If the user is managing their own CLI, they can use a prerelease without specifying the
+     * includePrerelease option.
+     * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
+     * includePrerelease config option is set.
+     */
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this.config.includePrerelease;
+
+    if (!semver.satisfies(version, this.versionRange, { includePrerelease })) {
+      return {
+        distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
         version,
       };
     }
-    if (version === undefined) {
-      return {
-        codeQlPath,
-        kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
-      };
-    }
     return {
-      codeQlPath,
+      distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
       version
     };
@@ -100,43 +136,57 @@ export class DistributionManager implements DistributionProvider {
     return result.kind !== FindDistributionResultKind.NoDistribution;
   }
 
+  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    return distribution?.codeQlPath;
+  }
+
   /**
    * Returns the path to a possibly-compatible CodeQL launcher binary, or undefined if a binary not be found.
    */
-  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
+  async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath) {
-      if (!await fs.pathExists(this._config.customCodeQlPath)) {
-        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
-          "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
-          "that a CodeQL executable exists at the specified path or remove the setting.");
+    if (this.config.customCodeQlPath) {
+      if (!await fs.pathExists(this.config.customCodeQlPath)) {
+        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
+          'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
+          'that a CodeQL executable exists at the specified path or remove the setting.');
         return undefined;
       }
 
       // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
       if (
         deprecatedCodeQlLauncherName() &&
-        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        this.config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
         await this.hasNewLauncherName()
       ) {
         warnDeprecatedLauncher();
       }
-      return this._config.customCodeQlPath;
+      return {
+        codeQlPath: this.config.customCodeQlPath,
+        kind: DistributionKind.CustomPathConfig
+      };
     }
 
-    const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionSpecificCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
-      return extensionSpecificCodeQlPath;
+      return {
+        codeQlPath: extensionSpecificCodeQlPath,
+        kind: DistributionKind.ExtensionManaged
+      };
     }
 
     if (process.env.PATH) {
       for (const searchDirectory of process.env.PATH.split(path.delimiter)) {
         const expectedLauncherPath = await getExecutableFromDirectory(searchDirectory);
         if (expectedLauncherPath) {
-          return expectedLauncherPath;
+          return {
+            codeQlPath: expectedLauncherPath,
+            kind: DistributionKind.PathEnvironmentVariable
+          };
         }
       }
-      logger.log("INFO: Could not find CodeQL on path.");
+      logger.log('INFO: Could not find CodeQL on path.');
     }
 
     return undefined;
@@ -150,13 +200,13 @@ export class DistributionManager implements DistributionProvider {
    */
   public async checkForUpdatesToExtensionManagedDistribution(
     minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
-    const codeQlPath = await this.getCodeQlPathWithoutVersionCheck();
-    const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
-    if (codeQlPath !== undefined && codeQlPath !== extensionManagedCodeQlPath) {
+    const distribution = await this.getDistributionWithoutVersionCheck();
+    const extensionManagedCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
       return createInvalidLocationResult();
     }
-    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    const updateCheckResult = await this.updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
     switch (updateCheckResult.kind) {
       case InvocationRateLimiterResultKind.Invoked:
         return updateCheckResult.result;
@@ -172,7 +222,7 @@ export class DistributionManager implements DistributionProvider {
    */
   public installExtensionManagedDistributionRelease(release: Release,
     progressCallback?: helpers.ProgressCallback): Promise<void> {
-    return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
+    return this.extensionSpecificDistributionManager!.installDistributionRelease(release, progressCallback);
   }
 
   public get onDidChangeDistribution(): Event<void> | undefined {
@@ -185,27 +235,27 @@ export class DistributionManager implements DistributionProvider {
    * installation. False otherwise.
    */
   private async hasNewLauncherName(): Promise<boolean> {
-    if (!this._config.customCodeQlPath) {
+    if (!this.config.customCodeQlPath) {
       // not managed externally
       return false;
     }
-    const dir = path.dirname(this._config.customCodeQlPath);
+    const dir = path.dirname(this.config.customCodeQlPath);
     const newLaunderPath = path.join(dir, codeQlLauncherName());
     return await fs.pathExists(newLaunderPath);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
-  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
+  private readonly extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionConstraint: VersionConstraint;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionConstraint: VersionConstraint) {
-    this._extensionContext = extensionContext;
-    this._config = config;
-    this._versionConstraint = versionConstraint;
+  constructor(
+    private readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    private readonly extensionContext: ExtensionContext
+  ) {
+    /**/
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@@ -219,7 +269,7 @@ class ExtensionSpecificDistributionManager {
       try {
         await this.removeDistribution();
       } catch (e) {
-        logger.log("WARNING: Tried to remove corrupted CodeQL CLI at " +
+        logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
           `${this.getDistributionStoragePath()} but encountered an error: ${e}.`);
       }
     }
@@ -268,40 +318,32 @@ class ExtensionSpecificDistributionManager {
         `but encountered an error: ${e}.`);
     }
 
-    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(release.assets[0]);
-    const tmpDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-codeql"));
+    // Filter assets to the unique one that we require.
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
+    const assets = release.assets.filter(asset => asset.name === requiredAssetName);
+    if (assets.length === 0) {
+      throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
+    }
+    if (assets.length > 1) {
+      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+        assets.map(asset => asset.name).join(', '));
+    }
+
+    const assetStream = await this.createReleasesApiConsumer().streamBinaryContentOfAsset(assets[0]);
+    const tmpDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'vscode-codeql'));
 
     try {
-      const archivePath = path.join(tmpDirectory, "distributionDownload.zip");
+      const archivePath = path.join(tmpDirectory, 'distributionDownload.zip');
       const archiveFile = fs.createWriteStream(archivePath);
 
-      const contentLength = assetStream.headers.get("content-length");
-      let numBytesDownloaded = 0;
-
-      if (progressCallback && contentLength !== null) {
-        const totalNumBytes = parseInt(contentLength, 10);
-        const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-        const updateProgress = (): void => {
-          progressCallback({
-            step: numBytesDownloaded,
-            maxStep: totalNumBytes,
-            message: `Downloading CodeQL CLI… [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
-          });
-        };
-
-        // Display the progress straight away rather than waiting for the first chunk.
-        updateProgress();
-
-        assetStream.body.on("data", data => {
-          numBytesDownloaded += data.length;
-          updateProgress();
-        });
-      }
+      const contentLength = assetStream.headers.get('content-length');
+      const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
+      helpers.reportStreamProgress(assetStream.body, 'Downloading CodeQL CLI…', totalNumBytes, progressCallback);
 
       await new Promise((resolve, reject) =>
         assetStream.body.pipe(archiveFile)
-          .on("finish", resolve)
-          .on("error", reject)
+          .on('finish', resolve)
+          .on('error', reject)
       );
 
       await this.bumpDistributionFolderIndex();
@@ -326,31 +368,45 @@ class ExtensionSpecificDistributionManager {
   }
 
   private async getLatestRelease(): Promise<Release> {
-    const release = await this.createReleasesApiConsumer().getLatestRelease(this._versionConstraint, this._config.includePrerelease);
-    if (release.assets.length !== 1) {
-      throw new Error("Release had an unexpected number of assets");
-    }
-    return release;
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
+    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    return this.createReleasesApiConsumer().getLatestRelease(
+      this.versionRange,
+      this.config.includePrerelease,
+      release => {
+        const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
+        if (matchingAssets.length === 0) {
+          // For example, this could be a release with no platform-specific assets.
+          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          return false;
+        }
+        if (matchingAssets.length > 1) {
+          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          return false;
+        }
+        return true;
+      }
+    );
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
-    const ownerName = this._config.ownerName ? this._config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
-    const repositoryName = this._config.repositoryName ? this._config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
-    return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
+    const ownerName = this.config.ownerName ? this.config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
+    const repositoryName = this.config.repositoryName ? this.config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
+    return new ReleasesApiConsumer(ownerName, repositoryName, this.config.personalAccessToken);
   }
 
   private async bumpDistributionFolderIndex(): Promise<void> {
-    const index = this._extensionContext.globalState.get(
+    const index = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    await this._extensionContext.globalState.update(
+    await this.extensionContext.globalState.update(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
   }
 
   private getDistributionStoragePath(): string {
     // Use an empty string for the initial distribution for backwards compatibility.
-    const distributionFolderIndex = this._extensionContext.globalState.get(
-      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || "";
-    return path.join(this._extensionContext.globalStoragePath,
+    const distributionFolderIndex = this.extensionContext.globalState.get(
+      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || '';
+    return path.join(this.extensionContext.globalStoragePath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName + distributionFolderIndex);
   }
 
@@ -360,37 +416,33 @@ class ExtensionSpecificDistributionManager {
   }
 
   private getInstalledRelease(): Release | undefined {
-    return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
+    return this.extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
   }
 
   private async storeInstalledRelease(release: Release | undefined): Promise<void> {
-    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+    await this.extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionContext: ExtensionContext;
-  private readonly _versionConstraint: VersionConstraint;
-
-  private static readonly _currentDistributionFolderBaseName = "distribution";
-  private static readonly _currentDistributionFolderIndexStateKey = "distributionFolderIndex";
-  private static readonly _installedReleaseStateKey = "distributionRelease";
-  private static readonly _codeQlExtractedFolderName = "codeql";
+  private static readonly _currentDistributionFolderBaseName = 'distribution';
+  private static readonly _currentDistributionFolderIndexStateKey = 'distributionFolderIndex';
+  private static readonly _installedReleaseStateKey = 'distributionRelease';
+  private static readonly _codeQlExtractedFolderName = 'codeql';
 }
 
 export class ReleasesApiConsumer {
   constructor(ownerName: string, repoName: string, personalAccessToken?: string) {
     // Specify version of the GitHub API
-    this._defaultHeaders["accept"] = "application/vnd.github.v3+json";
+    this._defaultHeaders['accept'] = 'application/vnd.github.v3+json';
 
     if (personalAccessToken) {
-      this._defaultHeaders["authorization"] = `token ${personalAccessToken}`;
+      this._defaultHeaders['authorization'] = `token ${personalAccessToken}`;
     }
 
     this._ownerName = ownerName;
     this._repoName = repoName;
   }
 
-  public async getLatestRelease(versionConstraint: VersionConstraint, includePrerelease = false): Promise<Release> {
+  public async getLatestRelease(versionRange: semver.Range, includePrerelease = false, additionalCompatibilityCheck?: (release: GithubRelease) => boolean): Promise<Release> {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases`;
     const allReleases: GithubRelease[] = await (await this.makeApiCall(apiPath)).json();
     const compatibleReleases = allReleases.filter(release => {
@@ -398,24 +450,24 @@ export class ReleasesApiConsumer {
         return false;
       }
 
-      const version = tryParseVersionString(release.tag_name);
-      if (version === undefined || !versionConstraint.isVersionCompatible(version)) {
+      const version = semver.parse(release.tag_name);
+      if (version === null || !semver.satisfies(version, versionRange, { includePrerelease })) {
         return false;
       }
 
-      return true;
+      return !additionalCompatibilityCheck || additionalCompatibilityCheck(release);
     });
-    // tryParseVersionString must succeed due to the previous filtering step
+    // Tag names must all be parsable to semvers due to the previous filtering step.
     const latestRelease = compatibleReleases.sort((a, b) => {
-      const versionComparison = versionCompare(tryParseVersionString(b.tag_name)!, tryParseVersionString(a.tag_name)!);
-      if (versionComparison === 0) {
-        return b.created_at.localeCompare(a.created_at);
+      const versionComparison = semver.compare(semver.parse(b.tag_name)!, semver.parse(a.tag_name)!);
+      if (versionComparison !== 0) {
+        return versionComparison;
       }
-      return versionComparison;
+      return b.created_at.localeCompare(a.created_at, 'en-US');
     })[0];
     if (latestRelease === undefined) {
-      throw new Error("No compatible CodeQL CLI releases were found. " +
-        "Please check that the CodeQL extension is up to date.");
+      throw new Error('No compatible CodeQL CLI releases were found. ' +
+        'Please check that the CodeQL extension is up to date.');
     }
     const assets: ReleaseAsset[] = latestRelease.assets.map(asset => {
       return {
@@ -437,7 +489,7 @@ export class ReleasesApiConsumer {
     const apiPath = `/repos/${this._ownerName}/${this._repoName}/releases/assets/${asset.id}`;
 
     return await this.makeApiCall(apiPath, {
-      "accept": "application/octet-stream"
+      'accept': 'application/octet-stream'
     });
   }
 
@@ -447,7 +499,7 @@ export class ReleasesApiConsumer {
 
     if (!response.ok) {
       // Check for rate limiting
-      const rateLimitResetValue = response.headers.get("X-RateLimit-Reset");
+      const rateLimitResetValue = response.headers.get('X-RateLimit-Reset');
       if (response.status === 403 && rateLimitResetValue) {
         const secondsToMillisecondsFactor = 1000;
         const rateLimitResetDate = new Date(parseInt(rateLimitResetValue, 10) * secondsToMillisecondsFactor);
@@ -464,21 +516,21 @@ export class ReleasesApiConsumer {
     redirectCount = 0): Promise<fetch.Response> {
     const response = await fetch.default(requestUrl, {
       headers,
-      redirect: "manual"
+      redirect: 'manual'
     });
 
-    const redirectUrl = response.headers.get("location");
+    const redirectUrl = response.headers.get('location');
     if (isRedirectStatusCode(response.status) && redirectUrl && redirectCount < ReleasesApiConsumer._maxRedirects) {
       const parsedRedirectUrl = url.parse(redirectUrl);
-      if (parsedRedirectUrl.protocol != "https:") {
-        throw new Error("Encountered a non-https redirect, rejecting");
+      if (parsedRedirectUrl.protocol != 'https:') {
+        throw new Error('Encountered a non-https redirect, rejecting');
       }
-      if (parsedRedirectUrl.host != "api.github.com") {
+      if (parsedRedirectUrl.host != 'api.github.com') {
         // Remove authorization header if we are redirected outside of the GitHub API.
         //
         // This is necessary to stream release assets since AWS fails if more than one auth
         // mechanism is provided.
-        delete headers["authorization"];
+        delete headers['authorization'];
       }
       return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
     }
@@ -490,7 +542,7 @@ export class ReleasesApiConsumer {
   private readonly _ownerName: string;
   private readonly _repoName: string;
 
-  private static readonly _apiBase = "https://api.github.com";
+  private static readonly _apiBase = 'https://api.github.com';
   private static readonly _maxRedirects = 20;
 }
 
@@ -511,35 +563,12 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-/**
- * Comparison of semantic versions.
- *
- * Returns a positive number if a is greater than b.
- * Returns 0 if a equals b.
- * Returns a negative number if a is less than b.
- */
-export function versionCompare(a: Version, b: Version): number {
-  if (a.majorVersion !== b.majorVersion) {
-    return a.majorVersion - b.majorVersion;
-  }
-  if (a.minorVersion !== b.minorVersion) {
-    return a.minorVersion - b.minorVersion;
-  }
-  if (a.patchVersion !== b.patchVersion) {
-    return a.patchVersion - b.patchVersion;
-  }
-  if (a.prereleaseVersion !== undefined && b.prereleaseVersion !== undefined) {
-    return a.prereleaseVersion.localeCompare(b.prereleaseVersion);
-  }
-  return 0;
-}
-
-function codeQlLauncherName(): string {
-  return (os.platform() === "win32") ? "codeql.exe" : "codeql";
+export function codeQlLauncherName(): string {
+  return (os.platform() === 'win32') ? 'codeql.exe' : 'codeql';
 }
 
 function deprecatedCodeQlLauncherName(): string | undefined {
-  return (os.platform() === "win32") ? "codeql.cmd" : undefined;
+  return (os.platform() === 'win32') ? 'codeql.cmd' : undefined;
 }
 
 function isRedirectStatusCode(statusCode: number): boolean {
@@ -550,6 +579,17 @@ function isRedirectStatusCode(statusCode: number): boolean {
  * Types and helper functions relating to those types.
  */
 
+export enum DistributionKind {
+  CustomPathConfig,
+  ExtensionManaged,
+  PathEnvironmentVariable
+}
+
+export interface Distribution {
+  codeQlPath: string;
+  kind: DistributionKind;
+}
+
 export enum FindDistributionResultKind {
   CompatibleDistribution,
   UnknownCompatibilityDistribution,
@@ -563,21 +603,27 @@ export type FindDistributionResult =
   | IncompatibleDistributionResult
   | NoDistributionResult;
 
-interface CompatibleDistributionResult {
-  codeQlPath: string;
-  kind: FindDistributionResultKind.CompatibleDistribution;
-  version: Version;
+/**
+ * A result representing a distribution of the CodeQL CLI that may or may not be compatible with
+ * the extension.
+ */
+interface DistributionResult {
+  distribution: Distribution;
+  kind: FindDistributionResultKind;
 }
 
-interface UnknownCompatibilityDistributionResult {
-  codeQlPath: string;
+interface CompatibleDistributionResult extends DistributionResult {
+  kind: FindDistributionResultKind.CompatibleDistribution;
+  version: semver.SemVer;
+}
+
+interface UnknownCompatibilityDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.UnknownCompatibilityDistribution;
 }
 
-interface IncompatibleDistributionResult {
-  codeQlPath: string;
+interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: Version;
+  version: semver.SemVer;
 }
 
 interface NoDistributionResult {
@@ -655,7 +701,7 @@ export async function getExecutableFromDirectory(directory: string, warnWhenNotF
   }
   if (warnWhenNotFound) {
     logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
-      "Will try PATH.");
+      'Will try PATH.');
   }
   return undefined;
 }
@@ -717,7 +763,7 @@ export interface GithubRelease {
   assets: GithubReleaseAsset[];
 
   /**
-   * The creation date of the release on GitHub.
+   * The creation date of the release on GitHub, in ISO 8601 format.
    */
   created_at: string;
 
@@ -762,11 +808,6 @@ export interface GithubReleaseAsset {
   size: number;
 }
 
-interface VersionConstraint {
-  description: string;
-  isVersionCompatible(version: Version): boolean;
-}
-
 export class GithubApiError extends Error {
   constructor(public status: number, public body: string) {
     super(`API call failed with status code ${status}, body: ${body}`);

extensions/ql-vscode/src/extension.ts

@@ -1,26 +1,64 @@
-import { commands, Disposable, ExtensionContext, extensions, languages, ProgressLocation, ProgressOptions, Uri, window as Window } from 'vscode';
+import {
+  CancellationToken,
+  commands,
+  Disposable,
+  ExtensionContext,
+  extensions,
+  languages,
+  ProgressLocation,
+  ProgressOptions,
+  Uri,
+  window as Window,
+  env,
+  window
+} from 'vscode';
 import { LanguageClient } from 'vscode-languageclient';
+import * as path from 'path';
 import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
+
+import { AstViewer } from './astViewer';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
 import { CodeQLCliServer } from './cli';
-import { DistributionConfigListener, QueryHistoryConfigListener, QueryServerConfigListener } from './config';
+import {
+  CliConfigListener,
+  DistributionConfigListener,
+  MAX_QUERIES,
+  QueryHistoryConfigListener,
+  QueryServerConfigListener
+} from './config';
 import * as languageSupport from './languageSupport';
 import { DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
-import { TemplateQueryDefinitionProvider, TemplateQueryReferenceProvider } from './definitions';
-import { DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT, DistributionManager, DistributionUpdateCheckResultKind, FindDistributionResult, FindDistributionResultKind, GithubApiError, GithubRateLimitedError } from './distribution';
+import {
+  TemplateQueryDefinitionProvider,
+  TemplateQueryReferenceProvider,
+  TemplatePrintAstProvider
+} from './contextual/templateProvider';
+import {
+  DEFAULT_DISTRIBUTION_VERSION_RANGE,
+  DistributionKind,
+  DistributionManager,
+  DistributionUpdateCheckResultKind,
+  FindDistributionResult,
+  FindDistributionResultKind,
+  GithubApiError,
+  GithubRateLimitedError
+} from './distribution';
 import * as helpers from './helpers';
-import { assertNever } from './helpers-pure';
+import { assertNever } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
-import { InterfaceManager, WebviewReveal } from './interface';
+import { InterfaceManager } from './interface';
+import { WebviewReveal } from './interface-utils';
 import { ideServerLogger, logger, queryServerLogger } from './logging';
 import { QueryHistoryManager } from './query-history';
 import { CompletedQuery } from './query-results';
 import * as qsClient from './queryserver-client';
 import { displayQuickQuery } from './quick-query';
-import { compileAndRunQueryAgainstDatabase, tmpDirDisposal, UserCancellationException } from './run-queries';
+import { compileAndRunQueryAgainstDatabase, tmpDirDisposal } from './run-queries';
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
+import { CompareInterfaceManager } from './compare/compare-interface';
+import { gatherQlFiles } from './pure/files';
 
 /**
  * extension.ts
@@ -55,7 +93,7 @@ let isInstallingOrUpdatingDistribution = false;
  *
  * @param excludedCommands List of commands for which we should not register error stubs.
  */
-function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => void): void {
+function registerErrorStubs(excludedCommands: string[], stubGenerator: (command: string) => () => Promise<void>): void {
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
@@ -70,26 +108,51 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 
   stubbedCommands.forEach(command => {
     if (excludedCommands.indexOf(command) === -1) {
-      errorStubs.push(commands.registerCommand(command, stubGenerator(command)));
+      errorStubs.push(helpers.commandRunner(command, stubGenerator(command)));
     }
   });
 }
 
-export async function activate(ctx: ExtensionContext): Promise<void> {
+/**
+ * The publicly available interface for this extension. This is to
+ * be used in our tests.
+ */
+export interface CodeQLExtensionInterface {
+  readonly ctx: ExtensionContext;
+  readonly cliServer: CodeQLCliServer;
+  readonly qs: qsClient.QueryServerClient;
+  readonly distributionManager: DistributionManager;
+  readonly databaseManager: DatabaseManager;
+  readonly databaseUI: DatabaseUI;
+  readonly dispose: () => void;
+}
+
+/**
+ * Returns the CodeQLExtensionInterface, or an empty object if the interface is not
+ * available afer activation is complete. This will happen if there is no cli
+ * installed when the extension starts. Downloading and installing the cli
+ * will happen at a later time.
+ *
+ * @param ctx The extension context
+ *
+ * @returns CodeQLExtensionInterface
+ */
+export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | {}> {
   logger.log('Starting CodeQL extension');
 
+  const distributionConfigListener = new DistributionConfigListener();
   initializeLogging(ctx);
   languageSupport.install();
 
-  const distributionConfigListener = new DistributionConfigListener();
   ctx.subscriptions.push(distributionConfigListener);
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, DEFAULT_DISTRIBUTION_VERSION_CONSTRAINT);
+  const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
+  const distributionManager = new DistributionManager(distributionConfigListener, codeQlVersionRange, ctx);
 
-  const shouldUpdateOnNextActivationKey = "shouldUpdateOnNextActivation";
+  const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
-  registerErrorStubs([checkForUpdatesCommand], command => () => {
+  registerErrorStubs([checkForUpdatesCommand], command => (async () => {
     helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
-  });
+  }));
 
   interface DistributionUpdateConfig {
     isUserInitiated: boolean;
@@ -108,7 +171,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
     switch (result.kind) {
       case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
-        logger.log("Didn't perform CodeQL CLI update check since a check was already performed within the previous " +
+        logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
           `${minSecondsSinceLastUpdateCheck} seconds.`);
         break;
       case DistributionUpdateCheckResultKind.AlreadyUpToDate:
@@ -127,10 +190,10 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
           }
         } else {
           const progressOptions: ProgressOptions = {
-            location: ProgressLocation.Notification,
             title: progressTitle,
-            cancellable: false,
+            location: ProgressLocation.Notification,
           };
+
           await helpers.withProgress(progressOptions, progress =>
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
@@ -145,16 +208,16 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
   async function installOrUpdateDistribution(config: DistributionUpdateConfig): Promise<void> {
     if (isInstallingOrUpdatingDistribution) {
-      throw new Error("Already installing or updating CodeQL CLI");
+      throw new Error('Already installing or updating CodeQL CLI');
     }
     isInstallingOrUpdatingDistribution = true;
     const codeQlInstalled = await distributionManager.getCodeQlPathWithoutVersionCheck() !== undefined;
     const willUpdateCodeQl = ctx.globalState.get(shouldUpdateOnNextActivationKey);
     const messageText = willUpdateCodeQl
-      ? "Updating CodeQL CLI"
+      ? 'Updating CodeQL CLI'
       : codeQlInstalled
-        ? "Checking for updates to CodeQL CLI"
-        : "Installing CodeQL CLI";
+        ? 'Checking for updates to CodeQL CLI'
+        : 'Installing CodeQL CLI';
 
     try {
       await installOrUpdateDistributionWithProgressTitle(messageText, config);
@@ -163,12 +226,12 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
       // or updating the distribution.
       const alertFunction = (codeQlInstalled && !config.isUserInitiated) ?
         helpers.showAndLogWarningMessage : helpers.showAndLogErrorMessage;
-      const taskDescription = (willUpdateCodeQl ? "update" :
-        codeQlInstalled ? "check for updates to" : "install") + " CodeQL CLI";
+      const taskDescription = (willUpdateCodeQl ? 'update' :
+        codeQlInstalled ? 'check for updates to' : 'install') + ' CodeQL CLI';
 
       if (e instanceof GithubRateLimitedError) {
         alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
-          `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString()}.`);
+          `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString(env.language)}.`);
       } else if (e instanceof GithubApiError) {
         alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
       }
@@ -182,17 +245,31 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.rawString})`);
+        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
-      case FindDistributionResultKind.IncompatibleDistribution:
-        helpers.showAndLogWarningMessage("The current version of the CodeQL CLI is incompatible with this extension.");
+      case FindDistributionResultKind.IncompatibleDistribution: {
+        const fixGuidanceMessage = (() => {
+          switch (result.distribution.kind) {
+            case DistributionKind.ExtensionManaged:
+              return 'Please update the CodeQL CLI by running the "CodeQL: Check for CLI Updates" command.';
+            case DistributionKind.CustomPathConfig:
+              return `Please update the \"CodeQL CLI Executable Path\" setting to point to a CLI in the version range ${codeQlVersionRange}.`;
+            case DistributionKind.PathEnvironmentVariable:
+              return `Please update the CodeQL CLI on your PATH to a version compatible with ${codeQlVersionRange}, or ` +
+                `set the \"CodeQL CLI Executable Path\" setting to the path of a CLI version compatible with ${codeQlVersionRange}.`;
+          }
+        })();
+
+        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
+          'is incompatible with this extension. ' + fixGuidanceMessage);
         break;
+      }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
-        helpers.showAndLogWarningMessage("Compatibility with the configured CodeQL CLI could not be determined. " +
-          "You may experience problems using the extension.");
+        helpers.showAndLogWarningMessage('Compatibility with the configured CodeQL CLI could not be determined. ' +
+          'You may experience problems using the extension.');
         break;
       case FindDistributionResultKind.NoDistribution:
-        helpers.showAndLogErrorMessage("The CodeQL CLI could not be found.");
+        helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
         break;
       default:
         assertNever(result);
@@ -200,22 +277,22 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
+  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<CodeQLExtensionInterface | {}> {
     await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
-
+    let extensionInterface: CodeQLExtensionInterface | {} = {};
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(ctx, distributionManager);
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
-        const installActionName = "Install CodeQL CLI";
+        const installActionName = 'Install CodeQL CLI';
         const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
           items: [installActionName]
         });
         if (chosenAction === installActionName) {
-          installOrUpdateThenTryActivate({
+          await installOrUpdateThenTryActivate({
             isUserInitiated: true,
             shouldDisplayMessageWhenNoUpdates: false,
             allowAutoUpdating: true
@@ -223,20 +300,21 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
         }
       });
     }
+    return extensionInterface;
   }
 
-  ctx.subscriptions.push(distributionConfigListener.onDidChangeDistributionConfiguration(() => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(distributionConfigListener.onDidChangeConfiguration(() => installOrUpdateThenTryActivate({
     isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: false,
     allowAutoUpdating: true
   })));
-  ctx.subscriptions.push(commands.registerCommand(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(helpers.commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
     isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: true,
     allowAutoUpdating: true
   })));
 
-  await installOrUpdateThenTryActivate({
+  return await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
 
@@ -246,90 +324,162 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
   });
 }
 
-async function activateWithInstalledDistribution(ctx: ExtensionContext, distributionManager: DistributionManager): Promise<void> {
+async function activateWithInstalledDistribution(
+  ctx: ExtensionContext,
+  distributionManager: DistributionManager
+): Promise<CodeQLExtensionInterface> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
-  errorStubs.forEach(stub => stub.dispose());
+  errorStubs.forEach((stub) => stub.dispose());
 
   logger.log('Initializing configuration listener...');
-  const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(distributionManager);
+  const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(
+    distributionManager
+  );
   ctx.subscriptions.push(qlConfigurationListener);
 
   logger.log('Initializing CodeQL cli server...');
-  const cliServer = new CodeQLCliServer(distributionManager, logger);
+  const cliServer = new CodeQLCliServer(
+    distributionManager,
+    new CliConfigListener(),
+    logger
+  );
   ctx.subscriptions.push(cliServer);
 
   logger.log('Initializing query server client.');
-  const qs = new qsClient.QueryServerClient(qlConfigurationListener, cliServer, {
-    logger: queryServerLogger,
-  }, task => Window.withProgress({ title: 'CodeQL query server', location: ProgressLocation.Window }, task));
+  const qs = new qsClient.QueryServerClient(
+    qlConfigurationListener,
+    cliServer,
+    {
+      logger: queryServerLogger,
+    },
+    (task) =>
+      Window.withProgress(
+        { title: 'CodeQL query server', location: ProgressLocation.Window },
+        task
+      )
+  );
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
   logger.log('Initializing database manager.');
-  const dbm = new DatabaseManager(ctx, qlConfigurationListener, logger);
+  const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
   ctx.subscriptions.push(dbm);
   logger.log('Initializing database panel.');
-  const databaseUI = new DatabaseUI(ctx, cliServer, dbm, qs, getContextStoragePath(ctx));
+  const databaseUI = new DatabaseUI(
+    cliServer,
+    dbm,
+    qs,
+    getContextStoragePath(ctx),
+    ctx.extensionPath
+  );
+  databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
   logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  ctx.subscriptions.push(queryHistoryConfigurationListener);
+  const showResults = async (item: CompletedQuery) =>
+    showResultsForCompletedQuery(item, WebviewReveal.Forced);
+
   const qhm = new QueryHistoryManager(
-    ctx,
+    qs,
+    ctx.extensionPath,
     queryHistoryConfigurationListener,
-    async item => showResultsForCompletedQuery(item, WebviewReveal.Forced)
+    showResults,
+    async (from: CompletedQuery, to: CompletedQuery) =>
+      showResultsForComparison(from, to),
   );
+  ctx.subscriptions.push(qhm);
   logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
+
+  logger.log('Initializing compare panel interface.');
+  const cmpm = new CompareInterfaceManager(
+    ctx,
+    dbm,
+    cliServer,
+    queryServerLogger,
+    showResults
+  );
+  ctx.subscriptions.push(cmpm);
+
   logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
-  async function showResultsForCompletedQuery(query: CompletedQuery, forceReveal: WebviewReveal): Promise<void> {
+  async function showResultsForComparison(
+    from: CompletedQuery,
+    to: CompletedQuery
+  ): Promise<void> {
+    try {
+      await cmpm.showResults(from, to);
+    } catch (e) {
+      helpers.showAndLogErrorMessage(e.message);
+    }
+  }
+
+  async function showResultsForCompletedQuery(
+    query: CompletedQuery,
+    forceReveal: WebviewReveal
+  ): Promise<void> {
     await intm.showResults(query, forceReveal, false);
   }
 
-  async function compileAndRunQuery(quickEval: boolean, selectedQuery: Uri | undefined): Promise<void> {
+  async function compileAndRunQuery(
+    quickEval: boolean,
+    selectedQuery: Uri | undefined,
+    progress: helpers.ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> {
     if (qs !== undefined) {
-      try {
-        const dbItem = await databaseUI.getDatabaseItem();
-        if (dbItem === undefined) {
-          throw new Error('Can\'t run query without a selected database');
-        }
-        const info = await compileAndRunQueryAgainstDatabase(cliServer, qs, dbItem, quickEval, selectedQuery);
-        const item = qhm.addQuery(info);
-        await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      } catch (e) {
-        if (e instanceof UserCancellationException) {
-          helpers.showAndLogWarningMessage(e.message);
-        } else if (e instanceof Error) {
-          helpers.showAndLogErrorMessage(e.message);
-        } else {
-          throw e;
-        }
+      const dbItem = await databaseUI.getDatabaseItem(progress, token);
+      if (dbItem === undefined) {
+        throw new Error('Can\'t run query without a selected database');
       }
+      const info = await compileAndRunQueryAgainstDatabase(
+        cliServer,
+        qs,
+        dbItem,
+        quickEval,
+        selectedQuery,
+        progress,
+        token
+      );
+      const item = qhm.addQuery(info);
+      await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
+      // The call to showResults potentially creates SARIF file;
+      // Update the tree item context value to allow viewing that
+      // SARIF file from context menu.
+      await qhm.refreshTreeView(item);
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
   logger.log('Initializing CodeQL language server.');
-  const client = new LanguageClient('CodeQL Language Server', () => spawnIdeServer(qlConfigurationListener), {
-    documentSelector: [
-      { language: 'ql', scheme: 'file' },
-      { language: 'yaml', scheme: 'file', pattern: '**/qlpack.yml' }
-    ],
-    synchronize: {
-      configurationSection: 'codeQL'
+  const client = new LanguageClient(
+    'CodeQL Language Server',
+    () => spawnIdeServer(qlConfigurationListener),
+    {
+      documentSelector: [
+        { language: 'ql', scheme: 'file' },
+        { language: 'yaml', scheme: 'file', pattern: '**/qlpack.yml' },
+      ],
+      synchronize: {
+        configurationSection: 'codeQL',
+      },
+      // Ensure that language server exceptions are logged to the same channel as its output.
+      outputChannel: ideServerLogger.outputChannel,
     },
-    // Ensure that language server exceptions are logged to the same channel as its output.
-    outputChannel: ideServerLogger.outputChannel
-  }, true);
+    true
+  );
 
   logger.log('Initializing QLTest interface.');
-  const testExplorerExtension = extensions.getExtension<TestHub>(testExplorerExtensionId);
+  const testExplorerExtension = extensions.getExtension<TestHub>(
+    testExplorerExtensionId
+  );
   if (testExplorerExtension) {
     const testHub = testExplorerExtension.exports;
     const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
@@ -340,17 +490,154 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
   }
 
   logger.log('Registering top-level command palette commands.');
-  ctx.subscriptions.push(commands.registerCommand('codeQL.runQuery', async (uri: Uri | undefined) => await compileAndRunQuery(false, uri)));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.quickEval', async (uri: Uri | undefined) => await compileAndRunQuery(true, uri)));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.quickQuery', async () => displayQuickQuery(ctx, cliServer, databaseUI)));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.restartQueryServer', async () => {
-    await qs.restartQueryServer();
-    helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', { outputLogger: queryServerLogger });
-  }));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseFolder', () => databaseUI.handleChooseDatabaseFolder()));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseArchive', () => databaseUI.handleChooseDatabaseArchive()));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseLgtm', () => databaseUI.handleChooseDatabaseLgtm()));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseInternet', () => databaseUI.handleChooseDatabaseInternet()));
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress(
+      'codeQL.runQuery',
+      async (
+        progress: helpers.ProgressCallback,
+        token: CancellationToken,
+        uri: Uri | undefined
+      ) => await compileAndRunQuery(false, uri, progress, token),
+      {
+        title: 'Running query',
+        cancellable: true
+      }
+    )
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress(
+      'codeQL.runQueries',
+      async (
+        progress: helpers.ProgressCallback,
+        token: CancellationToken,
+        _: Uri | undefined,
+        multi: Uri[]
+      ) => {
+        const maxQueryCount = MAX_QUERIES.getValue() as number;
+        const [files, dirFound] = await gatherQlFiles(multi.map(uri => uri.fsPath));
+        if (files.length > maxQueryCount) {
+          throw new Error(`You tried to run ${files.length} queries, but the maximum is ${maxQueryCount}. Try selecting fewer queries or changing the 'codeQL.runningQueries.maxQueries' setting.`);
+        }
+        // warn user and display selected files when a directory is selected because some ql
+        // files may be hidden from the user.
+        if (dirFound) {
+          const fileString = files.map(file => path.basename(file)).join(', ');
+          const res = await helpers.showBinaryChoiceDialog(
+            `You are about to run ${files.length} queries: ${fileString} Do you want to continue?`
+          );
+          if (!res) {
+            return;
+          }
+        }
+        const queryUris = files.map(path => Uri.parse(`file:${path}`, true));
+
+        // Use a wrapped progress so that messages appear with the queries remaining in it.
+        let queriesRemaining = queryUris.length;
+        function wrappedProgress(update: helpers.ProgressUpdate) {
+          const message = queriesRemaining > 1
+            ? `${queriesRemaining} remaining. ${update.message}`
+            : update.message;
+          progress({
+            ...update,
+            message
+          });
+        }
+
+        if (queryUris.length > 1) {
+          // Try to upgrade the current database before running any queries
+          // so that the user isn't confronted with multiple upgrade
+          // requests for each query to run.
+          // Only do it if running multiple queries since this check is
+          // performed on each query run anyway.
+          await databaseUI.tryUpgradeCurrentDatabase(progress, token);
+        }
+
+        wrappedProgress({
+          maxStep: queryUris.length,
+          step: queryUris.length - queriesRemaining,
+          message: ''
+        });
+
+        await Promise.all(queryUris.map(async uri =>
+          compileAndRunQuery(false, uri, wrappedProgress, token)
+            .then(() => queriesRemaining--)
+        ));
+      },
+      {
+        title: 'Running queries',
+        cancellable: true
+      })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress(
+      'codeQL.quickEval',
+      async (
+        progress: helpers.ProgressCallback,
+        token: CancellationToken,
+        uri: Uri | undefined
+      ) => await compileAndRunQuery(true, uri, progress, token),
+      {
+        title: 'Running query',
+        cancellable: true
+      })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress('codeQL.quickQuery', async (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      displayQuickQuery(ctx, cliServer, databaseUI, progress, token),
+      {
+        title: 'Run Quick Query'
+      }
+    )
+  );
+
+  ctx.subscriptions.push(
+    helpers.commandRunner('codeQL.restartQueryServer', async () => {
+      await qs.restartQueryServer();
+      helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
+        outputLogger: queryServerLogger,
+      });
+    })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunner('codeQL.chooseDatabaseFolder', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseFolder(progress, token)
+    )
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunner('codeQL.chooseDatabaseArchive', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseArchive(progress, token)
+    )
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseLgtm(progress, token),
+      {
+        title: 'Adding database from LGTM',
+      })
+  );
+  ctx.subscriptions.push(
+    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
+      progress: helpers.ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseInternet(progress, token),
+
+      {
+        title: 'Adding database from URL',
+      })
+  );
 
   logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
@@ -361,12 +648,43 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
     new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
   );
+
   languages.registerReferenceProvider(
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
     new TemplateQueryReferenceProvider(cliServer, qs, dbm)
   );
 
+  const astViewer = new AstViewer();
+  ctx.subscriptions.push(astViewer);
+  ctx.subscriptions.push(helpers.commandRunnerWithProgress('codeQL.viewAst', async (
+    progress: helpers.ProgressCallback,
+    token: CancellationToken
+  ) => {
+    const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)
+      .provideAst(window.activeTextEditor?.document);
+    if (ast) {
+      astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
+    }
+  }, {
+    cancellable: true,
+    title: 'Calculate AST'
+  }));
+
+  commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
+
   logger.log('Successfully finished extension initialization.');
+
+  return {
+    ctx,
+    cliServer,
+    qs,
+    distributionManager,
+    databaseManager: dbm,
+    databaseUI,
+    dispose: () => {
+      ctx.subscriptions.forEach(d => d.dispose());
+    }
+  };
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {

extensions/ql-vscode/src/helpers.ts

@@ -2,10 +2,28 @@ import * as fs from 'fs-extra';
 import * as glob from 'glob-promise';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
-import { CancellationToken, ExtensionContext, ProgressOptions, window as Window, workspace } from 'vscode';
+import {
+  CancellationToken,
+  ExtensionContext,
+  ProgressOptions,
+  window as Window,
+  workspace,
+  commands,
+  Disposable,
+  ProgressLocation
+} from 'vscode';
 import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
-import { QueryInfo } from './run-queries';
+
+export class UserCancellationException extends Error {
+  /**
+   * @param message The error message
+   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
+   */
+  constructor(message?: string, public readonly silent = false) {
+    super(message);
+  }
+}
 
 export interface ProgressUpdate {
   /**
@@ -24,18 +42,57 @@ export interface ProgressUpdate {
 
 export type ProgressCallback = (p: ProgressUpdate) => void;
 
+/**
+ * A task that handles command invocations from `commandRunner`
+ * and includes a progress monitor.
+ *
+ *
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `ProgressTask` instance.
+ *
+ * @param progress a progress handler function. Call this
+ * function with a `ProgressUpdate` instance in order to
+ * denote some progress being achieved on this task.
+ * @param token a cencellation token
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+export type ProgressTask<R> = (
+  progress: ProgressCallback,
+  token: CancellationToken,
+  ...args: any[]
+) => Thenable<R>;
+
+/**
+ * A task that handles command invocations from `commandRunner`.
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `NoProgressTask` instance.
+ *
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+type NoProgressTask = ((...args: any[]) => Promise<any>);
+
 /**
  * This mediates between the kind of progress callbacks we want to
  * write (where we *set* current progress position and give
  * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%)
+ * (which increment progress by a certain amount out of 100%).
+ *
+ * Where possible, the `commandRunner` function below should be used
+ * instead of this function. The commandRunner is meant for wrapping
+ * top-level commands and provides error handling and other support
+ * automatically.
+ *
+ * Only use this function if you need a progress monitor and the
+ * control flow does not always come from a command (eg- during
+ * extension activation, or from an internal language server
+ * request).
  */
 export function withProgress<R>(
   options: ProgressOptions,
-  task: (
-    progress: (p: ProgressUpdate) => void,
-    token: CancellationToken
-  ) => Thenable<R>
+  task: ProgressTask<R>,
+  ...args: any[]
 ): Thenable<R> {
   let progressAchieved = 0;
   return Window.withProgress(options,
@@ -45,10 +102,82 @@ export function withProgress<R>(
         const increment = 100 * (step - progressAchieved) / maxStep;
         progressAchieved = step;
         progress.report({ message, increment });
-      }, token);
+      }, token, ...args);
     });
 }
 
+/**
+ * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
+ *
+ * In this variant of the command runner, no progress monitor is used.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task.
+ */
+export function commandRunner(
+  commandId: string,
+  task: NoProgressTask,
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    try {
+      return await task(...args);
+    } catch (e) {
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          logger.log(errorMessage);
+        } else {
+          showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        showAndLogErrorMessage(errorMessage);
+      }
+      return undefined;
+    }
+  });
+}
+
+/**
+ * A generic wrapper for command registration.  This wrapper adds uniform error handling,
+ * progress monitoring, and cancellation for commands.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task after the progress callback
+ * and cancellation token.
+ * @param progressOptions Progress options to be sent to the progress monitor.
+ */
+export function commandRunnerWithProgress<R>(
+  commandId: string,
+  task: ProgressTask<R>,
+  progressOptions: Partial<ProgressOptions>
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const progressOptionsWithDefaults = {
+      location: ProgressLocation.Notification,
+      ...progressOptions
+    };
+    try {
+      return await withProgress(progressOptionsWithDefaults, task, ...args);
+    } catch (e) {
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          logger.log(errorMessage);
+        } else {
+          showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        showAndLogErrorMessage(errorMessage);
+      }
+      return undefined;
+    }
+  });
+}
+
 /**
  * Show an error message and log it to the console
  *
@@ -118,7 +247,7 @@ export async function showBinaryChoiceDialog(message: string): Promise<boolean>
   const yesItem = { title: 'Yes', isCloseAffordance: false };
   const noItem = { title: 'No', isCloseAffordance: true };
   const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
-  return chosenItem === yesItem;
+  return chosenItem?.title === yesItem.title;
 }
 
 /**
@@ -139,30 +268,12 @@ export function getOnDiskWorkspaceFolders() {
   const workspaceFolders = workspace.workspaceFolders || [];
   const diskWorkspaceFolders: string[] = [];
   for (const workspaceFolder of workspaceFolders) {
-    if (workspaceFolder.uri.scheme === "file")
+    if (workspaceFolder.uri.scheme === 'file')
       diskWorkspaceFolders.push(workspaceFolder.uri.fsPath);
   }
   return diskWorkspaceFolders;
 }
 
-/**
- * Gets a human-readable name for an evaluated query.
- * Uses metadata if it exists, and defaults to the query file name.
- */
-export function getQueryName(query: QueryInfo) {
-  // Queries run through quick evaluation are not usually the entire query file.
-  // Label them differently and include the line numbers.
-  if (query.quickEvalPosition !== undefined) {
-    const { line, endLine, fileName } = query.quickEvalPosition;
-    const lineInfo = line === endLine ? `${line}` : `${line}-${endLine}`;
-    return `Quick evaluation of ${path.basename(fileName)}:${lineInfo}`;
-  } else if (query.metadata && query.metadata.name) {
-    return query.metadata.name;
-  } else {
-    return path.basename(query.program.queryPath);
-  }
-}
-
 /**
  * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
  * the last invocation of that function.
@@ -213,7 +324,7 @@ export class InvocationRateLimiter<T> {
   private readonly _func: () => Promise<T>;
   private readonly _funcIdentifier: string;
 
-  private static readonly _invocationRateLimiterPrefix = "invocationRateLimiter_lastInvocationDate_";
+  private static readonly _invocationRateLimiterPrefix = 'invocationRateLimiter_lastInvocationDate_';
 }
 
 export enum InvocationRateLimiterResultKind {
@@ -251,12 +362,6 @@ function createRateLimitedResult(): RateLimitedResult {
   };
 }
 
-
-export type DatasetFolderInfo = {
-  dbscheme: string;
-  qlpack: string;
-}
-
 export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
   const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
   const packs: { packDir: string | undefined; packName: string }[] =
@@ -275,7 +380,7 @@ export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemeP
     });
   for (const { packDir, packName } of packs) {
     if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8'));
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme: string };
       if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
         return packName;
       }
@@ -284,7 +389,7 @@ export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemeP
   throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
 }
 
-export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise<DatasetFolderInfo> {
+export async function getPrimaryDbscheme(datasetFolder: string): Promise<string> {
   const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'));
 
   if (dbschemes.length < 1) {
@@ -293,12 +398,11 @@ export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFo
 
   dbschemes.sort();
   const dbscheme = dbschemes[0];
+
   if (dbschemes.length > 1) {
     Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
   }
-
-  const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
-  return { dbscheme, qlpack };
+  return dbscheme;
 }
 
 /**
@@ -357,3 +461,117 @@ export class CachedOperation<U> {
     }
   }
 }
+
+
+
+/**
+ * The following functions al heuristically determine metadata about databases.
+ */
+
+/**
+ * Note that this heuristic is only being used for backwards compatibility with
+ * CLI versions before the langauge name was introduced to dbInfo. Features
+ * that do not require backwards compatibility should call
+ * `cli.CodeQLCliServer.resolveDatabase` and use the first entry in the
+ * `languages` property.
+ *
+ * @see cli.CodeQLCliServer.supportsLanguageName
+ * @see cli.CodeQLCliServer.resolveDatabase
+ */
+const dbSchemeToLanguage = {
+  'semmlecode.javascript.dbscheme': 'javascript',
+  'semmlecode.cpp.dbscheme': 'cpp',
+  'semmlecode.dbscheme': 'java',
+  'semmlecode.python.dbscheme': 'python',
+  'semmlecode.csharp.dbscheme': 'csharp',
+  'go.dbscheme': 'go'
+};
+
+/**
+ * Returns the initial contents for an empty query, based on the language of the selected
+ * databse.
+ *
+ * First try to use the given language name. If that doesn't exist, try to infer it based on
+ * dbscheme. Otherwise return no import statement.
+ *
+ * @param language the database language or empty string if unknown
+ * @param dbscheme path to the dbscheme file
+ *
+ * @returns an import and empty select statement appropriate for the selected language
+ */
+export function getInitialQueryContents(language: string, dbscheme: string) {
+  if (!language) {
+    const dbschemeBase = path.basename(dbscheme) as keyof typeof dbSchemeToLanguage;
+    language = dbSchemeToLanguage[dbschemeBase];
+  }
+
+  return language
+    ? `import ${language}\n\nselect ""`
+    : 'select ""';
+}
+
+/**
+ * Heuristically determines if the directory passed in corresponds
+ * to a database root.
+ *
+ * @param maybeRoot
+ */
+export async function isLikelyDatabaseRoot(maybeRoot: string) {
+  const [a, b, c] = (await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    fs.pathExists(path.join(maybeRoot, '.dbinfo')),
+    fs.pathExists(path.join(maybeRoot, 'codeql-database.yml')),
+
+    // they *must* have a db-{language} folder
+    glob('db-*/', { cwd: maybeRoot })
+  ]));
+
+  return !!((a || b) && c);
+}
+
+export function isLikelyDbLanguageFolder(dbPath: string) {
+  return !!path.basename(dbPath).startsWith('db-');
+}
+
+
+/**
+ * Displays a progress monitor that indicates how much progess has been made
+ * reading from a stream.
+ *
+ * @param readable The stream to read progress from
+ * @param messagePrefix A prefix for displaying the message
+ * @param totalNumBytes Total number of bytes in this stream
+ * @param progress The progress callback used to set messages
+ */
+export function reportStreamProgress(
+  readable: NodeJS.ReadableStream,
+  messagePrefix: string,
+  totalNumBytes?: number,
+  progress?: ProgressCallback
+) {
+  if (progress && totalNumBytes) {
+    let numBytesDownloaded = 0;
+    const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+    const updateProgress = () => {
+      progress({
+        step: numBytesDownloaded,
+        maxStep: totalNumBytes,
+        message: `${messagePrefix} [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
+      });
+    };
+
+    // Display the progress straight away rather than waiting for the first chunk.
+    updateProgress();
+
+    readable.on('data', data => {
+      numBytesDownloaded += data.length;
+      updateProgress();
+    });
+  } else if (progress) {
+    progress({
+      step: 1,
+      maxStep: 2,
+      message: `${messagePrefix} (Size unknown)`,
+    });
+  }
+}

extensions/ql-vscode/src/ide-server.ts

@@ -11,11 +11,15 @@ import { ideServerLogger } from './logging';
 /** Starts a new CodeQL language server process, sending progress messages to the status bar. */
 export async function spawnIdeServer(config: QueryServerConfig): Promise<StreamInfo> {
   return window.withProgress({ title: 'CodeQL language server', location: ProgressLocation.Window }, async (progressReporter, _) => {
+    const args = ['--check-errors', 'ON_CHANGE'];
+    if (cli.shouldDebugIdeServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9009,server=y,suspend=n,quiet=y');
+    }
     const child = cli.spawnServer(
       config.codeQlPath,
       'CodeQL language server',
       ['execute', 'language-server'],
-      ['--check-errors', 'ON_CHANGE'],
+      args,
       ideServerLogger,
       data => ideServerLogger.log(data.toString(), { trailingNewline: false }),
       data => ideServerLogger.log(data.toString(), { trailingNewline: false }),

extensions/ql-vscode/src/interface-types.ts

@@ -1,159 +0,0 @@
-import * as sarif from 'sarif';
-import { ResolvableLocationValue } from 'semmle-bqrs';
-import { RawResultSet } from './adapt';
-
-/**
- * Only ever show this many results per run in interpreted results.
- */
-export const INTERPRETED_RESULTS_PER_RUN_LIMIT = 100;
-
-/**
- * Only ever show this many rows in a raw result table.
- */
-export const RAW_RESULTS_LIMIT = 10000;
-
-export interface DatabaseInfo {
-  name: string;
-  databaseUri: string;
-}
-
-/** Arbitrary query metadata */
-export interface QueryMetadata {
-  name?: string;
-  description?: string;
-  id?: string;
-  kind?: string;
-}
-
-export interface PreviousExecution {
-  queryName: string;
-  time: string;
-  databaseName: string;
-  durationSeconds: number;
-}
-
-export interface Interpretation {
-  sourceLocationPrefix: string;
-  numTruncatedResults: number;
-  /**
-   * sortState being undefined means don't sort, just present results in the order
-   * they appear in the sarif file.
-   */
-  sortState?: InterpretedResultsSortState;
-  sarif: sarif.Log;
-}
-
-export interface ResultsPaths {
-  resultsPath: string;
-  interpretedResultsPath: string;
-}
-
-export interface SortedResultSetInfo {
-  resultsPath: string;
-  sortState: RawResultsSortState;
-}
-
-export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
-
-/**
- * A message to indicate that the results are being updated.
- *
- * As a result of receiving this message, listeners might want to display a loading indicator.
- */
-export interface ResultsUpdatingMsg {
-  t: 'resultsUpdating';
-}
-
-export interface SetStateMsg {
-  t: 'setState';
-  resultsPath: string;
-  origResultsPaths: ResultsPaths;
-  sortedResultsMap: SortedResultsMap;
-  interpretation: undefined | Interpretation;
-  database: DatabaseInfo;
-  metadata?: QueryMetadata;
-  /**
-   * Whether to keep displaying the old results while rendering the new results.
-   *
-   * This is useful to prevent properties like scroll state being lost when rendering the sorted results after sorting a column.
-   */
-  shouldKeepOldResultsWhileRendering: boolean;
-
-  /**
-   * An experimental way of providing results from the extension.
-   * Should be undefined unless config.EXPERIMENTAL_BQRS_SETTING is set to true.
-   */
-  resultSets?: RawResultSet[];
-}
-
-/** Advance to the next or previous path no in the path viewer */
-export interface NavigatePathMsg {
-  t: 'navigatePath';
-
-  /** 1 for next, -1 for previous */
-  direction: number;
-}
-
-export type IntoResultsViewMsg = ResultsUpdatingMsg | SetStateMsg | NavigatePathMsg;
-
-export type FromResultsViewMsg =
-  | ViewSourceFileMsg
-  | ToggleDiagnostics
-  | ChangeRawResultsSortMsg
-  | ChangeInterpretedResultsSortMsg
-  | ResultViewLoaded;
-
-interface ViewSourceFileMsg {
-  t: 'viewSourceFile';
-  loc: ResolvableLocationValue;
-  databaseUri: string;
-}
-
-interface ToggleDiagnostics {
-  t: 'toggleDiagnostics';
-  databaseUri: string;
-  metadata?: QueryMetadata;
-  origResultsPaths: ResultsPaths;
-  visible: boolean;
-  kind?: string;
-}
-
-interface ResultViewLoaded {
-  t: 'resultViewLoaded';
-}
-
-export enum SortDirection {
-  asc, desc
-}
-
-export interface RawResultsSortState {
-  columnIndex: number;
-  sortDirection: SortDirection;
-}
-
-export type InterpretedResultsSortColumn =
-  'alert-message';
-
-export interface InterpretedResultsSortState {
-  sortBy: InterpretedResultsSortColumn;
-  sortDirection: SortDirection;
-}
-
-interface ChangeRawResultsSortMsg {
-  t: 'changeSort';
-  resultSetName: string;
-  /**
-   * sortState being undefined means don't sort, just present results in the order
-   * they appear in the sarif file.
-   */
-  sortState?: RawResultsSortState;
-}
-
-interface ChangeInterpretedResultsSortMsg {
-  t: 'changeInterpretedSort';
-  /**
-   * sortState being undefined means don't sort, just present results in the order
-   * they appear in the sarif file.
-   */
-  sortState?: InterpretedResultsSortState;
-}

extensions/ql-vscode/src/interface-utils.ts

@@ -0,0 +1,238 @@
+import * as crypto from 'crypto';
+import {
+  Uri,
+  Location,
+  Range,
+  WebviewPanel,
+  Webview,
+  workspace,
+  window as Window,
+  ViewColumn,
+  Selection,
+  TextEditorRevealType,
+  ThemeColor,
+} from 'vscode';
+import {
+  tryGetResolvableLocation,
+  isLineColumnLoc
+} from './pure/bqrs-utils';
+import { DatabaseItem, DatabaseManager } from './databases';
+import { ViewSourceFileMsg } from './pure/interface-types';
+import { Logger } from './logging';
+import {
+  LineColumnLocation,
+  WholeFileLocation,
+  UrlValue,
+  ResolvableLocationValue
+} from './pure/bqrs-cli-types';
+
+/**
+ * This module contains functions and types that are sharedd between
+ * interface.ts and compare-interface.ts.
+ */
+
+/** Gets a nonce string created with 128 bits of entropy. */
+export function getNonce(): string {
+  return crypto.randomBytes(16).toString('base64');
+}
+
+/**
+ * Whether to force webview to reveal
+ */
+export enum WebviewReveal {
+  Forced,
+  NotForced,
+}
+
+/**
+ * Converts a filesystem URI into a webview URI string that the given panel
+ * can use to read the file.
+ */
+export function fileUriToWebviewUri(
+  panel: WebviewPanel,
+  fileUriOnDisk: Uri
+): string {
+  return panel.webview.asWebviewUri(fileUriOnDisk).toString();
+}
+
+/**
+ * Resolves the specified CodeQL location to a URI into the source archive.
+ * @param loc CodeQL location to resolve. Must have a non-empty value for `loc.file`.
+ * @param databaseItem Database in which to resolve the file location.
+ */
+function resolveFivePartLocation(
+  loc: LineColumnLocation,
+  databaseItem: DatabaseItem
+): Location {
+  // `Range` is a half-open interval, and is zero-based. CodeQL locations are closed intervals, and
+  // are one-based. Adjust accordingly.
+  const range = new Range(
+    Math.max(0, loc.startLine - 1),
+    Math.max(0, loc.startColumn - 1),
+    Math.max(0, loc.endLine - 1),
+    Math.max(0, loc.endColumn)
+  );
+
+  return new Location(databaseItem.resolveSourceFile(loc.uri), range);
+}
+
+/**
+ * Resolves the specified CodeQL filesystem resource location to a URI into the source archive.
+ * @param loc CodeQL location to resolve, corresponding to an entire filesystem resource. Must have a non-empty value for `loc.file`.
+ * @param databaseItem Database in which to resolve the filesystem resource location.
+ */
+function resolveWholeFileLocation(
+  loc: WholeFileLocation,
+  databaseItem: DatabaseItem
+): Location {
+  // A location corresponding to the start of the file.
+  const range = new Range(0, 0, 0, 0);
+  return new Location(databaseItem.resolveSourceFile(loc.uri), range);
+}
+
+/**
+ * Try to resolve the specified CodeQL location to a URI into the source archive. If no exact location
+ * can be resolved, returns `undefined`.
+ * @param loc CodeQL location to resolve
+ * @param databaseItem Database in which to resolve the file location.
+ */
+export function tryResolveLocation(
+  loc: UrlValue | undefined,
+  databaseItem: DatabaseItem
+): Location | undefined {
+  const resolvableLoc = tryGetResolvableLocation(loc);
+  if (!resolvableLoc || typeof resolvableLoc === 'string') {
+    return;
+  } else if (isLineColumnLoc(resolvableLoc)) {
+    return resolveFivePartLocation(resolvableLoc, databaseItem);
+  } else {
+    return resolveWholeFileLocation(resolvableLoc, databaseItem);
+  }
+}
+
+/**
+ * Returns HTML to populate the given webview.
+ * Uses a content security policy that only loads the given script.
+ */
+export function getHtmlForWebview(
+  webview: Webview,
+  scriptUriOnDisk: Uri,
+  stylesheetUriOnDisk: Uri
+): string {
+  // Convert the on-disk URIs into webview URIs.
+  const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
+  const stylesheetWebviewUri = webview.asWebviewUri(stylesheetUriOnDisk);
+  // Use a nonce in the content security policy to uniquely identify the above resources.
+  const nonce = getNonce();
+  /*
+   * Content security policy:
+   * default-src: allow nothing by default.
+   * script-src: allow only the given script, using the nonce.
+   * style-src: allow only the given stylesheet, using the nonce.
+   * connect-src: only allow fetch calls to webview resource URIs
+   * (this is used to load BQRS result files).
+   */
+  return `
+<html>
+  <head>
+    <meta http-equiv="Content-Security-Policy"
+          content="default-src 'none'; script-src 'nonce-${nonce}'; style-src 'nonce-${nonce}'; connect-src ${webview.cspSource};">
+    <link nonce="${nonce}" rel="stylesheet" href="${stylesheetWebviewUri}">
+  </head>
+  <body>
+    <div id=root>
+    </div>
+      <script nonce="${nonce}" src="${scriptWebviewUri}">
+    </script>
+  </body>
+</html>`;
+}
+
+export async function showResolvableLocation(
+  loc: ResolvableLocationValue,
+  databaseItem: DatabaseItem
+): Promise<void> {
+  await showLocation(tryResolveLocation(loc, databaseItem));
+}
+
+export async function showLocation(location?: Location) {
+  if (!location) {
+    return;
+  }
+
+  const doc = await workspace.openTextDocument(location.uri);
+  const editorsWithDoc = Window.visibleTextEditors.filter(
+    (e) => e.document === doc
+  );
+  const editor =
+    editorsWithDoc.length > 0
+      ? editorsWithDoc[0]
+      : await Window.showTextDocument(
+        doc, {
+        // avoid preview mode so editor is sticky and will be added to navigation and search histories.
+        preview: false,
+        viewColumn: ViewColumn.One,
+      });
+
+  const range = location.range;
+  // When highlighting the range, vscode's occurrence-match and bracket-match highlighting will
+  // trigger based on where we place the cursor/selection, and will compete for the user's attention.
+  // For reference:
+  // - Occurences are highlighted when the cursor is next to or inside a word or a whole word is selected.
+  // - Brackets are highlighted when the cursor is next to a bracket and there is an empty selection.
+  // - Multi-line selections explicitly highlight line-break characters, but multi-line decorators do not.
+  //
+  // For single-line ranges, select the whole range, mainly to disable bracket highlighting.
+  // For multi-line ranges, place the cursor at the beginning to avoid visual artifacts from selected line-breaks.
+  // Multi-line ranges are usually large enough to overshadow the noise from bracket highlighting.
+  const selectionEnd =
+    range.start.line === range.end.line ? range.end : range.start;
+  editor.selection = new Selection(range.start, selectionEnd);
+  editor.revealRange(range, TextEditorRevealType.InCenter);
+  editor.setDecorations(shownLocationDecoration, [range]);
+  editor.setDecorations(shownLocationLineDecoration, [range]);
+}
+
+const findMatchBackground = new ThemeColor('editor.findMatchBackground');
+const findRangeHighlightBackground = new ThemeColor(
+  'editor.findRangeHighlightBackground'
+);
+
+
+export const shownLocationDecoration = Window.createTextEditorDecorationType({
+  backgroundColor: findMatchBackground,
+});
+
+export const shownLocationLineDecoration = Window.createTextEditorDecorationType(
+  {
+    backgroundColor: findRangeHighlightBackground,
+    isWholeLine: true,
+  }
+);
+
+export async function jumpToLocation(
+  msg: ViewSourceFileMsg,
+  databaseManager: DatabaseManager,
+  logger: Logger
+) {
+  const databaseItem = databaseManager.findDatabaseItem(
+    Uri.parse(msg.databaseUri)
+  );
+  if (databaseItem !== undefined) {
+    try {
+      await showResolvableLocation(msg.loc, databaseItem);
+    } catch (e) {
+      if (e instanceof Error) {
+        if (e.message.match(/File not found/)) {
+          Window.showErrorMessage(
+            'Original file of this result is not in the database\'s source archive.'
+          );
+        } else {
+          logger.log(`Unable to handleMsgFromView: ${e.message}`);
+        }
+      } else {
+        logger.log(`Unable to handleMsgFromView: ${e}`);
+      }
+    }
+  }
+}

extensions/ql-vscode/src/interface.ts

@@ -1,23 +1,52 @@
-import * as crypto from 'crypto';
 import * as path from 'path';
 import * as Sarif from 'sarif';
-import { FivePartLocation, LocationStyle, LocationValue, ResolvableLocationValue, tryGetResolvableLocation, WholeFileLocation } from 'semmle-bqrs';
-import { DisposableObject } from 'semmle-vscode-utils';
+import { DisposableObject } from './vscode-utils/disposable-object';
 import * as vscode from 'vscode';
-import { Diagnostic, DiagnosticRelatedInformation, DiagnosticSeverity, languages, Location, Range, Uri, window as Window, workspace } from 'vscode';
+import {
+  Diagnostic,
+  DiagnosticRelatedInformation,
+  DiagnosticSeverity,
+  languages,
+  Uri,
+  window as Window,
+  env
+} from 'vscode';
 import * as cli from './cli';
 import { CodeQLCliServer } from './cli';
-import { DatabaseItem, DatabaseManager } from './databases';
+import { DatabaseEventKind, DatabaseItem, DatabaseManager } from './databases';
 import { showAndLogErrorMessage } from './helpers';
-import { assertNever } from './helpers-pure';
-import { FromResultsViewMsg, Interpretation, INTERPRETED_RESULTS_PER_RUN_LIMIT, IntoResultsViewMsg, QueryMetadata, ResultsPaths, SortedResultSetInfo, SortedResultsMap, InterpretedResultsSortState, SortDirection } from './interface-types';
+import { assertNever } from './pure/helpers-pure';
+import {
+  FromResultsViewMsg,
+  Interpretation,
+  IntoResultsViewMsg,
+  QueryMetadata,
+  ResultsPaths,
+  SortedResultSetInfo,
+  SortedResultsMap,
+  InterpretedResultsSortState,
+  SortDirection,
+  ALERTS_TABLE_NAME,
+  RawResultsSortState,
+} from './pure/interface-types';
 import { Logger } from './logging';
-import * as messages from './messages';
+import { commandRunner } from './helpers';
+import * as messages from './pure/messages';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
-import { parseSarifLocation, parseSarifPlainTextMessage } from './sarif-utils';
-import { adaptSchema, adaptBqrs, RawResultSet } from './adapt';
-import { EXPERIMENTAL_BQRS_SETTING } from './config';
+import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
+import {
+  WebviewReveal,
+  fileUriToWebviewUri,
+  tryResolveLocation,
+  getHtmlForWebview,
+  shownLocationDecoration,
+  shownLocationLineDecoration,
+  jumpToLocation,
+} from './interface-utils';
+import { getDefaultResultSetName, ParsedResultSets } from './pure/interface-types';
+import { RawResultSet, transformBqrsResultSet, ResultSetSchema } from './pure/bqrs-cli-types';
+import { PAGE_SIZE } from './config';
 
 /**
  * interface.ts
@@ -27,87 +56,30 @@ import { EXPERIMENTAL_BQRS_SETTING } from './config';
  * webview asks us to.
  */
 
-/** Gets a nonce string created with 128 bits of entropy. */
-function getNonce(): string {
-  return crypto.randomBytes(16).toString('base64');
-}
-
-/**
- * Whether to force webview to reveal
- */
-export enum WebviewReveal {
-  Forced,
-  NotForced,
-}
-
-/**
- * Returns HTML to populate the given webview.
- * Uses a content security policy that only loads the given script.
- */
-function getHtmlForWebview(
-  webview: vscode.Webview,
-  scriptUriOnDisk: vscode.Uri,
-  stylesheetUriOnDisk: vscode.Uri
-): void {
-  // Convert the on-disk URIs into webview URIs.
-  const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
-  const stylesheetWebviewUri = webview.asWebviewUri(stylesheetUriOnDisk);
-  // Use a nonce in the content security policy to uniquely identify the above resources.
-  const nonce = getNonce();
-  /*
-   * Content security policy:
-   * default-src: allow nothing by default.
-   * script-src: allow only the given script, using the nonce.
-   * style-src: allow only the given stylesheet, using the nonce.
-   * connect-src: only allow fetch calls to webview resource URIs
-   * (this is used to load BQRS result files).
-   */
-  const html = `
-<html>
-  <head>
-    <meta http-equiv="Content-Security-Policy"
-          content="default-src 'none'; script-src 'nonce-${nonce}'; style-src 'nonce-${nonce}'; connect-src ${webview.cspSource};">
-    <link nonce="${nonce}" rel="stylesheet" href="${stylesheetWebviewUri}">
-  </head>
-  <body>
-    <div id=root>
-    </div>
-      <script nonce="${nonce}" src="${scriptWebviewUri}">
-    </script>
-  </body>
-</html>`;
-  webview.html = html;
-}
-
-/** Converts a filesystem URI into a webview URI string that the given panel can use to read the file. */
-export function fileUriToWebviewUri(panel: vscode.WebviewPanel, fileUriOnDisk: Uri): string {
-  return panel.webview.asWebviewUri(fileUriOnDisk).toString();
-}
-
-/** Converts a URI string received from a webview into a local filesystem URI for the same resource. */
-export function webviewUriToFileUri(webviewUri: string): Uri {
-  // Webview URIs used the vscode-resource scheme. The filesystem path of the resource can be obtained from the path component of the webview URI.
-  const path = Uri.parse(webviewUri).path;
-  // For this path to be interpreted on the filesystem, we need to parse it as a filesystem URI for the current platform.
-  return Uri.file(path);
-}
-
 function sortMultiplier(sortDirection: SortDirection): number {
   switch (sortDirection) {
-    case SortDirection.asc: return 1;
-    case SortDirection.desc: return -1;
+    case SortDirection.asc:
+      return 1;
+    case SortDirection.desc:
+      return -1;
   }
 }
 
-function sortInterpretedResults(results: Sarif.Result[], sortState: InterpretedResultsSortState | undefined): void {
+function sortInterpretedResults(
+  results: Sarif.Result[],
+  sortState: InterpretedResultsSortState | undefined
+): void {
   if (sortState !== undefined) {
     const multiplier = sortMultiplier(sortState.sortDirection);
     switch (sortState.sortBy) {
       case 'alert-message':
         results.sort((a, b) =>
-          a.message.text === undefined ? 0 :
-            b.message.text === undefined ? 0 :
-              multiplier * (a.message.text?.localeCompare(b.message.text)));
+          a.message.text === undefined
+            ? 0
+            : b.message.text === undefined
+              ? 0
+              : multiplier * a.message.text?.localeCompare(b.message.text, env.language)
+        );
         break;
       default:
         assertNever(sortState.sortBy);
@@ -115,14 +87,23 @@ function sortInterpretedResults(results: Sarif.Result[], sortState: InterpretedR
   }
 }
 
+function numPagesOfResultSet(resultSet: RawResultSet): number {
+  return Math.ceil(resultSet.schema.rows / PAGE_SIZE.getValue<number>());
+}
+
+function numInterpretedPages(interpretation: Interpretation | undefined): number {
+  return Math.ceil((interpretation?.sarif.runs[0].results?.length || 0) / PAGE_SIZE.getValue<number>());
+}
+
 export class InterfaceManager extends DisposableObject {
   private _displayedQuery?: CompletedQuery;
+  private _interpretation?: Interpretation;
   private _panel: vscode.WebviewPanel | undefined;
   private _panelLoaded = false;
   private _panelLoadedCallBacks: (() => void)[] = [];
 
   private readonly _diagnosticCollection = languages.createDiagnosticCollection(
-    `codeql-query-results`
+    'codeql-query-results'
   );
 
   constructor(
@@ -140,21 +121,32 @@ export class InterfaceManager extends DisposableObject {
     );
     logger.log('Registering path-step navigation commands.');
     this.push(
-      vscode.commands.registerCommand(
-        "codeQLQueryResults.nextPathStep",
+      commandRunner(
+        'codeQLQueryResults.nextPathStep',
         this.navigatePathStep.bind(this, 1)
       )
     );
     this.push(
-      vscode.commands.registerCommand(
-        "codeQLQueryResults.previousPathStep",
+      commandRunner(
+        'codeQLQueryResults.previousPathStep',
         this.navigatePathStep.bind(this, -1)
       )
     );
+
+    this.push(
+      this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
+        if (kind === DatabaseEventKind.Remove) {
+          this._diagnosticCollection.clear();
+          this.postMessage({
+            t: 'untoggleShowProblems'
+          });
+        }
+      })
+    );
   }
 
-  navigatePathStep(direction: number): void {
-    this.postMessage({ t: "navigatePath", direction });
+  async navigatePathStep(direction: number): Promise<void> {
+    this.postMessage({ t: 'navigatePath', direction });
   }
 
   // Returns the webview panel, creating it if it doesn't already
@@ -163,8 +155,8 @@ export class InterfaceManager extends DisposableObject {
     if (this._panel == undefined) {
       const { ctx } = this;
       const panel = (this._panel = Window.createWebviewPanel(
-        "resultsView", // internal name
-        "CodeQL Query Results", // user-visible name
+        'resultsView', // internal name
+        'CodeQL Query Results', // user-visible name
         { viewColumn: vscode.ViewColumn.Beside, preserveFocus: true },
         {
           enableScripts: true,
@@ -172,30 +164,31 @@ export class InterfaceManager extends DisposableObject {
           retainContextWhenHidden: true,
           localResourceRoots: [
             vscode.Uri.file(tmpDir.name),
-            vscode.Uri.file(path.join(this.ctx.extensionPath, "out"))
+            vscode.Uri.file(path.join(this.ctx.extensionPath, 'out'))
           ]
         }
       ));
       this._panel.onDidDispose(
         () => {
           this._panel = undefined;
+          this._displayedQuery = undefined;
         },
         null,
         ctx.subscriptions
       );
       const scriptPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath("out/resultsView.js")
+        ctx.asAbsolutePath('out/resultsView.js')
       );
       const stylesheetPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath("out/resultsView.css")
+        ctx.asAbsolutePath('out/resultsView.css')
       );
-      getHtmlForWebview(
+      panel.webview.html = getHtmlForWebview(
         panel.webview,
         scriptPathOnDisk,
         stylesheetPathOnDisk
       );
       panel.webview.onDidReceiveMessage(
-        async e => this.handleMsgFromView(e),
+        async (e) => this.handleMsgFromView(e),
         undefined,
         ctx.subscriptions
       );
@@ -203,55 +196,52 @@ export class InterfaceManager extends DisposableObject {
     return this._panel;
   }
 
-  private async changeSortState(
-    update: (query: CompletedQuery) => Promise<void>
+  private async changeInterpretedSortState(
+    sortState: InterpretedResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
       showAndLogErrorMessage(
-        "Failed to sort results since evaluation info was unknown."
+        'Failed to sort results since evaluation info was unknown.'
       );
       return;
     }
     // Notify the webview that it should expect new results.
-    await this.postMessage({ t: "resultsUpdating" });
-    await update(this._displayedQuery);
-    await this.showResults(
-      this._displayedQuery,
-      WebviewReveal.NotForced,
-      true
-    );
+    await this.postMessage({ t: 'resultsUpdating' });
+    this._displayedQuery.updateInterpretedSortState(sortState);
+    await this.showResults(this._displayedQuery, WebviewReveal.NotForced, true);
   }
 
-  private async handleMsgFromView(
-    msg: FromResultsViewMsg
+  private async changeRawSortState(
+    resultSetName: string,
+    sortState: RawResultsSortState | undefined
   ): Promise<void> {
+    if (this._displayedQuery === undefined) {
+      showAndLogErrorMessage(
+        'Failed to sort results since evaluation info was unknown.'
+      );
+      return;
+    }
+    // Notify the webview that it should expect new results.
+    await this.postMessage({ t: 'resultsUpdating' });
+    await this._displayedQuery.updateSortState(
+      this.cliServer,
+      resultSetName,
+      sortState
+    );
+    // Sorting resets to first page, as there is arguably no particular
+    // correlation between the results on the nth page that the user
+    // was previously viewing and the contents of the nth page in a
+    // new sorted order.
+    await this.showPageOfRawResults(resultSetName, 0, true);
+  }
+
+  private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
     switch (msg.t) {
-      case "viewSourceFile": {
-        const databaseItem = this.databaseManager.findDatabaseItem(
-          Uri.parse(msg.databaseUri)
-        );
-        if (databaseItem !== undefined) {
-          try {
-            await showLocation(msg.loc, databaseItem);
-          } catch (e) {
-            if (e instanceof Error) {
-              if (e.message.match(/File not found/)) {
-                vscode.window.showErrorMessage(
-                  `Original file of this result is not in the database's source archive.`
-                );
-              } else {
-                this.logger.log(
-                  `Unable to handleMsgFromView: ${e.message}`
-                );
-              }
-            } else {
-              this.logger.log(`Unable to handleMsgFromView: ${e}`);
-            }
-          }
-        }
+      case 'viewSourceFile': {
+        await jumpToLocation(msg, this.databaseManager, this.logger);
         break;
       }
-      case "toggleDiagnostics": {
+      case 'toggleDiagnostics': {
         if (msg.visible) {
           const databaseItem = this.databaseManager.findDatabaseItem(
             Uri.parse(msg.databaseUri)
@@ -269,24 +259,35 @@ export class InterfaceManager extends DisposableObject {
         }
         break;
       }
-      case "resultViewLoaded":
+      case 'resultViewLoaded':
         this._panelLoaded = true;
-        this._panelLoadedCallBacks.forEach(cb => cb());
+        this._panelLoadedCallBacks.forEach((cb) => cb());
         this._panelLoadedCallBacks = [];
         break;
-      case "changeSort":
-        await this.changeSortState(query =>
-          query.updateSortState(
-            this.cliServer,
-            msg.resultSetName,
-            msg.sortState
-          )
-        );
+      case 'changeSort':
+        await this.changeRawSortState(msg.resultSetName, msg.sortState);
         break;
-      case "changeInterpretedSort":
-        await this.changeSortState(query =>
-          query.updateInterpretedSortState(this.cliServer, msg.sortState)
-        );
+      case 'changeInterpretedSort':
+        await this.changeInterpretedSortState(msg.sortState);
+        break;
+      case 'changePage':
+        if (msg.selectedTable === ALERTS_TABLE_NAME) {
+          await this.showPageOfInterpretedResults(msg.pageNumber);
+        }
+        else {
+          await this.showPageOfRawResults(
+            msg.selectedTable,
+            msg.pageNumber,
+            // When we are in an unsorted state, we guarantee that
+            // sortedResultsInfo doesn't have an entry for the current
+            // result set. Use this to determine whether or not we use
+            // the sorted bqrs file.
+            this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
+          );
+        }
+        break;
+      case 'openFile':
+        await this.openFile(msg.filePath);
         break;
       default:
         assertNever(msg);
@@ -298,7 +299,7 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private waitForPanelLoaded(): Promise<void> {
-    return new Promise(resolve => {
+    return new Promise((resolve) => {
       if (this._panelLoaded) {
         resolve();
       } else {
@@ -308,14 +309,14 @@ export class InterfaceManager extends DisposableObject {
   }
 
   /**
-  * Show query results in webview panel.
-  * @param results Evaluation info for the executed query.
-  * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
-  * @param forceReveal Force the webview panel to be visible and
-  * Appropriate when the user has just performed an explicit
-  * UI interaction requesting results, e.g. clicking on a query
-  * history entry.
-  */
+   * Show query results in webview panel.
+   * @param results Evaluation info for the executed query.
+   * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
+   * @param forceReveal Force the webview panel to be visible and
+   * Appropriate when the user has just performed an explicit
+   * UI interaction requesting results, e.g. clicking on a query
+   * history entry.
+   */
   public async showResults(
     results: CompletedQuery,
     forceReveal: WebviewReveal,
@@ -325,7 +326,8 @@ export class InterfaceManager extends DisposableObject {
       return;
     }
 
-    const interpretation = await this.interpretResultsInfo(
+    this._interpretation = undefined;
+    const interpretationPage = await this.interpretResultsInfo(
       results.query,
       results.interpretedResultsSortState
     );
@@ -333,9 +335,7 @@ export class InterfaceManager extends DisposableObject {
     const sortedResultsMap: SortedResultsMap = {};
     results.sortedResultsInfo.forEach(
       (v, k) =>
-        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(
-          v
-        ))
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v))
     );
 
     this._displayedQuery = results;
@@ -349,50 +349,192 @@ export class InterfaceManager extends DisposableObject {
       // is not visible; it's in a not-currently-viewed tab. Show a
       // more asynchronous message to not so abruptly interrupt
       // user's workflow by immediately revealing the panel.
-      const showButton = "View Results";
+      const showButton = 'View Results';
       const queryName = results.queryName;
       const resultPromise = vscode.window.showInformationMessage(
-        `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ""}.`,
+        `Finished running query ${
+        queryName.length > 0 ? ` "${queryName}"` : ''
+        }.`,
         showButton
       );
       // Address this click asynchronously so we still update the
       // query history immediately.
-      resultPromise.then(result => {
+      resultPromise.then((result) => {
         if (result === showButton) {
           panel.reveal();
         }
       });
     }
 
-    let resultSets: RawResultSet[] | undefined;
+    // Note that the resultSetSchemas will return offsets for the default (unsorted) page,
+    // which may not be correct. However, in this case, it doesn't matter since we only
+    // need the first offset, which will be the same no matter which sorting we use.
+    const resultSetSchemas = await this.getResultSetSchemas(results);
+    const resultSetNames = resultSetSchemas.map(schema => schema.name);
 
-    if (EXPERIMENTAL_BQRS_SETTING.getValue()) {
-      resultSets = [];
-      const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath);
-      for (const schema of schemas["result-sets"]) {
-        const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name);
-        const adaptedSchema = adaptSchema(schema);
-        const resultSet = adaptBqrs(adaptedSchema, chunk);
-        resultSets.push(resultSet);
+    const selectedTable = getDefaultResultSetName(resultSetNames);
+    const schema = resultSetSchemas.find(
+      (resultSet) => resultSet.name == selectedTable
+    )!;
+
+    // Use sorted results path if it exists. This may happen if we are
+    // reloading the results view after it has been sorted in the past.
+    const resultsPath = results.getResultsPath(selectedTable);
+    const pageSize = PAGE_SIZE.getValue<number>();
+    const chunk = await this.cliServer.bqrsDecode(
+      resultsPath,
+      schema.name,
+      {
+        // Always send the first page.
+        // It may not wind up being the page we actually show,
+        // if there are interpreted results, but speculatively
+        // send anyway.
+        offset: schema.pagination?.offsets[0],
+        pageSize
       }
-    }
+    );
+    const resultSet = transformBqrsResultSet(schema, chunk);
+    const parsedResultSets: ParsedResultSets = {
+      pageNumber: 0,
+      pageSize,
+      numPages: numPagesOfResultSet(resultSet),
+      numInterpretedPages: numInterpretedPages(this._interpretation),
+      resultSet: { ...resultSet, t: 'RawResultSet' },
+      selectedTable: undefined,
+      resultSetNames,
+    };
 
     await this.postMessage({
-      t: "setState",
-      interpretation,
+      t: 'setState',
+      interpretation: interpretationPage,
       origResultsPaths: results.query.resultsPaths,
       resultsPath: this.convertPathToWebviewUri(
         results.query.resultsPaths.resultsPath
       ),
-      resultSets,
+      parsedResultSets,
       sortedResultsMap,
       database: results.database,
       shouldKeepOldResultsWhileRendering,
-      metadata: results.query.metadata
+      metadata: results.query.metadata,
+      queryName: results.toString(),
+      queryPath: results.query.program.queryPath
     });
   }
 
-  private async getTruncatedResults(
+  /**
+   * Show a page of interpreted results
+   */
+  public async showPageOfInterpretedResults(
+    pageNumber: number
+  ): Promise<void> {
+    if (this._displayedQuery === undefined) {
+      throw new Error('Trying to show interpreted results but displayed query was undefined');
+    }
+    if (this._interpretation === undefined) {
+      throw new Error('Trying to show interpreted results but interpretation was undefined');
+    }
+    if (this._interpretation.sarif.runs[0].results === undefined) {
+      throw new Error('Trying to show interpreted results but results were undefined');
+    }
+
+    const resultSetSchemas = await this.getResultSetSchemas(this._displayedQuery);
+    const resultSetNames = resultSetSchemas.map(schema => schema.name);
+
+    await this.postMessage({
+      t: 'showInterpretedPage',
+      interpretation: this.getPageOfInterpretedResults(pageNumber),
+      database: this._displayedQuery.database,
+      metadata: this._displayedQuery.query.metadata,
+      pageNumber,
+      resultSetNames,
+      pageSize: PAGE_SIZE.getValue(),
+      numPages: numInterpretedPages(this._interpretation),
+      queryName: this._displayedQuery.toString(),
+      queryPath: this._displayedQuery.query.program.queryPath
+    });
+  }
+
+  private async getResultSetSchemas(results: CompletedQuery, selectedTable = ''): Promise<ResultSetSchema[]> {
+    const resultsPath = results.getResultsPath(selectedTable);
+    const schemas = await this.cliServer.bqrsInfo(
+      resultsPath,
+      PAGE_SIZE.getValue()
+    );
+    return schemas['result-sets'];
+  }
+
+  public async openFile(filePath: string) {
+    const textDocument = await vscode.workspace.openTextDocument(filePath);
+    await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+  }
+
+  /**
+   * Show a page of raw results from the chosen table.
+   */
+  public async showPageOfRawResults(
+    selectedTable: string,
+    pageNumber: number,
+    sorted = false
+  ): Promise<void> {
+    const results = this._displayedQuery;
+    if (results === undefined) {
+      throw new Error('trying to view a page of a query that is not loaded');
+    }
+
+    const sortedResultsMap: SortedResultsMap = {};
+    results.sortedResultsInfo.forEach(
+      (v, k) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v))
+    );
+
+    const resultSetSchemas = await this.getResultSetSchemas(results, sorted ? selectedTable : '');
+    const resultSetNames = resultSetSchemas.map(schema => schema.name);
+
+    const schema = resultSetSchemas.find(
+      (resultSet) => resultSet.name == selectedTable
+    )!;
+    if (schema === undefined)
+      throw new Error(`Query result set '${selectedTable}' not found.`);
+
+    const pageSize = PAGE_SIZE.getValue<number>();
+    const chunk = await this.cliServer.bqrsDecode(
+      results.getResultsPath(selectedTable, sorted),
+      schema.name,
+      {
+        offset: schema.pagination?.offsets[pageNumber],
+        pageSize
+      }
+    );
+    const resultSet = transformBqrsResultSet(schema, chunk);
+
+    const parsedResultSets: ParsedResultSets = {
+      pageNumber,
+      pageSize,
+      resultSet: { t: 'RawResultSet', ...resultSet },
+      numPages: numPagesOfResultSet(resultSet),
+      numInterpretedPages: numInterpretedPages(this._interpretation),
+      selectedTable: selectedTable,
+      resultSetNames,
+    };
+
+    await this.postMessage({
+      t: 'setState',
+      interpretation: this._interpretation,
+      origResultsPaths: results.query.resultsPaths,
+      resultsPath: this.convertPathToWebviewUri(
+        results.query.resultsPaths.resultsPath
+      ),
+      parsedResultSets,
+      sortedResultsMap,
+      database: results.database,
+      shouldKeepOldResultsWhileRendering: false,
+      metadata: results.query.metadata,
+      queryName: results.toString(),
+      queryPath: results.query.program.queryPath
+    });
+  }
+
+  private async _getInterpretedResults(
     metadata: QueryMetadata | undefined,
     resultsPaths: ResultsPaths,
     sourceInfo: cli.SourceInfo | undefined,
@@ -405,32 +547,52 @@ export class InterfaceManager extends DisposableObject {
       resultsPaths,
       sourceInfo
     );
-    // For performance reasons, limit the number of results we try
-    // to serialize and send to the webview. TODO: possibly also
-    // limit number of paths per result, number of steps per path,
-    // or throw an error if we are in aggregate trying to send
-    // massively too much data, as it can make the extension
-    // unresponsive.
-
-    let numTruncatedResults = 0;
     sarif.runs.forEach(run => {
       if (run.results !== undefined) {
         sortInterpretedResults(run.results, sortState);
-        if (run.results.length > INTERPRETED_RESULTS_PER_RUN_LIMIT) {
-          numTruncatedResults +=
-            run.results.length - INTERPRETED_RESULTS_PER_RUN_LIMIT;
-          run.results = run.results.slice(
-            0,
-            INTERPRETED_RESULTS_PER_RUN_LIMIT
-          );
-        }
       }
     });
-    return {
+
+    const numTotalResults = (() => {
+      if (sarif.runs.length === 0) return 0;
+      if (sarif.runs[0].results === undefined) return 0;
+      return sarif.runs[0].results.length;
+    })();
+
+    const interpretation: Interpretation = {
       sarif,
       sourceLocationPrefix,
-      numTruncatedResults,
-      sortState
+      numTruncatedResults: 0,
+      numTotalResults,
+      sortState,
+    };
+    this._interpretation = interpretation;
+    return interpretation;
+  }
+
+  private getPageOfInterpretedResults(
+    pageNumber: number
+  ): Interpretation {
+
+    function getPageOfRun(run: Sarif.Run): Sarif.Run {
+      return {
+        ...run, results: run.results?.slice(
+          PAGE_SIZE.getValue<number>() * pageNumber,
+          PAGE_SIZE.getValue<number>() * (pageNumber + 1)
+        )
+      };
+    }
+
+    if (this._interpretation === undefined) {
+      throw new Error('Tried to get interpreted results before interpretation finished');
+    }
+    if (this._interpretation.sarif.runs.length !== 1) {
+      this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
+    }
+    const interp = this._interpretation;
+    return {
+      ...interp,
+      sarif: { ...interp.sarif, runs: [getPageOfRun(interp.sarif.runs[0])] },
     };
   }
 
@@ -438,7 +600,6 @@ export class InterfaceManager extends DisposableObject {
     query: QueryInfo,
     sortState: InterpretedResultsSortState | undefined
   ): Promise<Interpretation | undefined> {
-    let interpretation: Interpretation | undefined = undefined;
     if (
       (await query.canHaveInterpretedResults()) &&
       query.quickEvalPosition === undefined // never do results interpretation if quickEval
@@ -453,9 +614,9 @@ export class InterfaceManager extends DisposableObject {
             ? undefined
             : {
               sourceArchive: sourceArchiveUri.fsPath,
-              sourceLocationPrefix
+              sourceLocationPrefix,
             };
-        interpretation = await this.getTruncatedResults(
+        await this._getInterpretedResults(
           query.metadata,
           query.resultsPaths,
           sourceInfo,
@@ -470,7 +631,7 @@ export class InterfaceManager extends DisposableObject {
         );
       }
     }
-    return interpretation;
+    return this._interpretation && this.getPageOfInterpretedResults(0);
   }
 
   private async showResultsAsDiagnostics(
@@ -487,9 +648,10 @@ export class InterfaceManager extends DisposableObject {
         ? undefined
         : {
           sourceArchive: sourceArchiveUri.fsPath,
-          sourceLocationPrefix
+          sourceLocationPrefix,
         };
-    const interpretation = await this.getTruncatedResults(
+    // TODO: Performance-testing to determine whether this truncation is necessary.
+    const interpretation = await this._getInterpretedResults(
       metadata,
       resultsInfo,
       sourceInfo,
@@ -498,10 +660,7 @@ export class InterfaceManager extends DisposableObject {
     );
 
     try {
-      await this.showProblemResultsAsDiagnostics(
-        interpretation,
-        database
-      );
+      await this.showProblemResultsAsDiagnostics(interpretation, database);
     } catch (e) {
       const msg = e instanceof Error ? e.message : e.toString();
       this.logger.log(
@@ -519,7 +678,7 @@ export class InterfaceManager extends DisposableObject {
 
     if (!sarif.runs || !sarif.runs[0].results) {
       this.logger.log(
-        "Didn't find a run in the sarif results. Error processing sarif?"
+        'Didn\'t find a run in the sarif results. Error processing sarif?'
       );
       return;
     }
@@ -529,11 +688,11 @@ export class InterfaceManager extends DisposableObject {
     for (const result of sarif.runs[0].results) {
       const message = result.message.text;
       if (message === undefined) {
-        this.logger.log("Sarif had result without plaintext message");
+        this.logger.log('Sarif had result without plaintext message');
         continue;
       }
       if (!result.locations) {
-        this.logger.log("Sarif had result without location");
+        this.logger.log('Sarif had result without location');
         continue;
       }
 
@@ -541,12 +700,12 @@ export class InterfaceManager extends DisposableObject {
         result.locations[0],
         sourceLocationPrefix
       );
-      if (sarifLoc.t == "NoLocation") {
+      if ('hint' in sarifLoc) {
         continue;
       }
       const resultLocation = tryResolveLocation(sarifLoc, databaseItem);
       if (!resultLocation) {
-        this.logger.log("Sarif location was not resolvable " + sarifLoc);
+        this.logger.log('Sarif location was not resolvable ' + sarifLoc);
         continue;
       }
       const parsedMessage = parseSarifPlainTextMessage(message);
@@ -558,7 +717,7 @@ export class InterfaceManager extends DisposableObject {
       }
       const resultMessageChunks: string[] = [];
       for (const section of parsedMessage) {
-        if (typeof section === "string") {
+        if (typeof section === 'string') {
           resultMessageChunks.push(section);
         } else {
           resultMessageChunks.push(section.text);
@@ -566,7 +725,7 @@ export class InterfaceManager extends DisposableObject {
             relatedLocationsById[section.dest],
             sourceLocationPrefix
           );
-          if (sarifChunkLoc.t == "NoLocation") {
+          if ('hint' in sarifChunkLoc) {
             continue;
           }
           const referenceLocation = tryResolveLocation(
@@ -585,7 +744,7 @@ export class InterfaceManager extends DisposableObject {
       }
       const diagnostic = new Diagnostic(
         resultLocation.range,
-        resultMessageChunks.join(""),
+        resultMessageChunks.join(''),
         DiagnosticSeverity.Warning
       );
       diagnostic.relatedInformation = relatedInformation;
@@ -604,7 +763,7 @@ export class InterfaceManager extends DisposableObject {
   ): SortedResultSetInfo {
     return {
       resultsPath: this.convertPathToWebviewUri(info.resultsPath),
-      sortState: info.sortState
+      sortState: info.sortState,
     };
   }
 
@@ -621,93 +780,3 @@ export class InterfaceManager extends DisposableObject {
     }
   }
 }
-
-const findMatchBackground = new vscode.ThemeColor('editor.findMatchBackground');
-const findRangeHighlightBackground = new vscode.ThemeColor('editor.findRangeHighlightBackground');
-
-const shownLocationDecoration = vscode.window.createTextEditorDecorationType({
-  backgroundColor: findMatchBackground,
-});
-
-const shownLocationLineDecoration = vscode.window.createTextEditorDecorationType({
-  backgroundColor: findRangeHighlightBackground,
-  isWholeLine: true
-});
-
-async function showLocation(loc: ResolvableLocationValue, databaseItem: DatabaseItem): Promise<void> {
-  const resolvedLocation = tryResolveLocation(loc, databaseItem);
-  if (resolvedLocation) {
-    const doc = await workspace.openTextDocument(resolvedLocation.uri);
-    const editorsWithDoc = Window.visibleTextEditors.filter(e => e.document === doc);
-    const editor = editorsWithDoc.length > 0
-      ? editorsWithDoc[0]
-      : await Window.showTextDocument(doc, vscode.ViewColumn.One);
-    const range = resolvedLocation.range;
-    // When highlighting the range, vscode's occurrence-match and bracket-match highlighting will
-    // trigger based on where we place the cursor/selection, and will compete for the user's attention.
-    // For reference:
-    // - Occurences are highlighted when the cursor is next to or inside a word or a whole word is selected.
-    // - Brackets are highlighted when the cursor is next to a bracket and there is an empty selection.
-    // - Multi-line selections explicitly highlight line-break characters, but multi-line decorators do not.
-    //
-    // For single-line ranges, select the whole range, mainly to disable bracket highlighting.
-    // For multi-line ranges, place the cursor at the beginning to avoid visual artifacts from selected line-breaks.
-    // Multi-line ranges are usually large enough to overshadow the noise from bracket highlighting.
-    const selectionEnd = (range.start.line === range.end.line)
-      ? range.end
-      : range.start;
-    editor.selection = new vscode.Selection(range.start, selectionEnd);
-    editor.revealRange(range, vscode.TextEditorRevealType.InCenter);
-    editor.setDecorations(shownLocationDecoration, [range]);
-    editor.setDecorations(shownLocationLineDecoration, [range]);
-  }
-}
-
-/**
- * Resolves the specified CodeQL location to a URI into the source archive.
- * @param loc CodeQL location to resolve. Must have a non-empty value for `loc.file`.
- * @param databaseItem Database in which to resolve the file location.
- */
-function resolveFivePartLocation(loc: FivePartLocation, databaseItem: DatabaseItem): Location {
-  // `Range` is a half-open interval, and is zero-based. CodeQL locations are closed intervals, and
-  // are one-based. Adjust accordingly.
-  const range = new Range(Math.max(0, loc.lineStart - 1),
-    Math.max(0, loc.colStart - 1),
-    Math.max(0, loc.lineEnd - 1),
-    Math.max(0, loc.colEnd));
-
-  return new Location(databaseItem.resolveSourceFile(loc.file), range);
-}
-
-/**
- * Resolves the specified CodeQL filesystem resource location to a URI into the source archive.
- * @param loc CodeQL location to resolve, corresponding to an entire filesystem resource. Must have a non-empty value for `loc.file`.
- * @param databaseItem Database in which to resolve the filesystem resource location.
- */
-function resolveWholeFileLocation(loc: WholeFileLocation, databaseItem: DatabaseItem): Location {
-  // A location corresponding to the start of the file.
-  const range = new Range(0, 0, 0, 0);
-  return new Location(databaseItem.resolveSourceFile(loc.file), range);
-}
-
-/**
- * Try to resolve the specified CodeQL location to a URI into the source archive. If no exact location
- * can be resolved, returns `undefined`.
- * @param loc CodeQL location to resolve
- * @param databaseItem Database in which to resolve the file location.
- */
-function tryResolveLocation(loc: LocationValue | undefined,
-  databaseItem: DatabaseItem): Location | undefined {
-  const resolvableLoc = tryGetResolvableLocation(loc);
-  if (resolvableLoc === undefined) {
-    return undefined;
-  }
-  switch (resolvableLoc.t) {
-    case LocationStyle.FivePart:
-      return resolveFivePartLocation(resolvableLoc, databaseItem);
-    case LocationStyle.WholeFile:
-      return resolveWholeFileLocation(resolvableLoc, databaseItem);
-    default:
-      return undefined;
-  }
-}

extensions/ql-vscode/src/languageSupport.ts

@@ -1,5 +1,4 @@
-import { IndentAction, languages } from "vscode";
-
+import { languages, IndentAction, OnEnterRule } from 'vscode';
 
 /**
  * OnEnterRules are available in language-configurations, but you cannot specify them in the language-configuration.json.
@@ -17,36 +16,42 @@ export function install() {
   // setLanguageConfiguration requires a regexp for the wordpattern, not a string
   langConfig.wordPattern = new RegExp(langConfig.wordPattern);
   langConfig.onEnterRules = onEnterRules;
+  langConfig.indentationRules = {
+    decreaseIndentPattern: /^((?!.*?\/\*).*\*\/)?\s*[\}\]].*$/,
+    increaseIndentPattern: /^((?!\/\/).)*(\{[^}"'`]*|\([^)"'`]*|\[[^\]"'`]*)$/
+  };
 
   languages.setLanguageConfiguration('ql', langConfig);
   languages.setLanguageConfiguration('qll', langConfig);
   languages.setLanguageConfiguration('dbscheme', langConfig);
-
 }
 
-const onEnterRules = [
+const onEnterRules: OnEnterRule[] = [
   {
     // e.g. /** | */
     beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
     afterText: /^\s*\*\/$/,
-    action: { indentAction: IndentAction.IndentOutdent, appendText: ' * ' }
-  }, {
+    action: { indentAction: IndentAction.IndentOutdent, appendText: ' * ' },
+  },
+  {
     // e.g. /** ...|
     beforeText: /^\s*\/\*\*(?!\/)([^\*]|\*(?!\/))*$/,
-    action: { indentAction: IndentAction.None, appendText: ' * ' }
-  }, {
+    action: { indentAction: IndentAction.None, appendText: ' * ' },
+  },
+  {
     // e.g.  * ...|
     beforeText: /^(\t|[ ])*[ ]\*([ ]([^\*]|\*(?!\/))*)?$/,
-    oneLineAboveText: /^(\s*(\/\*\*|\*)).*/,
-    action: { indentAction: IndentAction.None, appendText: '* ' }
-  }, {
+    // oneLineAboveText: /^(\s*(\/\*\*|\*)).*/,
+    action: { indentAction: IndentAction.None, appendText: '* ' },
+  },
+  {
     // e.g.  */|
     beforeText: /^(\t|[ ])*[ ]\*\/\s*$/,
-    action: { indentAction: IndentAction.None, removeText: 1 }
+    action: { indentAction: IndentAction.None, removeText: 1 },
   },
   {
     // e.g.  *-----*/|
     beforeText: /^(\t|[ ])*[ ]\*[^/]*\*\/\s*$/,
-    action: { indentAction: IndentAction.None, removeText: 1 }
-  }
+    action: { indentAction: IndentAction.None, removeText: 1 },
+  },
 ];

extensions/ql-vscode/src/logging.ts

@@ -1,5 +1,5 @@
 import { window as Window, OutputChannel, Progress, Disposable } from 'vscode';
-import { DisposableObject } from 'semmle-vscode-utils';
+import { DisposableObject } from './vscode-utils/disposable-object';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 

extensions/ql-vscode/src/pure/bqrs-cli-types.ts

@@ -8,12 +8,12 @@ export const PAGE_SIZE = 1000;
  */
 // eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace ColumnKindCode {
-  export const FLOAT = "f";
-  export const INTEGER = "i";
-  export const STRING = "s";
-  export const BOOLEAN = "b";
-  export const DATE = "d";
-  export const ENTITY = "e";
+  export const FLOAT = 'f';
+  export const INTEGER = 'i';
+  export const STRING = 's';
+  export const BOOLEAN = 'b';
+  export const DATE = 'd';
+  export const ENTITY = 'e';
 }
 
 export type ColumnKind =
@@ -37,7 +37,7 @@ export interface ResultSetSchema {
 }
 
 export function getResultSetSchema(resultSetName: string, resultSets: BQRSInfo): ResultSetSchema | undefined {
-  for (const schema of resultSets["result-sets"]) {
+  for (const schema of resultSets['result-sets']) {
     if (schema.name === resultSetName) {
       return schema;
     }
@@ -45,17 +45,20 @@ export function getResultSetSchema(resultSetName: string, resultSets: BQRSInfo):
   return undefined;
 }
 export interface PaginationInfo {
-  "step-size": number;
+  'step-size': number;
   offsets: number[];
 }
 
 export interface BQRSInfo {
-  "result-sets": ResultSetSchema[];
+  'result-sets': ResultSetSchema[];
 }
 
+export type BqrsId = number;
+
 export interface EntityValue {
   url?: UrlValue;
   label?: string;
+  id?: BqrsId;
 }
 
 export interface LineColumnLocation {
@@ -64,18 +67,6 @@ export interface LineColumnLocation {
   startColumn: number;
   endLine: number;
   endColumn: number;
-  charOffset: never;
-  charLength: never;
-}
-
-export interface OffsetLengthLocation {
-  uri: string;
-  startLine: never;
-  startColumn: never;
-  endLine: never;
-  endColumn: never;
-  charOffset: number;
-  charLength: number;
 }
 
 export interface WholeFileLocation {
@@ -84,15 +75,34 @@ export interface WholeFileLocation {
   startColumn: never;
   endLine: never;
   endColumn: never;
-  charOffset: never;
-  charLength: never;
 }
 
-export type UrlValue = LineColumnLocation | OffsetLengthLocation | WholeFileLocation | string;
+export type ResolvableLocationValue = WholeFileLocation | LineColumnLocation;
 
+export type UrlValue = ResolvableLocationValue  | string;
 
 export type ColumnValue = EntityValue | number | string | boolean;
 
+export type ResultRow = ColumnValue[];
+
+export interface RawResultSet {
+  readonly schema: ResultSetSchema;
+  readonly rows: readonly ResultRow[];
+}
+
+// TODO: This function is not necessary. It generates a tuple that is slightly easier
+// to handle than the ResultSetSchema and DecodedBqrsChunk. But perhaps it is unnecessary
+// boilerplate.
+export function transformBqrsResultSet(
+  schema: ResultSetSchema,
+  page: DecodedBqrsChunk
+): RawResultSet {
+  return {
+    schema,
+    rows: Array.from(page.tuples),
+  };
+}
+
 export interface DecodedBqrsChunk {
   tuples: ColumnValue[][];
   next?: number;

extensions/ql-vscode/src/pure/bqrs-utils.ts

@@ -0,0 +1,96 @@
+import {
+  UrlValue,
+  ResolvableLocationValue,
+  LineColumnLocation,
+  WholeFileLocation
+} from './bqrs-cli-types';
+
+/**
+ * The CodeQL filesystem libraries use this pattern in `getURL()` predicates
+ * to describe the location of an entire filesystem resource.
+ * Such locations appear as `StringLocation`s instead of `FivePartLocation`s.
+ *
+ * Folder resources also get similar URLs, but with the `folder` scheme.
+ * They are deliberately ignored here, since there is no suitable location to show the user.
+ */
+const FILE_LOCATION_REGEX = /file:\/\/(.+):([0-9]+):([0-9]+):([0-9]+):([0-9]+)/;
+/**
+ * Gets a resolvable source file location for the specified `LocationValue`, if possible.
+ * @param loc The location to test.
+ */
+export function tryGetResolvableLocation(
+  loc: UrlValue | undefined
+): ResolvableLocationValue | undefined {
+  let resolvedLoc;
+  if (loc === undefined) {
+    resolvedLoc = undefined;
+  } else if (isWholeFileLoc(loc) || isLineColumnLoc(loc)) {
+    resolvedLoc = loc as ResolvableLocationValue;
+  } else if (isStringLoc(loc)) {
+    resolvedLoc = tryGetLocationFromString(loc);
+  } else {
+    resolvedLoc = undefined;
+  }
+
+  return resolvedLoc;
+}
+
+export function tryGetLocationFromString(
+  loc: string
+): ResolvableLocationValue | undefined {
+  const matches = FILE_LOCATION_REGEX.exec(loc);
+  if (matches && matches.length > 1 && matches[1]) {
+    if (isWholeFileMatch(matches)) {
+      return {
+        uri: matches[1],
+      } as WholeFileLocation;
+    } else {
+      return {
+        uri: matches[1],
+        startLine: Number(matches[2]),
+        startColumn: Number(matches[3]),
+        endLine: Number(matches[4]),
+        endColumn: Number(matches[5]),
+      };
+    }
+  } else {
+    return undefined;
+  }
+}
+
+function isWholeFileMatch(matches: RegExpExecArray): boolean {
+  return (
+    matches[2] === '0' &&
+    matches[3] === '0' &&
+    matches[4] === '0' &&
+    matches[5] === '0'
+  );
+}
+
+/**
+ * Checks whether the file path is empty. If so, we do not want to render this location
+ * as a link.
+ *
+ * @param uri A file uri
+ */
+export function isEmptyPath(uriStr: string) {
+  return !uriStr || uriStr === 'file:/';
+}
+
+export function isLineColumnLoc(loc: UrlValue): loc is LineColumnLocation {
+  return typeof loc !== 'string'
+    && !isEmptyPath(loc.uri)
+    && 'startLine' in loc
+    && 'startColumn' in loc
+    && 'endLine' in loc
+    && 'endColumn' in loc
+    && loc.endColumn > 0;
+}
+
+export function isWholeFileLoc(loc: UrlValue): loc is WholeFileLocation {
+  return typeof loc !== 'string' && !isEmptyPath(loc.uri) && !isLineColumnLoc(loc);
+}
+
+export function isStringLoc(loc: UrlValue): loc is string {
+  return typeof loc === 'string';
+}

extensions/ql-vscode/src/pure/files.ts

@@ -0,0 +1,30 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+
+/**
+ * Recursively finds all .ql files in this set of Uris.
+ *
+ * @param paths The list of Uris to search through
+ *
+ * @returns list of ql files and a boolean describing whether or not a directory was found/
+ */
+export async function gatherQlFiles(paths: string[]): Promise<[string[], boolean]> {
+  const gatheredUris: Set<string> = new Set();
+  let dirFound = false;
+  for (const nextPath of paths) {
+    if (
+      (await fs.pathExists(nextPath)) &&
+      (await fs.stat(nextPath)).isDirectory()
+    ) {
+      dirFound = true;
+      const subPaths = await fs.readdir(nextPath);
+      const fullPaths = subPaths.map(p => path.join(nextPath, p));
+      const nestedFiles = (await gatherQlFiles(fullPaths))[0];
+      nestedFiles.forEach(nested => gatheredUris.add(nested));
+    } else if (nextPath.endsWith('.ql')) {
+      gatheredUris.add(nextPath);
+    }
+  }
+  return [Array.from(gatheredUris), dirFound];
+}

extensions/ql-vscode/src/pure/helpers-pure.ts

@@ -2,7 +2,7 @@
  * helpers-pure.ts
  * ------------
  *
- * Helper functions that don't depend on vscode and therefore can be used by the front-end and pure unit tests.
+ * Helper functions that don't depend on vscode or the CLI and therefore can be used by the front-end and pure unit tests.
  */
 
 /**
@@ -10,7 +10,7 @@
  */
 class ExhaustivityCheckingError extends Error {
   constructor(public expectedExhaustiveValue: never) {
-    super("Internal error: exhaustivity checking failure");
+    super('Internal error: exhaustivity checking failure');
   }
 }
 
@@ -21,3 +21,11 @@ class ExhaustivityCheckingError extends Error {
 export function assertNever(value: never): never {
   throw new ExhaustivityCheckingError(value);
 }
+
+/**
+ * Use to perform array filters where the predicate is asynchronous.
+ */
+export const asyncFilter = async function <T>(arr: T[], predicate: (arg0: T) => Promise<boolean>) {
+  const results = await Promise.all(arr.map(predicate));
+  return arr.filter((_, index) => results[index]);
+};

extensions/ql-vscode/src/pure/interface-types.ts

@@ -0,0 +1,365 @@
+import * as sarif from 'sarif';
+import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationValue } from './bqrs-cli-types';
+
+/**
+ * This module contains types and code that are shared between
+ * the webview and the extension.
+ */
+
+export const SELECT_TABLE_NAME = '#select';
+export const ALERTS_TABLE_NAME = 'alerts';
+
+export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
+export type PathTableResultSet = {
+  t: 'SarifResultSet';
+  readonly schema: ResultSetSchema;
+  name: string;
+} & Interpretation;
+
+export type ResultSet = RawTableResultSet | PathTableResultSet;
+
+/**
+ * Only ever show this many rows in a raw result table.
+ */
+export const RAW_RESULTS_LIMIT = 10000;
+
+export interface DatabaseInfo {
+  name: string;
+  databaseUri: string;
+}
+
+/** Arbitrary query metadata */
+export interface QueryMetadata {
+  name?: string;
+  description?: string;
+  id?: string;
+  kind?: string;
+}
+
+export interface PreviousExecution {
+  queryName: string;
+  time: string;
+  databaseName: string;
+  durationSeconds: number;
+}
+
+export interface Interpretation {
+  sourceLocationPrefix: string;
+  numTruncatedResults: number;
+  numTotalResults: number;
+  /**
+   * sortState being undefined means don't sort, just present results in the order
+   * they appear in the sarif file.
+   */
+  sortState?: InterpretedResultsSortState;
+  sarif: sarif.Log;
+}
+
+export interface ResultsPaths {
+  resultsPath: string;
+  interpretedResultsPath: string;
+}
+
+export interface SortedResultSetInfo {
+  resultsPath: string;
+  sortState: RawResultsSortState;
+}
+
+export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
+
+/**
+ * A message to indicate that the results are being updated.
+ *
+ * As a result of receiving this message, listeners might want to display a loading indicator.
+ */
+export interface ResultsUpdatingMsg {
+  t: 'resultsUpdating';
+}
+
+/**
+ * Message to set the initial state of the results view with a new
+ * query.
+ */
+export interface SetStateMsg {
+  t: 'setState';
+  resultsPath: string;
+  origResultsPaths: ResultsPaths;
+  sortedResultsMap: SortedResultsMap;
+  interpretation: undefined | Interpretation;
+  database: DatabaseInfo;
+  metadata?: QueryMetadata;
+  queryName: string;
+  queryPath: string;
+  /**
+   * Whether to keep displaying the old results while rendering the new results.
+   *
+   * This is useful to prevent properties like scroll state being lost when rendering the sorted results after sorting a column.
+   */
+  shouldKeepOldResultsWhileRendering: boolean;
+
+  /**
+   * An experimental way of providing results from the extension.
+   * Should be in the WebviewParsedResultSets branch of the type
+   * unless config.EXPERIMENTAL_BQRS_SETTING is set to true.
+   */
+  parsedResultSets: ParsedResultSets;
+}
+
+/**
+ * Message indicating that the results view should display interpreted
+ * results.
+ */
+export interface ShowInterpretedPageMsg {
+  t: 'showInterpretedPage';
+  interpretation: Interpretation;
+  database: DatabaseInfo;
+  metadata?: QueryMetadata;
+  pageNumber: number;
+  numPages: number;
+  pageSize: number;
+  resultSetNames: string[];
+  queryName: string;
+  queryPath: string;
+}
+
+/** Advance to the next or previous path no in the path viewer */
+export interface NavigatePathMsg {
+  t: 'navigatePath';
+
+  /** 1 for next, -1 for previous */
+  direction: number;
+}
+
+/**
+ * A message indicating that the results view should untoggle the
+ * "Show results in Problems view" checkbox.
+ */
+export interface UntoggleShowProblemsMsg {
+  t: 'untoggleShowProblems';
+}
+
+/**
+ * A message sent into the results view.
+ */
+export type IntoResultsViewMsg =
+  | ResultsUpdatingMsg
+  | SetStateMsg
+  | ShowInterpretedPageMsg
+  | NavigatePathMsg
+  | UntoggleShowProblemsMsg;
+
+/**
+ * A message sent from the results view.
+ */
+export type FromResultsViewMsg =
+  | ViewSourceFileMsg
+  | ToggleDiagnostics
+  | ChangeRawResultsSortMsg
+  | ChangeInterpretedResultsSortMsg
+  | ResultViewLoaded
+  | ChangePage
+  | OpenFileMsg;
+
+/**
+ * Message from the results view to open a database source
+ * file at the provided location.
+ */
+export interface ViewSourceFileMsg {
+  t: 'viewSourceFile';
+  loc: ResolvableLocationValue;
+  databaseUri: string;
+}
+
+/**
+ * Message from the results view to open a file in an editor.
+ */
+export interface OpenFileMsg {
+  t: 'openFile';
+  /* Full path to the file to open. */
+  filePath: string;
+}
+
+/**
+ * Message from the results view to toggle the display of
+ * query diagnostics.
+ */
+interface ToggleDiagnostics {
+  t: 'toggleDiagnostics';
+  databaseUri: string;
+  metadata?: QueryMetadata;
+  origResultsPaths: ResultsPaths;
+  visible: boolean;
+  kind?: string;
+}
+
+/**
+ * Message from the results view to signal that loading the results
+ * is complete.
+ */
+interface ResultViewLoaded {
+  t: 'resultViewLoaded';
+}
+
+/**
+ * Message from the results view to signal a request to change the
+ * page.
+ */
+interface ChangePage {
+  t: 'changePage';
+  pageNumber: number; // 0-indexed, displayed to the user as 1-indexed
+  selectedTable: string;
+}
+
+export enum SortDirection {
+  asc,
+  desc,
+}
+
+export interface RawResultsSortState {
+  columnIndex: number;
+  sortDirection: SortDirection;
+}
+
+export type InterpretedResultsSortColumn = 'alert-message';
+
+export interface InterpretedResultsSortState {
+  sortBy: InterpretedResultsSortColumn;
+  sortDirection: SortDirection;
+}
+
+/**
+ * Message from the results view to request a sorting change.
+ */
+interface ChangeRawResultsSortMsg {
+  t: 'changeSort';
+  resultSetName: string;
+  /**
+   * sortState being undefined means don't sort, just present results in the order
+   * they appear in the sarif file.
+   */
+  sortState?: RawResultsSortState;
+}
+
+/**
+ * Message from the results view to request a sorting change in interpreted results.
+ */
+interface ChangeInterpretedResultsSortMsg {
+  t: 'changeInterpretedSort';
+  /**
+   * sortState being undefined means don't sort, just present results in the order
+   * they appear in the sarif file.
+   */
+  sortState?: InterpretedResultsSortState;
+}
+
+/**
+ * Message from the compare view to the extension.
+ */
+export type FromCompareViewMessage =
+  | CompareViewLoadedMessage
+  | ChangeCompareMessage
+  | ViewSourceFileMsg
+  | OpenQueryMessage;
+
+/**
+ * Message from the compare view to signal the completion of loading results.
+ */
+interface CompareViewLoadedMessage {
+  t: 'compareViewLoaded';
+}
+
+/**
+ * Message from the compare view to request opening a query.
+ */
+export interface OpenQueryMessage {
+  readonly t: 'openQuery';
+  readonly kind: 'from' | 'to';
+}
+
+/**
+ * Message from the compare view to request changing the result set to compare.
+ */
+interface ChangeCompareMessage {
+  t: 'changeCompare';
+  newResultSetName: string;
+}
+
+export type ToCompareViewMessage = SetComparisonsMessage;
+
+/**
+ * Message to the compare view that specifies the query results to compare.
+ */
+export interface SetComparisonsMessage {
+  readonly t: 'setComparisons';
+  readonly stats: {
+    fromQuery?: {
+      name: string;
+      status: string;
+      time: string;
+    };
+    toQuery?: {
+      name: string;
+      status: string;
+      time: string;
+    };
+  };
+  readonly columns: readonly Column[];
+  readonly commonResultSetNames: string[];
+  readonly currentResultSetName: string;
+  readonly rows: QueryCompareResult | undefined;
+  readonly message: string | undefined;
+  readonly datebaseUri: string;
+}
+
+export enum DiffKind {
+  Add = 'Add',
+  Remove = 'Remove',
+  Change = 'Change',
+}
+
+/**
+ * from is the set of rows that have changes in the "from" query.
+ * to is the set of rows that have changes in the "to" query.
+ * They are in the same order, so element 1 in "from" corresponds to
+ * element 1 in "to".
+ *
+ * If an array element is null, that means that the element was removed
+ * (or added) in the comparison.
+ */
+export type QueryCompareResult = {
+  from: ResultRow[];
+  to: ResultRow[];
+};
+
+/**
+ * Extract the name of the default result. Prefer returning
+ * 'alerts', or '#select'. Otherwise return the first in the list.
+ *
+ * Note that this is the only function in this module. It must be
+ * placed here since it is shared across the webview boundary.
+ *
+ * We should consider moving to a separate module to ensure this
+ * one is types only.
+ *
+ * @param resultSetNames
+ */
+export function getDefaultResultSetName(
+  resultSetNames: readonly string[]
+): string {
+  // Choose first available result set from the array
+  return [
+    ALERTS_TABLE_NAME,
+    SELECT_TABLE_NAME,
+    resultSetNames[0],
+  ].filter((resultSetName) => resultSetNames.includes(resultSetName))[0];
+}
+
+export interface ParsedResultSets {
+  pageNumber: number;
+  pageSize: number;
+  numPages: number;
+  numInterpretedPages: number;
+  selectedTable?: string; // when undefined, means 'show default table'
+  resultSetNames: string[];
+  resultSet: ResultSet;
+}

extensions/ql-vscode/src/pure/messages.ts

@@ -150,6 +150,11 @@ export interface CompilationOptions {
    * Whether to disable toString values in the results.
    */
   noComputeToString: boolean;
+  /**
+   * Whether to ensure that elements that do not have a displayString
+   * get reported anyway. Useful for universal compilation options.
+   */
+  computeDefaultStrings: boolean;
 }
 
 /**
@@ -837,7 +842,6 @@ export interface RunUpgradeParams {
   toRun: CompiledUpgrades;
 }
 
-
 /**
  * The result of running an upgrade
  */
@@ -857,6 +861,21 @@ export interface RunUpgradeResult {
   finalSha: string;
 }
 
+export interface RegisterDatabasesParams {
+  databases: Dataset[];
+}
+
+export interface DeregisterDatabasesParams {
+  databases: Dataset[];
+}
+
+export type RegisterDatabasesResult = {
+  registeredDatabases: Dataset[];
+};
+
+export type DeregisterDatabasesResult = {
+  registeredDatabases: Dataset[];
+};
 
 /**
  * Type for any action that could have progress messages.
@@ -934,6 +953,20 @@ export const runQueries = new rpc.RequestType<WithProgressId<EvaluateQueriesPara
  */
 export const runUpgrade = new rpc.RequestType<WithProgressId<RunUpgradeParams>, RunUpgradeResult, void, void>('evaluation/runUpgrade');
 
+export const registerDatabases = new rpc.RequestType<
+  WithProgressId<RegisterDatabasesParams>,
+  RegisterDatabasesResult,
+  void,
+  void
+>('evaluation/registerDatabases');
+
+export const deregisterDatabases = new rpc.RequestType<
+  WithProgressId<DeregisterDatabasesParams>,
+  DeregisterDatabasesResult,
+  void,
+  void
+>('evaluation/deregisterDatabases');
+
 /**
  * Request returned to the client to notify completion of a query.
  * The full runQueries job is completed when all queries are acknowledged.

extensions/ql-vscode/src/pure/sarif-utils.ts

@@ -1,21 +1,28 @@
-import * as Sarif from "sarif";
-import * as path from "path";
-import { LocationStyle, ResolvableLocationValue } from "semmle-bqrs";
+import * as Sarif from 'sarif';
+import { ResolvableLocationValue } from './bqrs-cli-types';
 
 export interface SarifLink {
   dest: number;
   text: string;
 }
 
+// The type of a result that has no associated location.
+// hint is a string intended for display to the user
+// that explains why there is no location.
+interface NoLocation {
+  hint: string;
+}
 
 type ParsedSarifLocation =
-  | ResolvableLocationValue
-  // Resolvable locations have a `file` field, but it will sometimes include
+  | (ResolvableLocationValue & {
+
+    userVisibleFile: string;
+  })
+  // Resolvable locations have a `uri` field, but it will sometimes include
   // a source location prefix, which contains build-specific information the user
   // doesn't really need to see. We ensure that `userVisibleFile` will not contain
   // that, and is appropriate for display in the UI.
-  & { userVisibleFile: string }
-  | { t: 'NoLocation'; hint: string };
+  | NoLocation;
 
 export type SarifMessageComponent = string | SarifLink
 
@@ -23,7 +30,10 @@ export type SarifMessageComponent = string | SarifLink
  * Unescape "[", "]" and "\\" like in sarif plain text messages
  */
 export function unescapeSarifText(message: string): string {
-  return message.replace(/\\\[/g, "[").replace(/\\\]/g, "]").replace(/\\\\/, "\\");
+  return message
+    .replace(/\\\[/g, '[')
+    .replace(/\\\]/g, ']')
+    .replace(/\\\\/g, '\\');
 }
 
 export function parseSarifPlainTextMessage(message: string): SarifMessageComponent[] {
@@ -38,8 +48,8 @@ export function parseSarifPlainTextMessage(message: string): SarifMessageCompone
   let curIndex = 0;
   while ((result = linkRegex.exec(message)) !== null) {
     results.push(unescapeSarifText(message.substring(curIndex, result.index)));
-    const linkText = result.groups!["linkText"];
-    const linkTarget = +result.groups!["linkTarget"];
+    const linkText = result.groups!['linkText'];
+    const linkTarget = +result.groups!['linkTarget'];
     results.push({ dest: linkTarget, text: unescapeSarifText(linkText) });
     curIndex = result.index + result[0].length;
   }
@@ -54,71 +64,98 @@ export function parseSarifPlainTextMessage(message: string): SarifMessageCompone
  * @param sourceLocationPrefix The source location prefix of a database. May be
  * unix style `/foo/bar/baz` or windows-style `C:\foo\bar\baz`.
  * @param sarifRelativeUri A uri relative to sourceLocationPrefix.
- * @returns A string that is valid for the `.file` field of a `FivePartLocation`:
+ *
+ * @returns A URI string that is valid for the `.file` field of a `FivePartLocation`:
  * directory separators are normalized, but drive letters `C:` may appear.
  */
-export function getPathRelativeToSourceLocationPrefix(sourceLocationPrefix: string, sarifRelativeUui: string) {
-  const normalizedSourceLocationPrefix = sourceLocationPrefix.replace(/\\/g, '/');
-  return path.join(normalizedSourceLocationPrefix, decodeURIComponent(sarifRelativeUui));
+export function getPathRelativeToSourceLocationPrefix(
+  sourceLocationPrefix: string,
+  sarifRelativeUri: string
+) {
+  // convert a platform specific path into encoded path uri segments
+  // need to be careful about drive letters and ensure that there
+  // is a starting '/'
+  let prefix = '';
+  if (sourceLocationPrefix[1] === ':') {
+    // assume this is a windows drive separator
+    prefix = sourceLocationPrefix.substring(0, 2);
+    sourceLocationPrefix = sourceLocationPrefix.substring(2);
+  }
+  const normalizedSourceLocationPrefix = prefix + sourceLocationPrefix.replace(/\\/g, '/')
+    .split('/')
+    .map(encodeURIComponent)
+    .join('/');
+  const slashPrefix = normalizedSourceLocationPrefix.startsWith('/') ? '' : '/';
+  return `file:${slashPrefix + normalizedSourceLocationPrefix}/${sarifRelativeUri}`;
 }
 
-export function parseSarifLocation(loc: Sarif.Location, sourceLocationPrefix: string): ParsedSarifLocation {
+/**
+ *
+ * @param loc specifies the database-relative location of the source location
+ * @param sourceLocationPrefix a file path (usually a full path) to the database containing the source location.
+ */
+export function parseSarifLocation(
+  loc: Sarif.Location,
+  sourceLocationPrefix: string
+): ParsedSarifLocation {
   const physicalLocation = loc.physicalLocation;
   if (physicalLocation === undefined)
-    return { t: 'NoLocation', hint: 'no physical location' };
+    return { hint: 'no physical location' };
   if (physicalLocation.artifactLocation === undefined)
-    return { t: 'NoLocation', hint: 'no artifact location' };
+    return { hint: 'no artifact location' };
   if (physicalLocation.artifactLocation.uri === undefined)
-    return { t: 'NoLocation', hint: 'artifact location has no uri' };
+    return { hint: 'artifact location has no uri' };
 
   // This is not necessarily really an absolute uri; it could either be a
   // file uri or a relative uri.
   const uri = physicalLocation.artifactLocation.uri;
 
   const fileUriRegex = /^file:/;
-  const effectiveLocation = uri.match(fileUriRegex) ?
-    decodeURIComponent(uri.replace(fileUriRegex, '')) :
-    getPathRelativeToSourceLocationPrefix(sourceLocationPrefix, uri);
-  const userVisibleFile = uri.match(fileUriRegex) ?
-    decodeURIComponent(uri.replace(fileUriRegex, '')) :
-    uri;
+  const hasFilePrefix = uri.match(fileUriRegex);
+  const effectiveLocation = hasFilePrefix
+    ? uri
+    : getPathRelativeToSourceLocationPrefix(sourceLocationPrefix, uri);
+  const userVisibleFile = decodeURIComponent(hasFilePrefix
+    ? uri.replace(fileUriRegex, '')
+    : uri);
 
   if (physicalLocation.region === undefined) {
     // If the region property is absent, the physicalLocation object refers to the entire file.
     // Source: https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Toc16012638.
-    // TODO: Do we get here if we provide a non-filesystem URL?
     return {
-      t: LocationStyle.WholeFile,
-      file: effectiveLocation,
-      userVisibleFile,
-    };
+      uri: effectiveLocation,
+      userVisibleFile
+    } as ParsedSarifLocation;
   } else {
     const region = physicalLocation.region;
     // We assume that the SARIF we're given always has startLine
     // This is not mandated by the SARIF spec, but should be true of
     // SARIF output by our own tools.
-    const lineStart = region.startLine!;
+    const startLine = region.startLine!;
 
     // These defaults are from SARIF 2.1.0 spec, section 3.30.2, "Text Regions"
     // https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Ref493492556
-    const lineEnd = region.endLine === undefined ? lineStart : region.endLine;
-    const colStart = region.startColumn === undefined ? 1 : region.startColumn;
+    const endLine = region.endLine === undefined ? startLine : region.endLine;
+    const startColumn = region.startColumn === undefined ? 1 : region.startColumn;
 
     // We also assume that our tools will always supply `endColumn` field, which is
     // fortunate, since the SARIF spec says that it defaults to the end of the line, whose
     // length we don't know at this point in the code.
     //
     // It is off by one with respect to the way vscode counts columns in selections.
-    const colEnd = region.endColumn! - 1;
+    const endColumn = region.endColumn! - 1;
 
     return {
-      t: LocationStyle.FivePart,
-      file: effectiveLocation,
+      uri: effectiveLocation,
       userVisibleFile,
-      lineStart,
-      colStart,
-      lineEnd,
-      colEnd,
+      startLine,
+      startColumn,
+      endLine,
+      endColumn,
     };
   }
 }
+
+export function isNoLocation(loc: ParsedSarifLocation): loc is NoLocation {
+  return 'hint' in loc;
+}

extensions/ql-vscode/src/qlpack-discovery.ts

@@ -1,59 +0,0 @@
-import { EventEmitter, Event, Uri, WorkspaceFolder, RelativePattern } from 'vscode';
-import { MultiFileSystemWatcher } from 'semmle-vscode-utils';
-import { CodeQLCliServer, QlpacksInfo } from './cli';
-import { Discovery } from './discovery';
-
-export interface QLPack {
-  name: string;
-  uri: Uri;
-}
-
-/**
- * Service to discover all available QL packs in a workspace folder.
- */
-export class QLPackDiscovery extends Discovery<QlpacksInfo> {
-  private readonly _onDidChangeQLPacks = this.push(new EventEmitter<void>());
-  private readonly watcher = this.push(new MultiFileSystemWatcher());
-  private _qlPacks: readonly QLPack[] = [];
-
-  constructor(private readonly workspaceFolder: WorkspaceFolder,
-    private readonly cliServer: CodeQLCliServer) {
-
-    super();
-
-    // Watch for any changes to `qlpack.yml` files in this workspace folder.
-    // TODO: The CLI server should tell us what paths to watch for.
-    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/qlpack.yml'));
-    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/.codeqlmanifest.json'));
-    this.push(this.watcher.onDidChange(this.handleQLPackFileChanged, this));
-
-    this.refresh();
-  }
-
-  public get onDidChangeQLPacks(): Event<void> { return this._onDidChangeQLPacks.event; }
-
-  public get qlPacks(): readonly QLPack[] { return this._qlPacks; }
-
-  private handleQLPackFileChanged(_uri: Uri): void {
-    this.refresh();
-  }
-
-  protected discover(): Promise<QlpacksInfo> {
-    // Only look for QL packs in this workspace folder.
-    return this.cliServer.resolveQlpacks([this.workspaceFolder.uri.fsPath], []);
-  }
-
-  protected update(results: QlpacksInfo): void {
-    const qlPacks: QLPack[] = [];
-    for (const id in results) {
-      qlPacks.push(...results[id].map(fsPath => {
-        return {
-          name: id,
-          uri: Uri.file(fsPath)
-        };
-      }));
-    }
-    this._qlPacks = qlPacks;
-    this._onDidChangeQLPacks.fire();
-  }
-}

extensions/ql-vscode/src/qltest-discovery.ts

@@ -1,9 +1,9 @@
 import * as path from 'path';
-import { QLPackDiscovery } from './qlpack-discovery';
 import { Discovery } from './discovery';
-import { EventEmitter, Event, Uri, RelativePattern } from 'vscode';
-import { MultiFileSystemWatcher } from 'semmle-vscode-utils';
+import { EventEmitter, Event, Uri, RelativePattern, WorkspaceFolder, env } from 'vscode';
+import { MultiFileSystemWatcher } from './vscode-utils/multi-file-system-watcher';
 import { CodeQLCliServer } from './cli';
+import * as fs from 'fs-extra';
 
 /**
  * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`.
@@ -29,9 +29,8 @@ export abstract class QLTestNode {
  * A directory containing one or more QL tests or other test directories.
  */
 export class QLTestDirectory extends QLTestNode {
-  private _children: QLTestNode[] = [];
 
-  constructor(_path: string, _name: string) {
+  constructor(_path: string, _name: string, private _children: QLTestNode[] = []) {
     super(_path, _name);
   }
 
@@ -55,10 +54,23 @@ export class QLTestDirectory extends QLTestNode {
   }
 
   public finish(): void {
-    this._children.sort((a, b) => a.name.localeCompare(b.name));
-    for (const child of this._children) {
+    // remove empty directories
+    this._children.filter(child =>
+      child instanceof QLTestFile || child.children.length > 0
+    );
+    this._children.sort((a, b) => a.name.localeCompare(b.name, env.language));
+    this._children.forEach((child, i) => {
       child.finish();
-    }
+      if (child.children?.length === 1 && child.children[0] instanceof QLTestDirectory) {
+        // collapse children
+        const replacement = new QLTestDirectory(
+          child.children[0].path,
+          child.name + ' / ' + child.children[0].name,
+          Array.from(child.children[0].children)
+        );
+        this._children[i] = replacement;
+      }
+    });
   }
 
   private createChildDirectory(name: string): QLTestDirectory {
@@ -96,14 +108,15 @@ export class QLTestFile extends QLTestNode {
  */
 interface QLTestDiscoveryResults {
   /**
-   * The root test directory for each QL pack that contains tests.
+   * A directory that contains one or more QL Tests, or other QLTestDirectories.
    */
-  testDirectories: QLTestDirectory[];
+  testDirectory: QLTestDirectory | undefined;
+
   /**
-   * The list of file system paths to watch. If any of these paths changes, the discovery results
-   * may be out of date.
+   * The file system path to a directory to watch. If any ql or qlref file changes in
+   * this directory, then this signifies a change in tests.
    */
-  watchPaths: string[];
+  watchPath: string;
 }
 
 /**
@@ -112,31 +125,30 @@ interface QLTestDiscoveryResults {
 export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
   private readonly _onDidChangeTests = this.push(new EventEmitter<void>());
   private readonly watcher: MultiFileSystemWatcher = this.push(new MultiFileSystemWatcher());
-  private _testDirectories: QLTestDirectory[] = [];
+  private _testDirectory: QLTestDirectory | undefined;
 
-  constructor(private readonly qlPackDiscovery: QLPackDiscovery,
-    private readonly cliServer: CodeQLCliServer) {
+  constructor(
+    private readonly workspaceFolder: WorkspaceFolder,
+    private readonly cliServer: CodeQLCliServer
+  ) {
+    super('QL Test Discovery');
 
-    super();
-
-    this.push(this.qlPackDiscovery.onDidChangeQLPacks(this.handleDidChangeQLPacks, this));
     this.push(this.watcher.onDidChange(this.handleDidChange, this));
-
-    this.refresh();
   }
 
   /**
    * Event to be fired when the set of discovered tests may have changed.
    */
-  public get onDidChangeTests(): Event<void> { return this._onDidChangeTests.event; }
+  public get onDidChangeTests(): Event<void> {
+    return this._onDidChangeTests.event;
+  }
 
   /**
-   * The root test directory for each QL pack that contains tests.
+   * The root directory. There is at least one test in this directory, or
+   * in a subdirectory of this.
    */
-  public get testDirectories(): QLTestDirectory[] { return this._testDirectories; }
-
-  private handleDidChangeQLPacks(): void {
-    this.refresh();
+  public get testDirectory(): QLTestDirectory | undefined {
+    return this._testDirectory;
   }
 
   private handleDidChange(uri: Uri): void {
@@ -144,55 +156,37 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
       this.refresh();
     }
   }
-
   protected async discover(): Promise<QLTestDiscoveryResults> {
-    const testDirectories: QLTestDirectory[] = [];
-    const watchPaths: string[] = [];
-    const qlPacks = this.qlPackDiscovery.qlPacks;
-    for (const qlPack of qlPacks) {
-      //HACK: Assume that only QL packs whose name ends with '-tests' contain tests.
-      if (qlPack.name.endsWith('-tests')) {
-        watchPaths.push(qlPack.uri.fsPath);
-        const testPackage = await this.discoverTests(qlPack.uri.fsPath, qlPack.name);
-        if (testPackage !== undefined) {
-          testDirectories.push(testPackage);
-        }
-      }
-    }
-
+    const testDirectory = await this.discoverTests();
     return {
-      testDirectories: testDirectories,
-      watchPaths: watchPaths
+      testDirectory,
+      watchPath: this.workspaceFolder.uri.fsPath
     };
   }
 
   protected update(results: QLTestDiscoveryResults): void {
-    this._testDirectories = results.testDirectories;
+    this._testDirectory = results.testDirectory;
 
     // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
     this.watcher.clear();
-    results.watchPaths.forEach(watchPath => {
-      this.watcher.addWatch(new RelativePattern(watchPath, '**/*.{ql,qlref}'));
-    });
+    this.watcher.addWatch(new RelativePattern(results.watchPath, '**/*.{ql,qlref}'));
     this._onDidChangeTests.fire();
   }
 
   /**
    * Discover all QL tests in the specified directory and its subdirectories.
-   * @param fullPath The full path of the test directory.
-   * @param name The display name to use for the returned `TestDirectory` object.
    * @returns A `QLTestDirectory` object describing the contents of the directory, or `undefined` if
    *   no tests were found.
    */
-  private async discoverTests(fullPath: string, name: string): Promise<QLTestDirectory | undefined> {
-    const resolvedTests = (await this.cliServer.resolveTests(fullPath))
-      .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
+  private async discoverTests(): Promise<QLTestDirectory> {
+    const fullPath = this.workspaceFolder.uri.fsPath;
+    const name = this.workspaceFolder.name;
+    const rootDirectory = new QLTestDirectory(fullPath, name);
 
-    if (resolvedTests.length === 0) {
-      return undefined;
-    }
-    else {
-      const rootDirectory = new QLTestDirectory(fullPath, name);
+    // Don't try discovery on workspace folders that don't exist on the filesystem
+    if ((await fs.pathExists(fullPath))) {
+      const resolvedTests = (await this.cliServer.resolveTests(fullPath))
+        .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
       for (const testPath of resolvedTests) {
         const relativePath = path.normalize(path.relative(fullPath, testPath));
         const dirName = path.dirname(relativePath);
@@ -201,9 +195,8 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
       }
 
       rootDirectory.finish();
-
-      return rootDirectory;
     }
+    return rootDirectory;
   }
 
   /**

extensions/ql-vscode/src/query-history.ts

@@ -1,12 +1,14 @@
 import * as path from 'path';
 import * as vscode from 'vscode';
-import { ExtensionContext, window as Window } from 'vscode';
+import { window as Window } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
 import * as helpers from './helpers';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
+import { QueryServerClient } from './queryserver-client';
+import { DisposableObject } from './vscode-utils/disposable-object';
 
 /**
  * query-history.ts
@@ -21,7 +23,7 @@ export type QueryHistoryItemOptions = {
   label?: string; // user-settable label
   queryText?: string; // text of the selected file
   isQuickQuery?: boolean;
-}
+};
 
 const SHOW_QUERY_TEXT_MSG = `\
 ////////////////////////////////////////////////////////////////////////////////////
@@ -53,31 +55,33 @@ const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
 /**
  * Tree data provider for the query history view.
  */
-class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery> {
+export class HistoryTreeDataProvider extends DisposableObject {
+  private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
 
-  /**
-   * XXX: This idiom for how to get a `.fire()`-able event emitter was
-   * cargo culted from another vscode extension. It seems rather
-   * involved and I hope there's something better that can be done
-   * instead.
-   */
-  private _onDidChangeTreeData: vscode.EventEmitter<CompletedQuery | undefined> = new vscode.EventEmitter<CompletedQuery | undefined>();
-  readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this._onDidChangeTreeData.event;
+  readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
+    ._onDidChangeTreeData.event;
 
   private history: CompletedQuery[] = [];
 
+  private failedIconPath: string;
+
   /**
    * When not undefined, must be reference-equal to an item in `this.databases`.
    */
   private current: CompletedQuery | undefined;
 
-  constructor(private ctx: ExtensionContext) {
+  constructor(extensionPath: string) {
+    super();
+    this.failedIconPath = path.join(
+      extensionPath,
+      FAILED_QUERY_HISTORY_ITEM_ICON
+    );
   }
 
   async getTreeItem(element: CompletedQuery): Promise<vscode.TreeItem> {
-    const it = new vscode.TreeItem(element.toString());
+    const treeItem = new vscode.TreeItem(element.toString());
 
-    it.command = {
+    treeItem.command = {
       title: 'Query History Item',
       command: 'codeQLQueryHistory.itemClicked',
       arguments: [element],
@@ -86,22 +90,22 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
     // Mark this query history item according to whether it has a
     // SARIF file so that we can make context menu items conditionally
     // available.
-    it.contextValue = await element.query.hasInterpretedResults() ? 'interpretedResultsItem' : 'rawResultsItem';
+    const hasResults = await element.query.hasInterpretedResults();
+    treeItem.contextValue = hasResults
+      ? 'interpretedResultsItem'
+      : 'rawResultsItem';
 
     if (!element.didRunSuccessfully) {
-      it.iconPath = path.join(this.ctx.extensionPath, FAILED_QUERY_HISTORY_ITEM_ICON);
+      treeItem.iconPath = this.failedIconPath;
     }
 
-    return it;
+    return treeItem;
   }
 
-  getChildren(element?: CompletedQuery): vscode.ProviderResult<CompletedQuery[]> {
-    if (element == undefined) {
-      return this.history;
-    }
-    else {
-      return [];
-    }
+  getChildren(
+    element?: CompletedQuery
+  ): vscode.ProviderResult<CompletedQuery[]> {
+    return element ? [] : this.history;
   }
 
   getParent(_element: CompletedQuery): vscode.ProviderResult<CompletedQuery> {
@@ -112,7 +116,7 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
     return this.current;
   }
 
-  push(item: CompletedQuery): void {
+  pushQuery(item: CompletedQuery): void {
     this.current = item;
     this.history.push(item);
     this.refresh();
@@ -123,9 +127,8 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
   }
 
   remove(item: CompletedQuery) {
-    if (this.current === item)
-      this.current = undefined;
-    const index = this.history.findIndex(i => i === item);
+    if (this.current === item) this.current = undefined;
+    const index = this.history.findIndex((i) => i === item);
     if (index >= 0) {
       this.history.splice(index, 1);
       if (this.current === undefined && this.history.length > 0) {
@@ -137,8 +140,16 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
     }
   }
 
-  refresh() {
-    this._onDidChangeTreeData.fire();
+  get allHistory(): CompletedQuery[] {
+    return this.history;
+  }
+
+  refresh(completedQuery?: CompletedQuery) {
+    this._onDidChangeTreeData.fire(completedQuery);
+  }
+
+  find(queryId: number): CompletedQuery | undefined {
+    return this.allHistory.find((query) => query.query.queryID === queryId);
   }
 }
 
@@ -148,12 +159,128 @@ class HistoryTreeDataProvider implements vscode.TreeDataProvider<CompletedQuery>
  */
 const DOUBLE_CLICK_TIME = 500;
 
-export class QueryHistoryManager {
+const NO_QUERY_SELECTED = 'No query selected. Select a query history item you have already run and try again.';
+export class QueryHistoryManager extends DisposableObject {
   treeDataProvider: HistoryTreeDataProvider;
-  ctx: ExtensionContext;
   treeView: vscode.TreeView<CompletedQuery>;
-  selectedCallback: ((item: CompletedQuery) => void) | undefined;
   lastItemClick: { time: Date; item: CompletedQuery } | undefined;
+  compareWithItem: CompletedQuery | undefined;
+
+  constructor(
+    private qs: QueryServerClient,
+    extensionPath: string,
+    private queryHistoryConfigListener: QueryHistoryConfig,
+    private selectedCallback: (item: CompletedQuery) => Promise<void>,
+    private doCompareCallback: (
+      from: CompletedQuery,
+      to: CompletedQuery
+    ) => Promise<void>
+  ) {
+    super();
+
+    const treeDataProvider = (this.treeDataProvider = new HistoryTreeDataProvider(
+      extensionPath
+    ));
+    this.treeView = Window.createTreeView('codeQLQueryHistory', {
+      treeDataProvider,
+      canSelectMany: true,
+    });
+    this.push(this.treeView);
+    this.push(treeDataProvider);
+
+    // Lazily update the tree view selection due to limitations of TreeView API (see
+    // `updateTreeViewSelectionIfVisible` doc for details)
+    this.push(
+      this.treeView.onDidChangeVisibility(async (_ev) =>
+        this.updateTreeViewSelectionIfVisible()
+      )
+    );
+    // Don't allow the selection to become empty
+    this.push(
+      this.treeView.onDidChangeSelection(async (ev) => {
+        if (ev.selection.length == 0) {
+          this.updateTreeViewSelectionIfVisible();
+        }
+        this.updateCompareWith(ev.selection);
+      })
+    );
+
+    logger.log('Registering query history panel commands.');
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.openQuery',
+        this.handleOpenQuery.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.removeHistoryItem',
+        this.handleRemoveHistoryItem.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.setLabel',
+        this.handleSetLabel.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.compareWith',
+        this.handleCompareWith.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.showQueryLog',
+        this.handleShowQueryLog.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.showQueryText',
+        this.handleShowQueryText.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.viewSarif',
+        this.handleViewSarif.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.viewDil',
+        this.handleViewDil.bind(this)
+      )
+    );
+    this.push(
+      helpers.commandRunner(
+        'codeQLQueryHistory.itemClicked',
+        async (item: CompletedQuery) => {
+          return this.handleItemClicked(item, [item]);
+        }
+      )
+    );
+    queryHistoryConfigListener.onDidChangeConfiguration(() => {
+      this.treeDataProvider.refresh();
+    });
+
+    // displays query text in a read-only document
+    vscode.workspace.registerTextDocumentContentProvider('codeql', {
+      provideTextDocumentContent(
+        uri: vscode.Uri
+      ): vscode.ProviderResult<string> {
+        const params = new URLSearchParams(uri.query);
+
+        return (
+          (JSON.parse(params.get('isQuickEval') || '')
+            ? SHOW_QUERY_TEXT_QUICK_EVAL_MSG
+            : SHOW_QUERY_TEXT_MSG) + params.get('queryText')
+        );
+      },
+    });
+  }
 
   async invokeCallbackOn(queryHistoryItem: CompletedQuery) {
     if (this.selectedCallback !== undefined) {
@@ -162,20 +289,49 @@ export class QueryHistoryManager {
     }
   }
 
-  async handleOpenQuery(queryHistoryItem: CompletedQuery): Promise<void> {
-    const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.query.program.queryPath));
-    const editor = await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
-    const queryText = queryHistoryItem.options.queryText;
-    if (queryText !== undefined && queryHistoryItem.options.isQuickQuery) {
-      await editor.edit(edit => edit.replace(textDocument.validateRange(
-        new vscode.Range(0, 0, textDocument.lineCount, 0)), queryText)
+  async handleOpenQuery(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ): Promise<void> {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+    if (!this.assertSingleQuery(finalMultiSelect)) {
+      return;
+    }
+
+    if (!finalSingleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
+    const textDocument = await vscode.workspace.openTextDocument(
+      vscode.Uri.file(finalSingleItem.query.program.queryPath)
+    );
+    const editor = await vscode.window.showTextDocument(
+      textDocument,
+      vscode.ViewColumn.One
+    );
+    const queryText = finalSingleItem.options.queryText;
+    if (queryText !== undefined && finalSingleItem.options.isQuickQuery) {
+      await editor.edit((edit) =>
+        edit.replace(
+          textDocument.validateRange(
+            new vscode.Range(0, 0, textDocument.lineCount, 0)
+          ),
+          queryText
+        )
       );
     }
   }
 
-  async handleRemoveHistoryItem(queryHistoryItem: CompletedQuery) {
-    this.treeDataProvider.remove(queryHistoryItem);
-    queryHistoryItem.dispose();
+  async handleRemoveHistoryItem(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+
+    (finalMultiSelect || [finalSingleItem]).forEach((item) => {
+      this.treeDataProvider.remove(item);
+      item.dispose();
+    });
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
       this.treeView.reveal(current);
@@ -183,99 +339,152 @@ export class QueryHistoryManager {
     }
   }
 
-  async handleSetLabel(queryHistoryItem: CompletedQuery) {
+  async handleSetLabel(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ): Promise<void> {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
     const response = await vscode.window.showInputBox({
       prompt: 'Label:',
       placeHolder: '(use default)',
-      value: queryHistoryItem.getLabel(),
+      value: singleItem.getLabel(),
     });
     // undefined response means the user cancelled the dialog; don't change anything
     if (response !== undefined) {
-      if (response === '')
-        // Interpret empty string response as "go back to using default"
-        queryHistoryItem.options.label = undefined;
-      else
-        queryHistoryItem.options.label = response;
-      this.treeDataProvider.refresh();
+      // Interpret empty string response as 'go back to using default'
+      singleItem.options.label = response === '' ? undefined : response;
+      this.treeDataProvider.refresh(singleItem);
     }
   }
 
-  async handleItemClicked(queryHistoryItem: CompletedQuery) {
-    this.treeDataProvider.setCurrentItem(queryHistoryItem);
+  async handleCompareWith(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    try {
+      if (!singleItem.didRunSuccessfully) {
+        throw new Error('Please select a successful query.');
+      }
+
+      const from = this.compareWithItem || singleItem;
+      const to = await this.findOtherQueryToCompare(from, multiSelect);
+
+      if (from && to) {
+        this.doCompareCallback(from, to);
+      }
+    } catch (e) {
+      helpers.showAndLogErrorMessage(e.message);
+    }
+  }
+
+  async handleItemClicked(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+    if (!this.assertSingleQuery(finalMultiSelect)) {
+      return;
+    }
+
+    if (!finalSingleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
+    this.treeDataProvider.setCurrentItem(finalSingleItem);
 
     const now = new Date();
     const prevItemClick = this.lastItemClick;
-    this.lastItemClick = { time: now, item: queryHistoryItem };
+    this.lastItemClick = { time: now, item: finalSingleItem };
 
-    if (prevItemClick !== undefined
-      && (now.valueOf() - prevItemClick.time.valueOf()) < DOUBLE_CLICK_TIME
-      && queryHistoryItem == prevItemClick.item) {
+    if (
+      prevItemClick !== undefined &&
+      now.valueOf() - prevItemClick.time.valueOf() < DOUBLE_CLICK_TIME &&
+      singleItem == prevItemClick.item
+    ) {
       // show original query file on double click
-      await this.handleOpenQuery(queryHistoryItem);
-    }
-    else {
+      await this.handleOpenQuery(singleItem, [singleItem]);
+    } else {
       // show results on single click
-      await this.invokeCallbackOn(queryHistoryItem);
+      await this.invokeCallbackOn(singleItem);
     }
   }
 
-  async handleShowQueryLog(queryHistoryItem: CompletedQuery) {
-    if (queryHistoryItem.logFileLocation) {
-      const uri = vscode.Uri.file(queryHistoryItem.logFileLocation);
-      try {
-        await vscode.window.showTextDocument(uri, {
-        });
-      } catch (e) {
-        if (e.message.includes('Files above 50MB cannot be synchronized with extensions')) {
-          const res = await helpers.showBinaryChoiceDialog('File is too large to open in the editor, do you want to open it externally?');
-          if (res) {
-            try {
-              await vscode.commands.executeCommand('revealFileInOS', uri);
-            } catch (e) {
-              helpers.showAndLogErrorMessage(e.message);
-            }
-          }
-        } else {
-          helpers.showAndLogErrorMessage(`Could not open log file ${queryHistoryItem.logFileLocation}`);
-          logger.log(e.message);
-          logger.log(e.stack);
-        }
+  async handleShowQueryLog(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
 
-      }
+    if (singleItem.logFileLocation) {
+      await this.tryOpenExternalFile(singleItem.logFileLocation);
     } else {
       helpers.showAndLogWarningMessage('No log file available');
     }
   }
 
-  async handleShowQueryText(queryHistoryItem: CompletedQuery) {
-    try {
-      const queryName = queryHistoryItem.queryName.endsWith('.ql') ? queryHistoryItem.queryName : queryHistoryItem.queryName + '.ql';
-      const params = new URLSearchParams({
-        isQuickEval: String(!!queryHistoryItem.query.quickEvalPosition),
-        queryText: await this.getQueryText(queryHistoryItem)
-      });
-      const uri = vscode.Uri.parse(`codeql:${queryHistoryItem.query.queryID}-${queryName}?${params.toString()}`);
-      const doc = await vscode.workspace.openTextDocument(uri);
-      await vscode.window.showTextDocument(doc, { preview: false });
-    } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+  async handleShowQueryText(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    if (!singleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
+    const queryName = singleItem.queryName.endsWith('.ql')
+      ? singleItem.queryName
+      : singleItem.queryName + '.ql';
+    const params = new URLSearchParams({
+      isQuickEval: String(!!singleItem.query.quickEvalPosition),
+      queryText: encodeURIComponent(await this.getQueryText(singleItem)),
+    });
+    const uri = vscode.Uri.parse(
+      `codeql:${singleItem.query.queryID}-${queryName}?${params.toString()}`, true
+    );
+    const doc = await vscode.workspace.openTextDocument(uri);
+    await vscode.window.showTextDocument(doc, { preview: false });
+  }
+
+  async handleViewSarif(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    const hasInterpretedResults = await singleItem.query.canHaveInterpretedResults();
+    if (hasInterpretedResults) {
+      await this.tryOpenExternalFile(
+        singleItem.query.resultsPaths.interpretedResultsPath
+      );
+    } else {
+      const label = singleItem.getLabel();
+      helpers.showAndLogInformationMessage(
+        `Query ${label} has no interpreted results.`
+      );
     }
   }
 
-  async handleViewSarif(queryHistoryItem: CompletedQuery) {
-    try {
-      const hasInterpretedResults = await queryHistoryItem.query.canHaveInterpretedResults();
-      if (hasInterpretedResults) {
-        const textDocument = await vscode.workspace.openTextDocument(vscode.Uri.file(queryHistoryItem.query.resultsPaths.interpretedResultsPath));
-        await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
-      }
-      else {
-        const label = queryHistoryItem.getLabel();
-        helpers.showAndLogInformationMessage(`Query ${label} has no interpreted results.`);
-      }
-    } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+  async handleViewDil(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[],
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
     }
+
+    await this.tryOpenExternalFile(
+      await singleItem.query.ensureDilPath(this.qs)
+    );
   }
 
   async getQueryText(queryHistoryItem: CompletedQuery): Promise<string> {
@@ -285,65 +494,28 @@ export class QueryHistoryManager {
       // capture all selected lines
       const startLine = queryHistoryItem.query.quickEvalPosition.line;
       const endLine = queryHistoryItem.query.quickEvalPosition.endLine;
-      const textDocument =
-        await vscode.workspace.openTextDocument(queryHistoryItem.query.quickEvalPosition.fileName);
-      return textDocument.getText(new vscode.Range(startLine - 1, 0, endLine, 0));
+      const textDocument = await vscode.workspace.openTextDocument(
+        queryHistoryItem.query.quickEvalPosition.fileName
+      );
+      return textDocument.getText(
+        new vscode.Range(startLine - 1, 0, endLine, 0)
+      );
     } else {
       return '';
     }
   }
 
-  constructor(
-    ctx: ExtensionContext,
-    private queryHistoryConfigListener: QueryHistoryConfig,
-    selectedCallback?: (item: CompletedQuery) => Promise<void>
-  ) {
-    this.ctx = ctx;
-    this.selectedCallback = selectedCallback;
-    const treeDataProvider = this.treeDataProvider = new HistoryTreeDataProvider(ctx);
-    this.treeView = Window.createTreeView('codeQLQueryHistory', { treeDataProvider });
-    // Lazily update the tree view selection due to limitations of TreeView API (see
-    // `updateTreeViewSelectionIfVisible` doc for details)
-    this.treeView.onDidChangeVisibility(async _ev => this.updateTreeViewSelectionIfVisible());
-    // Don't allow the selection to become empty
-    this.treeView.onDidChangeSelection(async ev => {
-      if (ev.selection.length == 0) {
-        this.updateTreeViewSelectionIfVisible();
-      }
-    });
-    logger.log('Registering query history panel commands.');
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.openQuery', this.handleOpenQuery));
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.removeHistoryItem', this.handleRemoveHistoryItem.bind(this)));
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.setLabel', this.handleSetLabel.bind(this)));
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryLog', this.handleShowQueryLog.bind(this)));
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.showQueryText', this.handleShowQueryText.bind(this)));
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.viewSarif', this.handleViewSarif.bind(this)));
-    ctx.subscriptions.push(vscode.commands.registerCommand('codeQLQueryHistory.itemClicked', async (item) => {
-      return this.handleItemClicked(item);
-    }));
-    queryHistoryConfigListener.onDidChangeQueryHistoryConfiguration(() => {
-      this.treeDataProvider.refresh();
-    });
-
-    // displays query text in a read-only document
-    vscode.workspace.registerTextDocumentContentProvider('codeql', {
-      provideTextDocumentContent(uri: vscode.Uri): vscode.ProviderResult<string> {
-        const params = new URLSearchParams(uri.query);
-
-        return (
-          JSON.parse(params.get('isQuickEval') || '') ? SHOW_QUERY_TEXT_QUICK_EVAL_MSG : SHOW_QUERY_TEXT_MSG
-        ) + params.get('queryText');
-      }
-    });
-  }
-
   addQuery(info: QueryWithResults): CompletedQuery {
     const item = new CompletedQuery(info, this.queryHistoryConfigListener);
-    this.treeDataProvider.push(item);
+    this.treeDataProvider.pushQuery(item);
     this.updateTreeViewSelectionIfVisible();
     return item;
   }
 
+  find(queryId: number): CompletedQuery | undefined {
+    return this.treeDataProvider.find(queryId);
+  }
+
   /**
    * Update the tree view selection if the tree view is visible.
    *
@@ -363,4 +535,152 @@ export class QueryHistoryManager {
       }
     }
   }
+
+  private async tryOpenExternalFile(fileLocation: string) {
+    const uri = vscode.Uri.file(fileLocation);
+    try {
+      await vscode.window.showTextDocument(uri, { preview: false });
+    } catch (e) {
+      if (
+        e.message.includes(
+          'Files above 50MB cannot be synchronized with extensions'
+        ) ||
+        e.message.includes('too large to open')
+      ) {
+        const res = await helpers.showBinaryChoiceDialog(
+          `VS Code does not allow extensions to open files >50MB. This file
+exceeds that limit. Do you want to open it outside of VS Code?
+
+You can also try manually opening it inside VS Code by selecting
+the file in the file explorer and dragging it into the workspace.`
+        );
+        if (res) {
+          try {
+            await vscode.commands.executeCommand('revealFileInOS', uri);
+          } catch (e) {
+            helpers.showAndLogErrorMessage(e.message);
+          }
+        }
+      } else {
+        helpers.showAndLogErrorMessage(`Could not open file ${fileLocation}`);
+        logger.log(e.message);
+        logger.log(e.stack);
+      }
+    }
+  }
+
+  private async findOtherQueryToCompare(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ): Promise<CompletedQuery | undefined> {
+    const dbName = singleItem.database.name;
+
+    // if exactly 2 queries are selected, use those
+    if (multiSelect?.length === 2) {
+      // return the query that is not the first selected one
+      const otherQuery =
+        singleItem === multiSelect[0] ? multiSelect[1] : multiSelect[0];
+      if (!otherQuery.didRunSuccessfully) {
+        throw new Error('Please select a successful query.');
+      }
+      if (otherQuery.database.name !== dbName) {
+        throw new Error('Query databases must be the same.');
+      }
+      return otherQuery;
+    }
+
+    if (multiSelect?.length > 1) {
+      throw new Error('Please select no more than 2 queries.');
+    }
+
+    // otherwise, let the user choose
+    const comparableQueryLabels = this.treeDataProvider.allHistory
+      .filter(
+        (otherQuery) =>
+          otherQuery !== singleItem &&
+          otherQuery.didRunSuccessfully &&
+          otherQuery.database.name === dbName
+      )
+      .map((otherQuery) => ({
+        label: otherQuery.toString(),
+        description: otherQuery.databaseName,
+        detail: otherQuery.statusString,
+        query: otherQuery,
+      }));
+    if (comparableQueryLabels.length < 1) {
+      throw new Error('No other queries available to compare with.');
+    }
+    const choice = await vscode.window.showQuickPick(comparableQueryLabels);
+    return choice?.query;
+  }
+
+  private assertSingleQuery(multiSelect: CompletedQuery[] = [], message = 'Please select a single query.') {
+    if (multiSelect.length > 1) {
+      helpers.showAndLogErrorMessage(
+        message
+      );
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Updates the compare with source query. This ensures that all compare command invocations
+   * when exactly 2 queries are selected always have the proper _from_ query. Always use
+   * compareWithItem as the _from_ query.
+   *
+   * The heuristic is this:
+   *
+   * 1. If selection is empty or has length > 2 delete compareWithItem.
+   * 2. If selection is length 1, then set that item to compareWithItem.
+   * 3. If selection is length 2, then make sure compareWithItem is one of the selected items
+   *    if not, then delete compareWithItem. If it is then, do nothing.
+   *
+   * This ensures that compareWithItem is always the first item selected if there are only
+   * two selected items.
+   *
+   * @param newSelection the new selection after the most recent selection change
+   */
+  private updateCompareWith(newSelection: CompletedQuery[]) {
+    if (newSelection.length === 1) {
+      this.compareWithItem = newSelection[0];
+    } else if (
+      newSelection.length !== 2 ||
+      !this.compareWithItem ||
+      !newSelection.includes(this.compareWithItem)
+    ) {
+      this.compareWithItem = undefined;
+    }
+  }
+
+  /**
+   * If no items are selected, attempt to grab the selection from the treeview.
+   * We need to use this method because when clicking on commands from the view title
+   * bar, the selections are not passed in.
+   *
+   * @param singleItem the single item selected, or undefined if no item is selected
+   * @param multiSelect a multi-select or undefined if no items are selected
+   */
+  private determineSelection(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ): { finalSingleItem: CompletedQuery; finalMultiSelect: CompletedQuery[] } {
+    if (singleItem === undefined && (multiSelect === undefined || multiSelect.length === 0 || multiSelect[0] === undefined)) {
+      const selection = this.treeView.selection;
+      if (selection) {
+        return {
+          finalSingleItem: selection[0],
+          finalMultiSelect: selection
+        };
+      }
+    }
+    return {
+      finalSingleItem: singleItem,
+      finalMultiSelect: multiSelect
+    };
+  }
+
+  async refreshTreeView(completedQuery: CompletedQuery): Promise<void> {
+    this.treeDataProvider.refresh(completedQuery);
+  }
 }

extensions/ql-vscode/src/query-results.ts

@@ -1,13 +1,14 @@
-import { QueryWithResults, tmpDir, QueryInfo } from "./run-queries";
-import * as messages from './messages';
-import * as helpers from './helpers';
+import { env } from 'vscode';
+
+import { QueryWithResults, tmpDir, QueryInfo } from './run-queries';
+import * as messages from './pure/messages';
 import * as cli from './cli';
 import * as sarif from 'sarif';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState, ResultsPaths } from "./interface-types";
-import { QueryHistoryConfig } from "./config";
-import { QueryHistoryItemOptions } from "./query-history";
+import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState, ResultsPaths } from './pure/interface-types';
+import { QueryHistoryConfig } from './config';
+import { QueryHistoryItemOptions } from './query-history';
 
 export class CompletedQuery implements QueryWithResults {
   readonly time: string;
@@ -43,7 +44,7 @@ export class CompletedQuery implements QueryWithResults {
     this.options = evaluation.options;
     this.dispose = evaluation.dispose;
 
-    this.time = new Date().toLocaleString();
+    this.time = new Date().toLocaleString(env.language);
     this.sortedResultsInfo = new Map();
   }
 
@@ -51,7 +52,7 @@ export class CompletedQuery implements QueryWithResults {
     return this.database.name;
   }
   get queryName(): string {
-    return helpers.getQueryName(this.query);
+    return getQueryName(this.query);
   }
 
   get statusString(): string {
@@ -59,7 +60,7 @@ export class CompletedQuery implements QueryWithResults {
       case messages.QueryResultType.CANCELLATION:
         return `cancelled after ${this.result.evaluationTime / 1000} seconds`;
       case messages.QueryResultType.OOM:
-        return `out of memory`;
+        return 'out of memory';
       case messages.QueryResultType.SUCCESS:
         return `finished in ${this.result.evaluationTime / 1000} seconds`;
       case messages.QueryResultType.TIMEOUT:
@@ -70,6 +71,13 @@ export class CompletedQuery implements QueryWithResults {
     }
   }
 
+  getResultsPath(selectedTable: string, useSorted = true): string {
+    if (!useSorted) {
+      return this.query.resultsPaths.resultsPath;
+    }
+    return this.sortedResultsInfo.get(selectedTable)?.resultsPath
+      || this.query.resultsPaths.resultsPath;
+  }
 
   interpolate(template: string): string {
     const { databaseName, queryName, time, statusString } = this;
@@ -87,9 +95,8 @@ export class CompletedQuery implements QueryWithResults {
   }
 
   getLabel(): string {
-    if (this.options.label !== undefined)
-      return this.options.label;
-    return this.config.format;
+    return this.options?.label
+      || this.config.format;
   }
 
   get didRunSuccessfully(): boolean {
@@ -100,7 +107,11 @@ export class CompletedQuery implements QueryWithResults {
     return this.interpolate(this.getLabel());
   }
 
-  async updateSortState(server: cli.CodeQLCliServer, resultSetName: string, sortState: RawResultsSortState | undefined): Promise<void> {
+  async updateSortState(
+    server: cli.CodeQLCliServer,
+    resultSetName: string,
+    sortState?: RawResultsSortState
+  ): Promise<void> {
     if (sortState === undefined) {
       this.sortedResultsInfo.delete(resultSetName);
       return;
@@ -111,19 +122,50 @@ export class CompletedQuery implements QueryWithResults {
       sortState
     };
 
-    await server.sortBqrs(this.query.resultsPaths.resultsPath, sortedResultSetInfo.resultsPath, resultSetName, [sortState.columnIndex], [sortState.sortDirection]);
+    await server.sortBqrs(
+      this.query.resultsPaths.resultsPath,
+      sortedResultSetInfo.resultsPath,
+      resultSetName,
+      [sortState.columnIndex],
+      [sortState.sortDirection]
+    );
     this.sortedResultsInfo.set(resultSetName, sortedResultSetInfo);
   }
 
-  async updateInterpretedSortState(_server: cli.CodeQLCliServer, sortState: InterpretedResultsSortState | undefined): Promise<void> {
+  async updateInterpretedSortState(sortState?: InterpretedResultsSortState): Promise<void> {
     this.interpretedResultsSortState = sortState;
   }
 }
 
+
+/**
+ * Gets a human-readable name for an evaluated query.
+ * Uses metadata if it exists, and defaults to the query file name.
+ */
+export function getQueryName(query: QueryInfo) {
+  // Queries run through quick evaluation are not usually the entire query file.
+  // Label them differently and include the line numbers.
+  if (query.quickEvalPosition !== undefined) {
+    const { line, endLine, fileName } = query.quickEvalPosition;
+    const lineInfo = line === endLine ? `${line}` : `${line}-${endLine}`;
+    return `Quick evaluation of ${path.basename(fileName)}:${lineInfo}`;
+  } else if (query.metadata?.name) {
+    return query.metadata.name;
+  } else {
+    return path.basename(query.program.queryPath);
+  }
+}
+
+
 /**
  * Call cli command to interpret results.
  */
-export async function interpretResults(server: cli.CodeQLCliServer, metadata: QueryMetadata | undefined, resultsPaths: ResultsPaths, sourceInfo?: cli.SourceInfo): Promise<sarif.Log> {
+export async function interpretResults(
+  server: cli.CodeQLCliServer,
+  metadata: QueryMetadata | undefined,
+  resultsPaths: ResultsPaths,
+  sourceInfo?: cli.SourceInfo
+): Promise<sarif.Log> {
   const { resultsPath, interpretedResultsPath } = resultsPaths;
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
@@ -138,7 +180,7 @@ export async function interpretResults(server: cli.CodeQLCliServer, metadata: Qu
   if (id === undefined) {
     // Interpretation per se doesn't really require an id, but the
     // SARIF format does, so in the absence of one, we use a dummy id.
-    id = "dummy-id";
+    id = 'dummy-id';
   }
   return await server.interpretBqrs({ kind, id }, resultsPath, interpretedResultsPath, sourceInfo);
 }

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,15 +1,14 @@
 import * as cp from 'child_process';
 import * as path from 'path';
-// Import from the specific module within `semmle-vscode-utils`, rather than via `index.ts`, because
-// we avoid taking an accidental runtime dependency on `vscode` this way.
-import { DisposableObject } from 'semmle-vscode-utils/out/disposable-object';
+import { DisposableObject } from './vscode-utils/disposable-object';
 import { Disposable } from 'vscode';
 import { CancellationToken, createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
 import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
-import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './messages';
-import * as messages from './messages';
+import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
+import * as messages from './pure/messages';
+import { SemVer } from 'semver';
 
 type ServerOpts = {
   logger: Logger;
@@ -49,6 +48,12 @@ type WithProgressReporting = (task: (progress: ProgressReporter, token: Cancella
  * to restart it (which disposes the existing process and starts a new one).
  */
 export class QueryServerClient extends DisposableObject {
+
+  /**
+   * Query Server version where database registration was introduced
+   */
+  private static VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
+
   serverProcess?: ServerProcess;
   evaluationResultCallbacks: { [key: number]: (res: EvaluationResult) => void };
   progressCallbacks: { [key: number]: ((res: ProgressMessage) => void) | undefined };
@@ -57,11 +62,16 @@ export class QueryServerClient extends DisposableObject {
   withProgressReporting: WithProgressReporting;
   public activeQueryName: string | undefined;
 
-  constructor(readonly config: QueryServerConfig, readonly cliServer: cli.CodeQLCliServer, readonly opts: ServerOpts, withProgressReporting: WithProgressReporting) {
+  constructor(
+    readonly config: QueryServerConfig,
+    readonly cliServer: cli.CodeQLCliServer,
+    readonly opts: ServerOpts,
+    withProgressReporting: WithProgressReporting
+  ) {
     super();
     // When the query server configuration changes, restart the query server.
-    if (config.onDidChangeQueryServerConfiguration !== undefined) {
-      this.push(config.onDidChangeQueryServerConfiguration(async () => {
+    if (config.onDidChangeConfiguration !== undefined) {
+      this.push(config.onDidChangeConfiguration(async () => {
         this.logger.log('Restarting query server due to configuration changes...');
         await this.restartQueryServer();
       }, this));
@@ -106,9 +116,19 @@ export class QueryServerClient extends DisposableObject {
   private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
+
+    if (await this.supportsDatabaseRegistration()) {
+      args.push('--require-db-registration');
+    }
+
     if (this.config.debug) {
       args.push('--debug', '--tuple-counting');
     }
+
+    if (cli.shouldDebugQueryServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9010,server=y,suspend=n,quiet=y');
+    }
+
     const child = cli.spawnServer(
       this.config.codeQlPath,
       'CodeQL query server',
@@ -154,6 +174,10 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
+  async supportsDatabaseRegistration() {
+    return (await this.cliServer.getVersion()).compare(QueryServerClient.VERSION_WITH_DB_REGISTRATION) >= 0;
+  }
+
   registerCallback(callback: (res: EvaluationResult) => void): number {
     const id = this.nextCallback++;
     this.evaluationResultCallbacks[id] = callback;