Fix CSS file paths

* Move/rename query metadata function

* Add test for `tryGetQueryMetadata`

* Split into two tests

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,7 +10,12 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
-**To Reproduce**
+**Version**
+The CodeQL and VS Code version in which the bug occurs.
+<!-- To copy version information for the CodeQL extension, click "CodeQL CLI vX.X.X" in the status bar at the bottom of the screen. 
+To copy detailed version information for VS Code itself, see https://code.visualstudio.com/docs/supporting/FAQ#_how-do-i-find-the-version. -->
+
+**To reproduce**
 Steps to reproduce the behavior.
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/new-extension-release.md

@@ -1,18 +0,0 @@
----
-name: New extension release
-about: Create an issue with a checklist for the release steps (write access required
-  for the steps)
-title: Release Checklist for version xx.xx.xx
-labels: ''
-assignees: ''
-
----
-
-- [ ] Update this issue title to refer to the version of the release
-- [ ] Trigger a release build on Actions by adding a new tag on branch `main` of the format `vxx.xx.xx`
-- [ ] Monitor the status of the release build in the `Release` workflow in the Actions tab.
-- [ ] Download the VSIX from the draft GitHub release that is created when the release build finishes.
-- [ ] Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
-- [ ] Click the `...` menu in the CodeQL row and click **Update**.
-- [ ] Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-- [ ] Publish the draft GitHub release and confirm the new release is marked as the latest release at https://github.com/github/vscode-codeql/releases.

.github/codeql/codeql-config.yml

@@ -0,0 +1,12 @@
+name: "CodeQL config"
+queries:
+  - name: Run standard queries
+    uses: security-and-quality
+  - name: Run custom javascript queries
+    uses: ./.github/codeql/queries
+paths:
+  - ./extensions/ql-vscode
+paths-ignore:
+  - '**/node_modules'
+  - '**/build'
+  - '**/out'

.github/codeql/queries/assert-pure.ql

@@ -0,0 +1,21 @@
+/**
+ * @name Unwanted dependency on vscode API
+ * @kind problem
+ * @problem.severity error
+ * @id vscode-codeql/assert-pure
+ * @description The modules stored under `pure` and tested in the `pure-tests`
+ * are intended to be "pure".
+ */
+import javascript
+
+class VSCodeImport extends ASTNode {
+  VSCodeImport() {
+    this.(Import).getImportedPath().getValue() = "vscode"
+  }
+}
+
+from Module m, VSCodeImport v
+where
+  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
+  m.getAnImportedModule*().getAnImport() = v
+select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/qlpack.yml

@@ -0,0 +1,3 @@
+name: vscode-codeql-custom-queries-javascript
+version: 0.0.0
+libraryPathDependencies: codeql-javascript

.github/pull_request_template.md

@@ -9,4 +9,4 @@ Replace this with a description of the changes your pull request makes.
 
 - [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
 - [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] `@github/docs-content-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.
+- [ ] `@github/docs-content-codeql` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -2,21 +2,30 @@ name: "Code Scanning - CodeQL"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
-    - cron: '0 0 * * 0'
+    - cron: '21 17 * * 0'
 
 jobs:
   codeql:
-    strategy:
-      fail-fast: false
     runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      security-events: write
+      pull-requests: read
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@main
+        with:
+            languages: javascript
+            config-file: ./.github/codeql/codeql-config.yml
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@main

.github/workflows/main.yml

@@ -1,6 +1,8 @@
 name: Build Extension
 on:
+  workflow_dispatch:
   pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
   push:
     branches:
       - main
@@ -20,17 +22,19 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
       - name: Install dependencies
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm install
         shell: bash
 
       - name: Build
+        working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
-          cd extensions/ql-vscode
           npm run build
         shell: bash
 
@@ -47,9 +51,26 @@ jobs:
           name: vscode-codeql-extension
           path: artifacts
 
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST"
+
   test:
     name: Test
     runs-on: ${{ matrix.os }}
+    needs: [find-nightly]
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
@@ -61,57 +82,115 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
-      # We have to build the dependencies in `lib` before running any tests.
       - name: Install dependencies
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm install
         shell: bash
 
       - name: Build
+        working-directory: extensions/ql-vscode
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
-          cd extensions/ql-vscode
           npm run build
         shell: bash
 
       - name: Lint
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run lint
 
-      - name: Install CodeQL
-        run: |
-          mkdir codeql-home
-          curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
-          unzip -q -o codeql-home/codeql.zip -d codeql-home
-          unzip -q -o codeql-home/codeql.zip codeql/codeql.exe -d codeql-home
-          rm codeql-home/codeql.zip
-        shell: bash
-
       - name: Run unit tests (Linux)
+        working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
-          cd extensions/ql-vscode
-          CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
+          npm run test
 
       - name: Run unit tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
-          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
         if: matrix.os == 'ubuntu-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           sudo apt-get install xvfb
           /usr/bin/xvfb-run npm run integration
 
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run integration
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    needs: [find-nightly]
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.3', 'nightly']
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.14.0'
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Decide on ref of CodeQL repo
+        id: choose-ref
+        shell: bash
+        run: |
+          if [[ "${{ matrix.version }}" == "nightly" ]]
+          then
+            REF="codeql-cli/latest"
+          elif [[ "${{ matrix.version }}" == "v2.2.6" || "${{ matrix.version }}" == "v2.3.3" ]]
+          then
+            REF="codeql-cli/v2.4.5"
+          else
+            REF="codeql-cli/${{ matrix.version }}"
+          fi
+          echo "::set-output name=ref::$REF"
+
+      - name: Checkout QL
+        uses: actions/checkout@v2
+        with:
+          repository: github/codeql
+          ref: ${{ steps.choose-ref.outputs.ref }}
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          /usr/bin/xvfb-run npm run cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run cli-integration

.github/workflows/release.yml

@@ -6,25 +6,16 @@
 
 name: Release
 on:
+  workflow_dispatch:
+
   push:
-    # Path filters are not evaluated for pushes to tags.
-    # (source: https://help.github.com/en/github/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions#onpushpull_requestpaths)
-    # So this workflow is triggered in the following events:
-    # - Release event: a SemVer tag, e.g. v1.0.0 or v1.0.0-alpha, is pushed
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
-    # OR
-    # - Test event: this file is modified on a branch in the main repo containing `/actions/` in the name.
-    branches:
-      - '**/actions/**'
-    paths:
-      - '**/workflows/release.yml'
 
 jobs:
   build:
     name: Release
     runs-on: ubuntu-latest
-    # TODO Share steps with the main workflow.
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -36,11 +27,14 @@ jobs:
       - name: Install dependencies
         run: |
           cd extensions/ql-vscode
-          npm install
+          npm ci
         shell: bash
 
       - name: Build
+        env:
+          APP_INSIGHTS_KEY: '${{ secrets.APP_INSIGHTS_KEY }}'
         run: |
+          echo "APP INSIGHTS KEY LENGTH: ${#APP_INSIGHTS_KEY}"
           cd extensions/ql-vscode
           npm run build -- --release
         shell: bash
@@ -59,9 +53,6 @@ jobs:
           REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
-      # Uploading artifacts is not necessary to create a release.
-      # This is just in case the release itself fails and we want to access the built artifacts from Actions.
-      # TODO Remove if not useful.
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -97,6 +88,10 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
+      ###
+      # Do Post release work: version bump and changelog PR
+      # Only do this if we are running from a PR (ie- this is part of the release process)
+
       # The checkout action does not fetch the main branch.
       # Fetch the main branch so that we can base the version bump PR against main.
       - name: Fetch main branch
@@ -121,7 +116,7 @@ jobs:
           perl -i -pe 's/^/## \[UNRELEASED\]\n\n/ if($.==3)' CHANGELOG.md
 
       - name: Create version bump PR
-        uses: peter-evans/create-pull-request@c7b64af0a489eae91f7890f2c1b63d13cc2d8ab7 # v2.4.2
+        uses: peter-evans/create-pull-request@c7f493a8000b8aeb17a1332e326ba76b57cb83eb # v3.4.1
         if: success()
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -130,3 +125,41 @@ jobs:
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
           base: main
+          draft: true
+
+  vscode-publish:
+    name: Publish to VS Code Marketplace
+    needs: build
+    environment: publish-vscode-marketplace
+    runs-on: ubuntu-latest
+    env:
+      VSCE_TOKEN: ${{ secrets.VSCE_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix || \
+            echo "Failed to publish to VS Code Marketplace. \
+                  If this was an authentication problem, please make sure the \
+                  auth token hasn't expired."
+
+  open-vsx-publish:
+    name: Publish to Open VSX Registry
+    needs: build
+    environment: publish-open-vsx
+    runs-on: ubuntu-latest
+    env:
+      OPEN_VSX_TOKEN: ${{ secrets.OPEN_VSX_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx ovsx publish -p $OPEN_VSX_TOKEN *.vsix

.gitignore

@@ -4,6 +4,7 @@
 # Generated files
 /dist/
 out/
+build/
 server/
 node_modules/
 gen/

.vscode/extensions.json

@@ -1,9 +1,9 @@
 {
-  // See http://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
+  // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
-    "eamodio.tsl-problem-matcher",
+    "amodio.tsl-problem-matcher",
     "dbaeumer.vscode-eslint",
     "eternalphane.tsfmt-vscode"
   ],

.vscode/launch.json

@@ -8,7 +8,9 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        // Add a reference to a workspace to open. Eg-
+        // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -16,8 +18,12 @@
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
       "env": {
-        // uncomment to allow debugging the language server Java process from a remote java debugger
-        // "DEBUG_LANGUAGE_SERVER": "true"
+        // change to 'true' debug the IDE or Query servers
+        "IDE_SERVER_JAVA_DEBUG": "false",
+        "QUERY_SERVER_JAVA_DEBUG": "false",
+        "CLI_SERVER_JAVA_DEBUG": "false",
+        // Uncomment to set the JAVA_HOME for the codeql instance to use
+        // "CODEQL_JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk-12.0.1.jdk/Contents/Home"
       }
     },
     {
@@ -76,6 +82,38 @@
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
+    },
+    {
+      "name": "Launch Integration Tests - With CLI",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
+        // Add a path to a checked out instance of the codeql repository so the libraries are
+        // available in the workspace for the tests.
+        // "${workspaceRoot}/../codeql"
+      ],
+      "env": {
+        // Optionally, set the version to use for the integration tests.
+        // Use "nightly" to use the latest nightly build.
+        // "CLI_VERSION": "2.7.0",
+
+        // If CLI_VERSION is set to nightly, set this to the url of the nightly build.
+        // "NIGHTLY_URL": "some url to grab the nightly build",
+
+        // Optionally, add a path to the codeql executable to be used during these tests.
+        // If not specified, one will be downloaded automatically.
+        // This option overrides the CLI_VERSION option.
+        // "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
+      },
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "outFiles": [
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+      ],
     }
   ]
 }

.vscode/settings.json

@@ -22,7 +22,8 @@
     "common/temp": true,
     "**/.vscode-test": true
   },
-  "typescript.tsdk": "./common/temp/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
+  "typescript.tsdk": "./extensions/ql-vscode/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
+  "typescript.enablePromptUseWorkspaceTsdk": true,
   "eslint.validate": [
     "javascript",
     "javascriptreact",

CODEOWNERS

@@ -0,0 +1 @@
+**/* @github/codeql-vscode-reviewers

CONTRIBUTING.md

@@ -25,12 +25,11 @@ Here are a few things you can do that will increase the likelihood of your pull
 * Follow the [style guide][style].
 * Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
 * Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
-Make sure you have a fairly recent version of vscode (>1.32) and are using nodejs
-version >=v10.13.0. (Tested on v10.15.1 and v10.16.0).
+Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
 
 ### Installing all packages
 
@@ -57,7 +56,8 @@ We recommend that you keep `npm run watch` running in the backgound and you only
 
 1. on first checkout
 2. whenever any of the non-TypeScript resources have changed
-3. on any change to files included in the webview
+3. on any change to files included in one of the webviews
+   - **Important**: This is easy to forget. You must explicitly run `npm run build` whenever one of the files in the webview is changed. These are the files in the `src/view` and `src/compare/view` folders.
 
 ### Installing the extension
 
@@ -77,11 +77,11 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
-### Running the unit/integration tests
+### Running the unit tests and integration tests that do not require a CLI instance
 
-Ensure the `CODEQL_PATH` environment variable is set to point to the `codeql` cli executable.
+Unit tests and many integration tests do not require a copy of the CodeQL CLI.
 
-Outside of vscode, run:
+Outside of vscode, in the `extensions/ql-vscode` directory, run:
 
 ```shell
 npm run test && npm run integration
@@ -89,26 +89,67 @@ npm run test && npm run integration
 
 Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
 
+Only the _With CLI_ tests require a CLI instance to run. See below on how to do that.
+
+Running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+### Running the integration tests
+
+The _Launch Integration Tests - With CLI_ tests require a CLI instance in order to run. There are several environment variables you can use to configure this.
+
+From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ uncomment and change the environment variables appropriate for your purpose.
+
 ## Releasing (write access required)
 
-1. Double-check the `CHANGELOG.md` contains all desired change comments
-   and has the version to be released with date at the top.
-1. Double-check that the extension `package.json` has the version you intend to release.
-   If you are doing a patch release (as opposed to minor or major version) this should already
-   be correct.
-1. Trigger a release build on Actions by adding a new tag on branch `main` of the format `vxx.xx.xx`
+1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
+    * Go through all recent PRs and make sure they are properly accounted for.
+    * Make sure all changelog entries have links back to their PR(s) if appropriate.
+1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
+1. Create a PR for this release:
+    * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
+    * Create a new branch for the release named after the new version. For example: `v1.3.6`
+    * Create a new commit with a message the same as the branch name.
+    * Create a PR for this branch.
+    * Wait for the PR to be merged into `main`
+1. Trigger a release build on Actions by adding a new tag on branch `main` named after the release, as above. Note that when you push to upstream, you will need to fully qualify the ref. A command like this will work:
+
+    ```bash
+    git push upstream refs/tags/v1.3.6
+    ```
+
+    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
+    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
+
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
-1. Optionally unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
+1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
-1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
-1. Click the `...` menu in the CodeQL row and click **Update**.
-1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+    - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
+1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
+1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
 1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
 1. If documentation changes need to be published, notify documentation team that release has been made.
 1. Review and merge the version bump PR that is automatically created by Actions.
 
+## Secrets and authentication for publishing
+
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+
+To regenerate the Open VSX token:
+
+1. Log in to the [user settings page on Open VSX](https://open-vsx.org/user-settings/namespaces).
+1. Make sure you are a member of the GitHub namespace.
+1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
+1. Update the secret in the `publish-open-vsx` environment in the project settings.
+
+To regenerate the VSCode Marketplace token:
+
+1. Follow the instructions on [getting a PAT for Azure DevOps](https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token).
+1. Update the secret in the `publish-vscode-marketplace` environment in the project settings.
+
+Not that Azure DevOps PATs expire yearly and must be regenerated.
+
 ## Resources
 
 * [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)

README.md

@@ -16,7 +16,6 @@ To see what has changed in the last few versions of the extension, see the [Chan
 * Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
-
 ## Project goals and scope
 
 This project will track new feature development in CodeQL and, whenever appropriate, bring that functionality to the Visual Studio Code experience.

extensions/ql-vscode/.eslintrc.js

@@ -22,8 +22,10 @@ module.exports = {
       },
     ],
     "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/explicit-module-boundary-types": "off",
     "@typescript-eslint/no-non-null-assertion": "off",
     "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-floating-promises": [ "error", { ignoreVoid: true } ],
     "prefer-const": ["warn", { destructuring: "all" }],
     indent: "off",
     "@typescript-eslint/indent": "off",

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,179 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.5.9 - 17 December 2021
+
+- Avoid creating a third column when opening the results view. The results view will always open to the right of the active editor, unless the active editor is in the rightmost editor column. In that case open in the leftmost column. [#1037](https://github.com/github/vscode-codeql/pull/1037)
+- Add a CodeLens to make the Quick Evaluation command more accessible. Click the `Quick Evaluation` prompt above a predicate definition in the editor to evaluate that predicate on its own. You can enable/disable this feature in the `codeQL.runningQueries.quickEvalCodelens` setting. [#1035](https://github.com/github/vscode-codeql/pull/1035) & [#1052](https://github.com/github/vscode-codeql/pull/1052)
+- Fix a bug where the _Alerts_ option would show in the results view even if there is no alerts table available. [#1038](https://github.com/github/vscode-codeql/pull/1038)
+
+## 1.5.8 - 2 December 2021
+
+- Emit a more explicit error message when a user tries to add a database with an unzipped source folder to the workspace. [#1021](https://github.com/github/vscode-codeql/pull/1021)
+- Ensure `src.zip` archives are used as the canonical source instead of `src` folders when importing databases. [#1025](https://github.com/github/vscode-codeql/pull/1025)
+
+## 1.5.7 - 23 November 2021
+
+- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979)
+- Support large SARIF results files (>4GB) without crashing VS Code. [#1004](https://github.com/github/vscode-codeql/pull/1004)
+- Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
+- Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
+- Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)
+- Allow case-insensitive project slugs for GitHub repositories when adding a CodeQL database from LGTM. [#978](https://github.com/github/vscode-codeql/pull/961)
+- Add a _CodeQL: Preview Query Help_ command to generate Markdown previews of `.qhelp` query help files. This command should only be run in trusted workspaces. See https://codeql.github.com/docs/codeql-cli/testing-query-help-files for more information about query help. [#988](https://github.com/github/vscode-codeql/pull/988)
+- Make "Open Referenced File" command accessible from the active editor menu. [#989](https://github.com/github/vscode-codeql/pull/989)
+- Fix a bug where result set names in the result set drop-down were disappearing when viewing a sorted table. [#1007](https://github.com/github/vscode-codeql/pull/1007)
+- Allow query result locations with 0 as the end column value. These are treated as the first column in the line. [#1002](https://github.com/github/vscode-codeql/pull/1002)
+
+## 1.5.6 - 07 October 2021
+
+- Add progress messages to LGTM download option. This makes the two-step process (selecting a project, then selecting a language) more clear. [#960](https://github.com/github/vscode-codeql/pull/960)
+- Remove line about selecting a language from the dropdown when downloading database from LGTM. This makes the download progress visible when the popup is not expanded. [#957](https://github.com/github/vscode-codeql/pull/957)
+- Fix a bug where copying the version information fails when a CodeQL CLI cannot be found. [#958](https://github.com/github/vscode-codeql/pull/958)
+- Avoid a race condition when deleting databases that can cause occasional errors. [#959](https://github.com/github/vscode-codeql/pull/959)
+- Update CodeQL logos. [#965](https://github.com/github/vscode-codeql/pull/965)
+
+## 1.5.5 - 08 September 2021
+
+- Fix bug where a query is sometimes run before the file is saved. [#947](https://github.com/github/vscode-codeql/pull/947)
+- Fix broken contextual queries, including _View AST_. [#949](https://github.com/github/vscode-codeql/pull/949)
+
+## 1.5.4 - 02 September 2021
+
+- Add support for filename pattern in history view. [#930](https://github.com/github/vscode-codeql/pull/930)
+- Add an option _View Results (CSV)_ to view the results of a non-alert query. The existing options for alert queries have been renamed to _View Alerts_ to avoid confusion. [#929](https://github.com/github/vscode-codeql/pull/929)
+- Allow users to specify the number of paths to display for each alert. [#931](https://github.com/github/vscode-codeql/pull/931)
+- Adjust pagination controls in _CodeQL Query Results_ to always be visible [#936](https://github.com/github/vscode-codeql/pull/936)
+- Fix bug where _View AST_ fails due to recent refactoring in the standard library and query packs. [#939](https://github.com/github/vscode-codeql/pull/939)
+
+## 1.5.3 - 18 August 2021
+
+- Add a command _CodeQL: Run Query on Multiple Databases_, which lets users select multiple databases to run a query on. [#898](https://github.com/github/vscode-codeql/pull/898)
+- Autodetect what language a query targets. This refines the _CodeQL: Run Query on Multiple Databases_ command to only show relevant databases. [#915](https://github.com/github/vscode-codeql/pull/915)
+- Adjust test log output to display diffs only when comparing failed test results with expected test results. [#920](https://github.com/github/vscode-codeql/pull/920)
+
+## 1.5.2 - 13 July 2021
+
+- Add the _Add Database Source to Workspace_ command to the right-click context menu in the databases view. This lets users re-add a database's source folder to the workspace and browse the source code. [#891](https://github.com/github/vscode-codeql/pull/891)
+- Fix markdown rendering in the description of the `codeQL.cli.executablePath` setting. [#908](https://github.com/github/vscode-codeql/pull/908)
+- Fix the _Open Query Results_ command in the query history view. [#909](https://github.com/github/vscode-codeql/pull/909)
+
+## 1.5.1 - 23 June 2021
+
+No user facing changes.
+
+## 1.5.0 - 14 June 2021
+
+- Display CodeQL CLI version being downloaded during an upgrade. [#862](https://github.com/github/vscode-codeql/pull/862)
+- Display a helpful message and link to documentation when a query produces no results. [#866](https://github.com/github/vscode-codeql/pull/866)
+- Refresh test databases automatically after a test run. [#868](https://github.com/github/vscode-codeql/pull/868)
+- Allow users to specify a custom directory for storing query server logs (`codeQL.runningQueries.customLogDirectory`). The extension will not delete these logs automatically. [#863](https://github.com/github/vscode-codeql/pull/863)
+- Support the VS Code [Workspace Trust feature](https://code.visualstudio.com/docs/editor/workspace-trust). This extension is now enabled in untrusted workspaces, but it restricts commands that contain arbitrary paths. [#861](https://github.com/github/vscode-codeql/pull/861)
+- Allow the `codeQL.cli.executablePath` configuration setting to be set in workspace-scoped configuration files. This means that each workspace can now specify its own CodeQL CLI compiler, a feature that is unblocked due to implementing Workspace Trust. [#861](https://github.com/github/vscode-codeql/pull/861)
+
+## 1.4.8 - 05 May 2021
+
+- Copy version information to the clipboard when a user clicks the CodeQL section of the status bar. [#845](https://github.com/github/vscode-codeql/pull/845)
+- Ensure changes in directories that contain tests will be properly updated in the test explorer. [#846](https://github.com/github/vscode-codeql/pull/846)
+- Remind users to choose a language when downloading a database from LGTM. [#852](https://github.com/github/vscode-codeql/pull/852)
+
+## 1.4.7 - 23 April 2021
+
+- Fix a bug that prevented the results view from being loaded. [#842](https://github.com/github/vscode-codeql/pull/842)
+
+## 1.4.6 - 21 April 2021
+
+- Avoid showing an error popup when running a query with `@kind table` metadata. [#814](https://github.com/github/vscode-codeql/pull/814)
+- Add an option to jump from a .qlref file to the .ql file it references. [#815](https://github.com/github/vscode-codeql/pull/815)
+- Avoid opening the results panel when a database is deleted. [#831](https://github.com/github/vscode-codeql/pull/831)
+- Forward all query metadata to the CLI when interpreting results. [#838](https://github.com/github/vscode-codeql/pull/838)
+
+## 1.4.5 - 22 March 2021
+
+- Avoid showing an error popup when user runs a query without `@kind` metadata. [#801](https://github.com/github/vscode-codeql/pull/801)
+- Fix running of tests when the `ms-python` extension is installed. [#803](https://github.com/github/vscode-codeql/pull/803)
+
+## 1.4.4 - 19 March 2021
+
+- Introduce evaluator options for saving intermediate results to the disk cache (`codeQL.runningQueries.saveCache`) and for limiting the size of this cache (`codeQL.runningQueries.cacheSize`). [#778](https://github.com/github/vscode-codeql/pull/778)
+- Respect the `codeQL.runningQueries.numberOfThreads` setting when creating SARIF files during result interpretation. [#771](https://github.com/github/vscode-codeql/pull/771)
+- Allow using raw LGTM project slugs for fetching LGTM databases. [#769](https://github.com/github/vscode-codeql/pull/769)
+- Better error messages when BQRS interpretation fails to produce SARIF. [#770](https://github.com/github/vscode-codeql/pull/770)
+- Implement sorting of the query history view by name, date, and results count. [#777](https://github.com/github/vscode-codeql/pull/777)
+- Add a configuration option to pass additional arguments to the CLI when running tests. [#785](https://github.com/github/vscode-codeql/pull/785)
+- Introduce option to view query results as CSV. [#784](https://github.com/github/vscode-codeql/pull/784)
+- Add some snippets for commonly used QL statements. [#782](https://github.com/github/vscode-codeql/pull/782)
+- More descriptive error messages on QL test failures. [#788](https://github.com/github/vscode-codeql/pull/788)
+
+## 1.4.3 - 22 February 2021
+
+- Avoid displaying an error when removing orphaned databases and the storage folder does not exist. [#748](https://github.com/github/vscode-codeql/pull/748)
+- Add better error messages when AST Viewer is unable to create an AST. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Cache AST viewing operations so that subsequent calls to view the AST of a single file will be extremely fast. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Ensure CodeQL version in status bar updates correctly when version changes. [#754](https://github.com/github/vscode-codeql/pull/754)
+- Avoid deleting the quick query file when it is re-opened. [#747](https://github.com/github/vscode-codeql/pull/747)
+
+## 1.4.2 - 2 February 2021
+
+- Add a status bar item for the CodeQL CLI to show the current version. [#741](https://github.com/github/vscode-codeql/pull/741)
+- Fix version constraint for flagging CLI support of non-destructive updates. [#744](https://github.com/github/vscode-codeql/pull/744)
+- Add a _More Information_ button in the telemetry popup that opens the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) in a browser tab. [#742](https://github.com/github/vscode-codeql/pull/742)
+
+## 1.4.1 - 29 January 2021
+
+- Reword the telemetry modal dialog box. [#738](https://github.com/github/vscode-codeql/pull/738)
+
+## 1.4.0 - 29 January 2021
+
+- Fix bug where databases are not reregistered when the query server restarts. [#734](https://github.com/github/vscode-codeql/pull/734)
+- Fix bug where upgrade requests were erroneously being marked as failed. [#734](https://github.com/github/vscode-codeql/pull/734)
+- On a strictly opt-in basis, collect anonymized usage data from the VS Code extension, helping improve CodeQL's usability and performance. See the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) for more information on exactly what data is collected and what it is used for. [#611](https://github.com/github/vscode-codeql/pull/611)
+
+## 1.3.10 - 20 January 2021
+
+- Include the full stack in error log messages to help with debugging. [#726](https://github.com/github/vscode-codeql/pull/726)
+
+## 1.3.9 - 12 January 2021
+
+- No changes visible to end users.
+
+## 1.3.8 - 17 December 2020
+
+- Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting the IDE. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)
+- Fix bug when removing databases where sometimes the source folder would not also be removed from the workspace or the database files would not be deleted from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692)
+- Query results with no string representation will now be displayed with placeholder text in query results. Previously, they were omitted. [#694](https://github.com/github/vscode-codeql/pull/694)
+- Add a label for the language of a database in the databases view. This will only take effect for new databases created with the CodeQL CLI v2.4.1 or later. [#697](https://github.com/github/vscode-codeql/pull/697)
+- Add clearer error message when running a query using a missing or invalid qlpack. [#702](https://github.com/github/vscode-codeql/pull/702)
+- Add clearer error message when trying to run a command from the query history view if no item in the history is selected. [#702](https://github.com/github/vscode-codeql/pull/702)
+- Fix a bug where it is not possible to download some database archives. This fix specifically addresses large archives and archives whose central directories do not align with file headers. [#700](https://github.com/github/vscode-codeql/pull/700)
+- Avoid error dialogs when QL test discovery or database cleanup encounters a missing directory. [#706](https://github.com/github/vscode-codeql/pull/706)
+- Add descriptive text and a link in the results view. [#711](https://github.com/github/vscode-codeql/pull/711)
+- Fix the _Set Label_ command in the query history view. [#710](https://github.com/github/vscode-codeql/pull/710)
+- Add the _CodeQL: View AST_ command to the right-click context menu when a source file in a database source archive is open in the editor. [#712](https://github.com/github/vscode-codeql/pull/712)
+
+## 1.3.7 - 24 November 2020
+
+- Editors opened by navigating from the results view are no longer opened in _preview mode_. Now they are opened as a persistent editor. [#630](https://github.com/github/vscode-codeql/pull/630)
+- When comparing the results of a failed QL test run and the `.expected` file does not exist, an empty `.expected` file is created and compared against the `.actual` file. [#669](https://github.com/github/vscode-codeql/pull/669)
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
+- Add more structured output for tests. [#626](https://github.com/github/vscode-codeql/pull/626)
+- Whenever the extension restarts, orphaned databases will be cleaned up. These are databases whose files are located inside of the extension's storage area, but are not imported into the workspace.
+- After renaming a database, the database list is re-sorted. [#685](https://github.com/github/vscode-codeql/pull/685)
+- Add a `codeQl.resultsDisplay.pageSize` setting to configure the number of results displayed in a single results view page. Increase the default page size from 100 to 200. [#686](https://github.com/github/vscode-codeql/pull/686)
+- Update the AST Viewer to include edge labels (if available) in addition to the target node labels. So far, only C/C++ databases take advantage of this change. [#688](https://github.com/github/vscode-codeql/pull/688)
+
+## 1.3.6 - 4 November 2020
+
+- Fix URI encoding for databases that were created with special characters in their paths. [#648](https://github.com/github/vscode-codeql/pull/648)
+- Disable CodeQL Test commands from the command palette [#667](https://github.com/github/vscode-codeql/pull/667)
+- Fix display of booleans in results view. [#657](https://github.com/github/vscode-codeql/pull/657)
+- Avoid recursive selection changes in AST Viewer. [#668](https://github.com/github/vscode-codeql/pull/668)
+
+## 1.3.5 - 27 October 2020
+
+- Fix a bug where archived source folders for databases were not showing any contents.
+- Fix URI encoding for databases that were created with special characters in their paths.
+
 ## 1.3.4 - 22 October 2020
 
 - Add friendly welcome message when the databases view is empty.
@@ -15,6 +189,8 @@
 - Add a `View DIL` command on query history items. This opens a text editor containing the Datalog Intermediary Language representation of the compiled query.
 - Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/exploring-the-structure-of-your-source-code.html).
 - The `codeQL.runningTests.numberOfThreads` setting is now used correctly when running tests.
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the qlpacks.
+- Ensure output of CodeQL test runs includes compilation error messages and test failure messages.
 
 ## 1.3.3 - 16 September 2020
 

extensions/ql-vscode/README.md

@@ -1,6 +1,6 @@
 # CodeQL extension for Visual Studio Code
 
-This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
+This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://codeql.github.com/docs/) and allows you to easily find problems in codebases. In particular, the extension:
 
 - Enables you to use CodeQL to query databases generated from source code.
 - Shows the flow of data through the results of path queries, which is essential for triaging security results.
@@ -12,7 +12,7 @@ To see what has changed in the last few versions of the extension, see the [Chan
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
-For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
+For information about other configurations, see the separate [CodeQL help](https://codeql.github.com/docs/codeql-for-visual-studio-code/).
 
 ### Quick start: Installing and configuring the extension
 
@@ -40,9 +40,9 @@ The CodeQL extension requires a minimum of Visual Studio Code 1.39. Older versio
 
 ### Checking access to the CodeQL CLI
 
-The extension uses the [CodeQL CLI](https://help.semmle.com/codeql/codeql-cli.html) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
+The extension uses the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
 
-If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#configuring-access-to-the-codeql-cli).
+If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#configuring-access-to-the-codeql-cli).
 
 If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Log** in the **Output** view for any error messages.
 
@@ -52,7 +52,7 @@ When you're working with CodeQL, you need access to the standard CodeQL librarie
 Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
-For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
+For information about configuring an existing workspace for CodeQL, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#updating-an-existing-workspace-for-codeql).
 
 ## Upgrading CodeQL standard libraries
 
@@ -75,7 +75,7 @@ You can find all the commands contributed by the extension in the Command Palett
 
 ### Importing a database from LGTM
 
-While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
+While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from LGTM.com.
 
 1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
 1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
@@ -100,13 +100,17 @@ If there are any problems running a query, a notification is displayed in the bo
 
 ## What next?
 
-For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
+For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:
 
-- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Create a database for a different codebase](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/).
 - [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Learn more about CodeQL](https://codeql.github.com/docs/).
 - [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
 
 The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the MIT License. The version of CodeQL used by the CodeQL extension is subject to the [GitHub CodeQL Terms & Conditions](https://securitylab.github.com/tools/codeql/license).
+
+## Data and Telemetry
+
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information about telemetry, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code).

extensions/ql-vscode/gulpfile.ts/appInsights.ts

@@ -0,0 +1,16 @@
+import * as gulp from 'gulp';
+import * as replace from 'gulp-replace';
+
+/** Inject the application insights key into the telemetry file */
+export function injectAppInsightsKey() {
+  if (!process.env.APP_INSIGHTS_KEY) {
+    // noop
+    console.log('APP_INSIGHTS_KEY environment variable is not set. So, cannot inject it into the application.');
+    return Promise.resolve();
+  }
+
+  // replace the key
+  return gulp.src(['out/telemetry.js'])
+    .pipe(replace(/REPLACE-APP-INSIGHTS-KEY/, process.env.APP_INSIGHTS_KEY))
+    .pipe(gulp.dest('out/'));
+}

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -13,6 +13,7 @@ const packageFiles = [
   'CHANGELOG.md',
   'README.md',
   'language-configuration.json',
+  'snippets.json',
   'media',
   'node_modules',
   'out'

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -4,7 +4,12 @@ import { compileTextMateGrammar } from './textmate';
 import { copyTestData } from './tests';
 import { compileView } from './webpack';
 import { packageExtension } from './package';
+import { injectAppInsightsKey } from './appInsights';
 
-export const buildWithoutPackage = gulp.parallel(compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss);
-export { compileTextMateGrammar, watchTypeScript, compileTypeScript };
-exports.default = gulp.series(exports.buildWithoutPackage, packageExtension);
+export const buildWithoutPackage =
+  gulp.parallel(
+    compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
+  );
+
+export { compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData, injectAppInsightsKey };
+export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

extensions/ql-vscode/gulpfile.ts/tests.ts

@@ -1,6 +1,17 @@
 import * as gulp from 'gulp';
 
 export function copyTestData() {
+  copyNoWorkspaceData();
+  copyCliIntegrationData();
+  return Promise.resolve();
+}
+
+function copyNoWorkspaceData() {
   return gulp.src('src/vscode-tests/no-workspace/data/**/*')
     .pipe(gulp.dest('out/vscode-tests/no-workspace/data'));
 }
+
+function copyCliIntegrationData() {
+  return gulp.src('src/vscode-tests/cli-integration/data/**/*')
+    .pipe(gulp.dest('out/vscode-tests/cli-integration/data'));
+}

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json.schemastore.org/tsconfig",
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
     "strict": true,

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -37,6 +37,6 @@ export function watchTypeScript() {
 
 /** Copy CSS files for the results view into the output directory. */
 export function copyViewCss() {
-  return gulp.src('src/view/*.css')
+  return gulp.src('src/**/view/*.css')
     .pipe(gulp.dest('out'));
 }

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -6,6 +6,7 @@ export const config: webpack.Configuration = {
   entry: {
     resultsView: './src/view/results.tsx',
     compareView: './src/compare/view/Compare.tsx',
+    remoteQueriesView: './src/remote-queries/view/RemoteQueries.tsx',
   },
   output: {
     path: path.resolve(__dirname, '..', 'out'),
@@ -13,7 +14,10 @@ export const config: webpack.Configuration = {
   },
   devtool: 'inline-source-map',
   resolve: {
-    extensions: ['.js', '.ts', '.tsx', '.json']
+    extensions: ['.js', '.ts', '.tsx', '.json'],
+    fallback: {
+      path: require.resolve('path-browserify')
+    }
   },
   module: {
     rules: [

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -6,21 +6,23 @@ export function compileView(cb: (err?: Error) => void) {
     if (error) {
       cb(error);
     }
-    console.log(stats.toString({
-      errorDetails: true,
-      colors: true,
-      assets: false,
-      builtAt: false,
-      version: false,
-      hash: false,
-      entrypoints: false,
-      timings: false,
-      modules: false,
-      errors: true
-    }));
-    if (stats.hasErrors()) {
-      cb(new Error('Compilation errors detected.'));
-      return;
+    if (stats) {
+      console.log(stats.toString({
+        errorDetails: true,
+        colors: true,
+        assets: false,
+        builtAt: false,
+        version: false,
+        hash: false,
+        entrypoints: false,
+        timings: false,
+        modules: false,
+        errors: true
+      }));
+      if (stats.hasErrors()) {
+        cb(new Error('Compilation errors detected.'));
+        return;
+      }
     }
 
     cb();

extensions/ql-vscode/media/dark/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" fill="#C5C5C5" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" fill="#C5C5C5"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/light/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" />
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" />
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" />
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/logo.svg

@@ -1,14 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="27px" height="16px" viewBox="0 0 27 16" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 59 (86127) - https://sketch.com -->
-    <title>Slice</title>
-    <desc>Created with Sketch.</desc>
-    <g id="light" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="QL" transform="translate(1.000000, 1.000000)">
-            <rect id="Rectangle-41" stroke="#2088FF" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" x="0" y="0" width="25" height="14" rx="2"></rect>
-            <line x1="17" y1="5" x2="19" y2="5" id="Stroke-15" stroke="#2088FF" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></line>
-            <line x1="17" y1="9" x2="21" y2="9" id="Stroke-15" stroke="#2088FF" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></line>
-            <path d="M8.85227273,7 C8.85227273,7.51894199 8.76988719,7.97537682 8.60511364,8.36931818 C8.44034009,8.76325955 8.21591051,9.08711994 7.93181818,9.34090909 L8.76420455,10.3863636 L7.61647727,10.3863636 L7.14772727,9.80965909 C6.83143781,9.92897787 6.49147909,9.98863636 6.12784091,9.98863636 C5.61079287,9.98863636 5.14678236,9.8712133 4.73579545,9.63636364 C4.32480855,9.40151398 4.00000119,9.06108178 3.76136364,8.61505682 C3.52272608,8.16903186 3.40340909,7.63068497 3.40340909,7 C3.40340909,6.36552713 3.52272608,5.8257598 3.76136364,5.38068182 C4.00000119,4.93560384 4.32480855,4.59611859 4.73579545,4.36221591 C5.14678236,4.12831322 5.61079287,4.01136364 6.12784091,4.01136364 C6.642995,4.01136364 7.10605855,4.12831322 7.51704545,4.36221591 C7.92803236,4.59611859 8.2533132,4.93560384 8.49289773,5.38068182 C8.73248226,5.8257598 8.85227273,6.36552713 8.85227273,7 Z M5.70170455,7.88636364 L6.74715909,7.88636364 L7.17897727,8.44034091 C7.31344764,8.27935526 7.41808675,8.07859969 7.49289773,7.83806818 C7.56770871,7.59753668 7.60511364,7.31818341 7.60511364,7 C7.60511364,6.38257267 7.47064528,5.91145996 7.20170455,5.58664773 C6.93276381,5.2618355 6.57481284,5.09943182 6.12784091,5.09943182 C5.68086898,5.09943182 5.32291801,5.2618355 5.05397727,5.58664773 C4.78503653,5.91145996 4.65056818,6.38257267 4.65056818,7 C4.65056818,7.61553338 4.78503653,8.08617261 5.05397727,8.41193182 C5.32291801,8.73769102 5.68086898,8.90056818 6.12784091,8.90056818 C6.23958389,8.90056818 6.34564344,8.89015162 6.44602273,8.86931818 L5.70170455,7.88636364 Z M10.1813315,10 L10.1813315,4 L11.4114451,4 L11.4114451,8.98579545 L13.9057633,8.98579545 L13.9057633,10 L10.1813315,10 Z" fill="#2088FF" fill-rule="nonzero"></path>
-        </g>
-    </g>
-</svg>
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M8.19789 8C8.19789 8.51894 8.1155 8.97538 7.95073 9.36932C7.78595 9.76326 7.56152 10.0871 7.27743 10.3409L8.10982 11.3864H6.96209L6.49334 10.8097C6.17705 10.929 5.83709 10.9886 5.47346 10.9886C4.95641 10.9886 4.4924 10.8712 4.08141 10.6364C3.67042 10.4015 3.34562 10.0611 3.10698 9.61506C2.86834 9.16903 2.74902 8.63068 2.74902 8C2.74902 7.36553 2.86834 6.82576 3.10698 6.38068C3.34562 5.9356 3.67042 5.59612 4.08141 5.36222C4.4924 5.12831 4.95641 5.01136 5.47346 5.01136C5.98861 5.01136 6.45167 5.12831 6.86266 5.36222C7.27365 5.59612 7.59893 5.9356 7.83851 6.38068C8.0781 6.82576 8.19789 7.36553 8.19789 8ZM5.04732 8.88636H6.09277L6.52459 9.44034C6.65906 9.27936 6.7637 9.0786 6.83851 8.83807C6.91332 8.59754 6.95073 8.31818 6.95073 8C6.95073 7.38257 6.81626 6.91146 6.54732 6.58665C6.27838 6.26184 5.92043 6.09943 5.47346 6.09943C5.02648 6.09943 4.66853 6.26184 4.39959 6.58665C4.13065 6.91146 3.99618 7.38257 3.99618 8C3.99618 8.61553 4.13065 9.08617 4.39959 9.41193C4.66853 9.73769 5.02648 9.90057 5.47346 9.90057C5.5852 9.90057 5.69126 9.89015 5.79164 9.86932L5.04732 8.88636ZM9.52695 11V5H10.7571V9.9858H13.2514V11H9.52695Z" fill="#24292F"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M13 1.5H3C2.17157 1.5 1.5 2.17157 1.5 3V13C1.5 13.8284 2.17157 14.5 3 14.5H13C13.8284 14.5 14.5 13.8284 14.5 13V3C14.5 2.17157 13.8284 1.5 13 1.5ZM3 0C1.34315 0 0 1.34315 0 3V13C0 14.6569 1.34315 16 3 16H13C14.6569 16 16 14.6569 16 13V3C16 1.34315 14.6569 0 13 0H3Z" fill="#24292F"/>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.3.4",
+  "version": "1.5.9",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,7 +13,7 @@
     "url": "https://github.com/github/vscode-codeql"
   },
   "engines": {
-    "vscode": "^1.43.0"
+    "vscode": "^1.57.0"
   },
   "categories": [
     "Programming Languages"
@@ -21,6 +21,16 @@
   "extensionDependencies": [
     "hbenl.vscode-test-explorer"
   ],
+  "capabilities": {
+    "untrustedWorkspaces": {
+      "supported": "limited",
+      "description": "Workspace trust is required to execute commands that can contain arbitrary paths.",
+      "restrictedConfigurations": [
+        "codeQL.cli.executablePath",
+        "codeQL.runningTests.additionalTestArguments"
+      ]
+    }
+  },
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
@@ -28,12 +38,15 @@
     "onView:codeQLAstViewer",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
+    "onCommand:codeQL.authenticateToGitHub",
     "onCommand:codeQLDatabases.chooseDatabaseFolder",
     "onCommand:codeQLDatabases.chooseDatabaseArchive",
     "onCommand:codeQLDatabases.chooseDatabaseInternet",
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.openReferencedFile",
+    "onCommand:codeQL.previewQueryHelp",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
@@ -106,15 +119,21 @@
         "path": "./out/syntaxes/dbscheme.tmLanguage.json"
       }
     ],
+    "snippets": [
+      {
+        "language": "ql",
+        "path": "./snippets.json"
+      }
+    ],
     "configuration": {
       "type": "object",
       "title": "CodeQL",
       "properties": {
         "codeQL.cli.executablePath": {
-          "scope": "machine",
+          "scope": "window",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
+          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -123,6 +142,21 @@
           "maximum": 1024,
           "description": "Number of threads for running queries."
         },
+        "codeQL.runningQueries.saveCache": {
+          "type": "boolean",
+          "default": false,
+          "scope": "window",
+          "description": "Aggressively save intermediate results to the disk cache. This may speed up subsequent queries if they are similar. Be aware that using this option will greatly increase disk usage and initial evaluation time."
+        },
+        "codeQL.runningQueries.cacheSize": {
+          "type": [
+            "integer",
+            "null"
+          ],
+          "default": null,
+          "minimum": 1024,
+          "description": "Maximum size of the disk cache (in MB). Leave blank to allow the evaluator to automatically adjust the size of the disk cache based on the size of the codebase and the complexity of the queries being executed."
+        },
         "codeQL.runningQueries.timeout": {
           "type": [
             "integer",
@@ -147,6 +181,13 @@
           "default": false,
           "description": "Enable debug logging and tuple counting when running CodeQL queries. This information is useful for debugging query performance."
         },
+        "codeQL.runningQueries.maxPaths": {
+          "type": "integer",
+          "default": 4,
+          "minimum": 1,
+          "maximum": 256,
+          "markdownDescription": "Max number of paths to display for each alert found by a path query (`@kind path-problem`)."
+        },
         "codeQL.runningQueries.autoSave": {
           "type": "boolean",
           "default": false,
@@ -157,10 +198,34 @@
           "default": 20,
           "description": "Max number of simultaneous queries to run using the 'CodeQL: Run Queries' command."
         },
+        "codeQL.runningQueries.customLogDirectory": {
+          "type": [
+            "string",
+            null
+          ],
+          "default": null,
+          "description": "Path to a directory where the CodeQL extension should store query server logs. If empty, the extension stores logs in a temporary workspace folder and deletes the contents after each run."
+        },
+        "codeQL.runningQueries.quickEvalCodelens": {
+          "type": "boolean",
+          "default": true,
+          "description": "Enable the 'Quick Evaluation' CodeLens."
+        },
+        "codeQL.resultsDisplay.pageSize": {
+          "type": "integer",
+          "default": 200,
+          "description": "Max number of query results to display per page in the results view."
+        },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "[%t] %q on %d - %s",
-          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+          "default": "%q on %d - %s, %r result count [%t]",
+          "markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
+        },
+        "codeQL.runningTests.additionalTestArguments": {
+          "scope": "window",
+          "type": "array",
+          "default": [],
+          "markdownDescription": "Additional command line arguments to pass to the CLI when [running tests](https://codeql.github.com/docs/codeql-cli/manual/test-run/). This setting should be an array of strings, each containing an argument to be passed."
         },
         "codeQL.runningTests.numberOfThreads": {
           "scope": "window",
@@ -169,14 +234,61 @@
           "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running CodeQL tests."
+        },
+        "codeQL.telemetry.enableTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "markdownDescription": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub. For more information, see the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code)"
+        },
+        "codeQL.telemetry.logTelemetry": {
+          "type": "boolean",
+          "default": false,
+          "scope": "application",
+          "description": "Specifies whether or not to write telemetry events to the extension log."
+        },
+        "codeQL.remoteQueries.repositoryLists": {
+          "type": [
+            "object",
+            null
+          ],
+          "patternProperties": {
+            ".*": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "default": null,
+          "markdownDescription": "[For internal use only] Lists of GitHub repositories that you want to query remotely. This should be a JSON object where each key is a user-specified name for this repository list, and the value is an array of GitHub repositories (of the form `<owner>/<repo>`)."
+        },
+        "codeQL.remoteQueries.controllerRepo": {
+          "type": "string",
+          "default": "",
+          "pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
+          "patternErrorMessage": "Please enter a valid GitHub repository",
+          "markdownDescription": "[For internal use only] The name of the GitHub repository where you can view the progress and results of the \"Run Remote query\" command. The repository should be of the form `<owner>/<repo>`)."
         }
       }
     },
     "commands": [
+      {
+        "command": "codeQL.authenticateToGitHub",
+        "title": "CodeQL: Authenticate to GitHub"
+      },
       {
         "command": "codeQL.runQuery",
         "title": "CodeQL: Run Query"
       },
+      {
+        "command": "codeQL.runQueryOnMultipleDatabases",
+        "title": "CodeQL: Run Query on Multiple Databases"
+      },
+      {
+        "command": "codeQL.runRemoteQuery",
+        "title": "CodeQL: Run Remote Query"
+      },
       {
         "command": "codeQL.runQueries",
         "title": "CodeQL: Run Queries in Selected Files"
@@ -185,10 +297,26 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.openReferencedFile",
+        "title": "CodeQL: Open Referenced File"
+      },
+      {
+        "command": "codeQL.previewQueryHelp",
+        "title": "CodeQL: Preview Query Help"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
       },
+      {
+        "command": "codeQL.openDocumentation",
+        "title": "CodeQL: Open Documentation"
+      },
+      {
+        "command": "codeQL.copyVersion",
+        "title": "CodeQL: Copy Version Information"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
@@ -197,6 +325,10 @@
           "dark": "media/dark/folder-opened-plus.svg"
         }
       },
+      {
+        "command": "codeQLDatabases.removeOrphanedDatabases",
+        "title": "Delete unused databases"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseArchive",
         "title": "Choose Database from Archive",
@@ -257,6 +389,10 @@
         "command": "codeQLDatabases.openDatabaseFolder",
         "title": "Show Database Directory"
       },
+      {
+        "command": "codeQLDatabases.addDatabaseSource",
+        "title": "Add Database Source to Workspace"
+      },
       {
         "command": "codeQL.chooseDatabaseFolder",
         "title": "CodeQL: Choose Database from Folder"
@@ -295,7 +431,7 @@
       },
       {
         "command": "codeQLQueryHistory.openQuery",
-        "title": "Open Query",
+        "title": "Open the query that produced these results",
         "icon": {
           "light": "media/light/edit.svg",
           "dark": "media/dark/edit.svg"
@@ -317,6 +453,30 @@
           "dark": "media/dark/trash.svg"
         }
       },
+      {
+        "command": "codeQLQueryHistory.sortByName",
+        "title": "Sort by Name",
+        "icon": {
+          "light": "media/light/sort-alpha.svg",
+          "dark": "media/dark/sort-alpha.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByDate",
+        "title": "Sort by Query Date",
+        "icon": {
+          "light": "media/light/sort-date.svg",
+          "dark": "media/dark/sort-date.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByCount",
+        "title": "Sort by Results Count",
+        "icon": {
+          "light": "media/light/sort-num.svg",
+          "dark": "media/dark/sort-num.svg"
+        }
+      },
       {
         "command": "codeQLQueryHistory.showQueryLog",
         "title": "Show Query Log"
@@ -326,8 +486,16 @@
         "title": "Show Query Text"
       },
       {
-        "command": "codeQLQueryHistory.viewSarif",
-        "title": "View SARIF"
+        "command": "codeQLQueryHistory.viewCsvResults",
+        "title": "View Results (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewCsvAlerts",
+        "title": "View Alerts (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarifAlerts",
+        "title": "View Alerts (SARIF)"
       },
       {
         "command": "codeQLQueryHistory.viewDil",
@@ -355,11 +523,11 @@
       },
       {
         "command": "codeQLTests.showOutputDifferences",
-        "title": "CodeQL: Show Test Output Differences"
+        "title": "Show Test Output Differences"
       },
       {
         "command": "codeQLTests.acceptOutput",
-        "title": "CodeQL: Accept Test Output"
+        "title": "Accept Test Output"
       },
       {
         "command": "codeQLAstViewer.gotoCode",
@@ -421,6 +589,21 @@
           "when": "view == codeQLQueryHistory",
           "group": "navigation"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
         {
           "command": "codeQLAstViewer.clear",
           "when": "view == codeQLAstViewer",
@@ -431,7 +614,7 @@
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
-          "when": "view == codeQLDatabases"
+          "when": "view == codeQLDatabases && viewItem != currentDatabase"
         },
         {
           "command": "codeQLDatabases.removeDatabase",
@@ -453,6 +636,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLDatabases"
         },
+        {
+          "command": "codeQLDatabases.addDatabaseSource",
+          "group": "9_qlCommands",
+          "when": "view == codeQLDatabases"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "group": "9_qlCommands",
@@ -484,7 +672,17 @@
           "when": "view == codeQLQueryHistory"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem != interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewCsvAlerts",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifAlerts",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
         },
@@ -493,6 +691,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
         },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && resourceScheme == .qhelp && isWorkspaceTrusted"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -517,14 +720,37 @@
         },
         {
           "command": "codeQL.runQueries",
-          "group": "9_qlCommands"
+          "group": "9_qlCommands",
+          "when": "resourceScheme != codeql-zip-archive"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .qlref"
+        },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
       ],
       "commandPalette": [
+        {
+          "command": "codeQL.authenticateToGitHub",
+          "when": "config.codeQL.canary"
+        },
         {
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueryOnMultipleDatabases",
+          "when": "resourceLangId == ql && resourceExtname == .ql"
+        },
+        {
+          "command": "codeQL.runRemoteQuery",
+          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
+        },
         {
           "command": "codeQL.runQueries",
           "when": "false"
@@ -533,6 +759,14 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
+        },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -553,6 +787,10 @@
           "command": "codeQLDatabases.openDatabaseFolder",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.addDatabaseSource",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.sortByName",
           "when": "false"
@@ -573,6 +811,10 @@
           "command": "codeQLDatabases.chooseDatabaseArchive",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.removeOrphanedDatabases",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.chooseDatabaseInternet",
           "when": "false"
@@ -606,7 +848,15 @@
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewCsvAlerts",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifAlerts",
           "when": "false"
         },
         {
@@ -621,6 +871,18 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "false"
+        },
         {
           "command": "codeQLAstViewer.gotoCode",
           "when": "false"
@@ -628,6 +890,14 @@
         {
           "command": "codeQLAstViewer.clear",
           "when": "false"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "when": "false"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -635,9 +905,29 @@
           "command": "codeQL.runQuery",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueryOnMultipleDatabases",
+          "when": "editorLangId == ql && resourceExtname == .ql"
+        },
+        {
+          "command": "codeQL.runRemoteQuery",
+          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
+        },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
+        },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
       ]
     },
@@ -669,7 +959,7 @@
     "viewsWelcome": [
       {
         "view": "codeQLAstViewer",
-        "contents": "Run the 'CodeQL: View AST' command on an open source file from a Code QL database.\n[View AST](command:codeQL.viewAst)"
+        "contents": "Run the 'CodeQL: View AST' command on an open source file from a CodeQL database.\n[View AST](command:codeQL.viewAst)"
       },
       {
         "view": "codeQLQueryHistory",
@@ -677,7 +967,7 @@
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a Code QL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
       }
     ]
   },
@@ -687,27 +977,34 @@
     "watch:extension": "tsc --watch",
     "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
-    "integration": "node ./out/vscode-tests/run-integration-tests.js",
+    "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
+    "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
     "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
     "format-staged": "lint-staged"
   },
   "dependencies": {
+    "@octokit/rest": "^18.5.6",
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.0.1",
     "glob-promise": "^3.4.0",
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.5",
     "node-fetch": "~2.6.0",
+    "path-browserify": "^1.0.1",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "semver": "~7.3.2",
+    "stream": "^0.0.2",
+    "stream-chain": "~2.2.4",
+    "stream-json": "~1.7.3",
     "tmp": "^0.1.0",
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
+    "vscode-extension-telemetry": "^0.1.6",
     "vscode-jsonrpc": "^5.0.1",
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
@@ -719,15 +1016,16 @@
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^8.0.0",
+    "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
-    "@types/gulp": "^4.0.6",
+    "@types/gulp": "^4.0.9",
+    "@types/gulp-replace": "0.0.31",
     "@types/gulp-sourcemaps": "0.0.32",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
-    "@types/mocha": "~8.0.3",
-    "@types/node": "^12.0.8",
+    "@types/mocha": "^9.0.0",
+    "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
     "@types/react": "^16.8.17",
@@ -736,15 +1034,18 @@
     "@types/semver": "~7.2.0",
     "@types/sinon": "~7.5.2",
     "@types/sinon-chai": "~3.2.3",
+    "@types/stream-chain": "~2.0.1",
+    "@types/stream-json": "~1.7.1",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.43.0",
+    "@types/vscode": "^1.57.0",
     "@types/webpack": "^4.32.1",
     "@types/xml2js": "~0.4.4",
-    "@typescript-eslint/eslint-plugin": "~2.23.0",
-    "@typescript-eslint/parser": "~2.23.0",
+    "@typescript-eslint/eslint-plugin": "^4.26.0",
+    "@typescript-eslint/parser": "^4.26.0",
     "ansi-colors": "^4.1.1",
+    "applicationinsights": "^1.8.7",
     "chai": "^4.2.0",
     "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
@@ -752,13 +1053,14 @@
     "eslint-plugin-react": "~7.19.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
+    "gulp-replace": "^1.0.0",
     "gulp-sourcemaps": "^2.6.5",
     "gulp-typescript": "^5.0.1",
     "husky": "~4.2.5",
     "jsonc-parser": "^2.3.0",
     "lint-staged": "~10.2.2",
-    "mocha": "~8.1.3",
-    "mocha-sinon": "~2.1.0",
+    "mocha": "^9.1.3",
+    "mocha-sinon": "~2.1.2",
     "npm-run-all": "^4.1.5",
     "prettier": "~2.0.5",
     "proxyquire": "~2.1.3",
@@ -766,15 +1068,15 @@
     "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
-    "ts-loader": "^5.4.5",
+    "ts-loader": "^8.1.0",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "~3.8.3",
+    "typescript": "^4.3.2",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
     "vscode-test": "^1.4.0",
-    "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2"
+    "webpack": "^5.28.0",
+    "webpack-cli": "^4.6.0"
   },
   "husky": {
     "hooks": {
@@ -790,5 +1092,8 @@
       "tsfmt -r",
       "eslint --fix"
     ]
+  },
+  "resolutions": {
+    "glob-parent": "~6.0.0"
   }
 }

extensions/ql-vscode/snippets.json

@@ -0,0 +1,134 @@
+{
+  "Query Metadata": {
+    "prefix": "querymetadata",
+    "body": [
+      "/**",
+      " * @name $1",
+      " * @description $2",
+      " * @kind $3",
+      " * @id $4",
+      " * @tags $5",
+      " */"
+    ],
+    "description": "Metadata for a query"
+  },
+  "Class": {
+    "prefix": "class",
+    "body": ["class $1 extends $2 {", "\t$0", "}"],
+    "description": "A class"
+  },
+  "From/Where/Select": {
+    "prefix": "from",
+    "body": ["from $1", "where $2", "select $3"],
+    "description": "A from/where/select statement"
+  },
+  "Predicate": {
+    "prefix": "predicate",
+    "body": ["predicate $1($2) {", "\t$0", "}"],
+    "description": "A predicate"
+  },
+  "Dataflow Tracking Class": {
+    "prefix": "dataflowtracking",
+    "body": [
+      "class $1 extends DataFlow::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a dataflow tracking class"
+  },
+  "Taint Tracking Class": {
+    "prefix": "tainttracking",
+    "body": [
+      "class $1 extends TaintTracking::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a taint tracking class"
+  },
+  "Count": {
+    "prefix": "count",
+    "body": ["count($1 | $2 | $3)"],
+    "description": "A count aggregate"
+  },
+  "Max": {
+    "prefix": "max",
+    "body": ["max($1 | $2 | $3)"],
+    "description": "A max aggregate"
+  },
+  "Min": {
+    "prefix": "min",
+    "body": ["min($1 | $2 | $3)"],
+    "description": "A min aggregate"
+  },
+  "Average": {
+    "prefix": "avg",
+    "body": ["avg($1 | $2 | $3)"],
+    "description": "An average aggregate"
+  },
+  "Sum": {
+    "prefix": "sum",
+    "body": ["sum($1 | $2 | $3)"],
+    "description": "A sum aggregate"
+  },
+  "Concatenation": {
+    "prefix": "concat",
+    "body": ["concat($1 | $2 | $3)"],
+    "description": "A concatenation aggregate"
+  },
+  "Rank": {
+    "prefix": "rank",
+    "body": ["rank[$1]($2 | $3 | $4)"],
+    "description": "A rank aggregate"
+  },
+  "Strict Sum": {
+    "prefix": "strictsum",
+    "body": ["strictsum($1 | $2 | $3)"],
+    "description": "A strict sum aggregate"
+  },
+  "Strict Concatenation": {
+    "prefix": "strictconcat",
+    "body": ["strictconcat($1 | $2 | $3)"],
+    "description": "A strict concatenation aggregate"
+  },
+  "Strict Count": {
+    "prefix": "strictcount",
+    "body": ["strictcount($1 | $2 | $3)"],
+    "description": "A strict count aggregate"
+  },
+  "Unique": {
+    "prefix": "unique",
+    "body": ["unique($1 | $2 | $3)"],
+    "description": "A unique aggregate"
+  },
+  "Exists": {
+    "prefix": "exists",
+    "body": ["exists($1 | $2 | $3)"],
+    "description": "An exists quantifier"
+  },
+  "For All": {
+    "prefix": "forall",
+    "body": ["forall($1 | $2 | $3)"],
+    "description": "A for all quantifier"
+  },
+  "For All and Exists": {
+    "prefix": "forex",
+    "body": ["forex($1 | $2 | $3)"],
+    "description": "A for all and exists quantifier"
+  }
+}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -84,7 +84,7 @@ export function encodeSourceArchiveUri(ref: ZipFileReference): vscode.Uri {
   // This lets us separate the paths, ignoring the leading slash if we added one.
   const sourceArchiveZipPathEndIndex = sourceArchiveZipPathStartIndex + sourceArchiveZipPath.length;
   const authority = `${sourceArchiveZipPathStartIndex}-${sourceArchiveZipPathEndIndex}`;
-  return vscode.Uri.parse(zipArchiveScheme + ':/').with({
+  return vscode.Uri.parse(zipArchiveScheme + ':/', true).with({
     path: encodedPath,
     authority,
   });
@@ -115,9 +115,9 @@ class InvalidSourceArchiveUriError extends Error {
 export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
   if (!uri.authority) {
     // Uri is malformed, but this is recoverable
-    logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
+    void logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
     return {
-      pathWithinSourceArchive: '',
+      pathWithinSourceArchive: '/',
       sourceArchiveZipPath: uri.path
     };
   }
@@ -129,7 +129,7 @@ export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
   if (isNaN(zipPathStartIndex) || isNaN(zipPathEndIndex))
     throw new InvalidSourceArchiveUriError(uri);
   return {
-    pathWithinSourceArchive: uri.path.substring(zipPathEndIndex),
+    pathWithinSourceArchive: uri.path.substring(zipPathEndIndex) || '/',
     sourceArchiveZipPath: uri.path.substring(zipPathStartIndex, zipPathEndIndex),
   };
 }
@@ -141,7 +141,7 @@ function ensureFile(map: DirectoryHierarchyMap, file: string) {
   const dirname = path.dirname(file);
   if (dirname === '.') {
     const error = `Ill-formed path ${file} in zip archive (expected absolute path)`;
-    logger.log(error);
+    void logger.log(error);
     throw new Error(error);
   }
   ensureDir(map, dirname);

extensions/ql-vscode/src/astViewer.ts

@@ -8,17 +8,19 @@ import {
   TreeItem,
   TreeView,
   TextEditorSelectionChangeEvent,
+  TextEditorSelectionChangeKind,
   Location,
   Range
 } from 'vscode';
 import * as path from 'path';
 
 import { DatabaseItem } from './databases';
-import { UrlValue, BqrsId } from './bqrs-cli-types';
+import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
 import { showLocation } from './interface-utils';
-import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './bqrs-utils';
-import { commandRunner } from './helpers';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
+import { commandRunner } from './commandRunner';
+import { DisposableObject } from './pure/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
 
 export interface AstItem {
   id: BqrsId;
@@ -33,13 +35,13 @@ export interface ChildAstItem extends AstItem {
   parent: ChildAstItem | AstItem;
 }
 
-class AstViewerDataProvider  extends DisposableObject implements TreeDataProvider<AstItem> {
+class AstViewerDataProvider extends DisposableObject implements TreeDataProvider<AstItem> {
 
   public roots: AstItem[] = [];
   public db: DatabaseItem | undefined;
 
   private _onDidChangeTreeData =
-    new EventEmitter<AstItem | undefined>();
+    this.push(new EventEmitter<AstItem | undefined>());
   readonly onDidChangeTreeData: Event<AstItem | undefined> =
     this._onDidChangeTreeData.event;
 
@@ -47,14 +49,14 @@ class AstViewerDataProvider  extends DisposableObject implements TreeDataProvide
     super();
     this.push(
       commandRunner('codeQLAstViewer.gotoCode',
-      async (item: AstItem) => {
-        await showLocation(item.fileLocation);
-      })
+        async (item: AstItem) => {
+          await showLocation(item.fileLocation);
+        })
     );
   }
 
   refresh(): void {
-    this._onDidChangeTreeData.fire();
+    this._onDidChangeTreeData.fire(undefined);
   }
   getChildren(item?: AstItem): ProviderResult<AstItem[]> {
     const children = item ? item.children : this.roots;
@@ -128,8 +130,13 @@ export class AstViewer extends DisposableObject {
     this.treeDataProvider.db = db;
     this.treeDataProvider.refresh();
     this.treeView.message = `AST for ${path.basename(fileName)}`;
-    this.treeView.reveal(roots[0], { focus: false });
     this.currentFile = fileName;
+    // Handle error on reveal. This could happen if
+    // the tree view is disposed during the reveal.
+    this.treeView.reveal(roots[0], { focus: false })?.then(
+      () => { /**/ },
+      err => showAndLogErrorMessage(err)
+    );
   }
 
   private updateTreeSelection(e: TextEditorSelectionChangeEvent) {
@@ -160,6 +167,11 @@ export class AstViewer extends DisposableObject {
       return;
     }
 
+    // Avoid recursive tree-source code updates.
+    if (e.kind === TextEditorSelectionChangeKind.Command) {
+      return;
+    }
+
     if (
       this.treeView.visible &&
       e.textEditor.document.uri.fsPath === this.currentFile &&
@@ -172,7 +184,12 @@ export class AstViewer extends DisposableObject {
 
       const targetItem = findBest(range, this.treeDataProvider.roots);
       if (targetItem) {
-        this.treeView.reveal(targetItem);
+        // Handle error on reveal. This could happen if
+        // the tree view is disposed during the reveal.
+        this.treeView.reveal(targetItem)?.then(
+          () => { /**/ },
+          err => showAndLogErrorMessage(err)
+        );
       }
     }
   }

extensions/ql-vscode/src/authentication.ts

@@ -0,0 +1,62 @@
+import * as vscode from 'vscode';
+import * as Octokit from '@octokit/rest';
+
+const GITHUB_AUTH_PROVIDER_ID = 'github';
+
+// 'repo' scope should be enough for triggering workflows. For a comprehensive list, see:
+// https://docs.github.com/apps/building-oauth-apps/understanding-scopes-for-oauth-apps
+const SCOPES = ['repo'];
+
+/** 
+ * Handles authentication to GitHub, using the VS Code [authentication API](https://code.visualstudio.com/api/references/vscode-api#authentication).
+ */
+export class Credentials {
+  private octokit: Octokit.Octokit | undefined;
+
+  // Explicitly make the constructor private, so that we can't accidentally call the constructor from outside the class
+  // without also initializing the class.
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() { }
+
+  static async initialize(context: vscode.ExtensionContext): Promise<Credentials> {
+    const c = new Credentials();
+    c.registerListeners(context);
+    c.octokit = await c.createOctokit(false);
+    return c;
+  }
+
+  private async createOctokit(createIfNone: boolean): Promise<Octokit.Octokit | undefined> {
+    const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
+
+    if (session) {
+      return new Octokit.Octokit({
+        auth: session.accessToken
+      });
+    } else {
+      return undefined;
+    }
+  }
+
+  registerListeners(context: vscode.ExtensionContext): void {
+    // Sessions are changed when a user logs in or logs out.
+    context.subscriptions.push(vscode.authentication.onDidChangeSessions(async e => {
+      if (e.provider.id === GITHUB_AUTH_PROVIDER_ID) {
+        this.octokit = await this.createOctokit(false);
+      }
+    }));
+  }
+
+  async getOctokit(): Promise<Octokit.Octokit> {
+    if (this.octokit) {
+      return this.octokit;
+    }
+
+    this.octokit = await this.createOctokit(true);
+    // octokit shouldn't be undefined, since we've set "createIfNone: true".
+    // The following block is mainly here to prevent a compiler error.
+    if (!this.octokit) {
+      throw new Error('Did not initialize Octokit.');
+    }
+    return this.octokit;
+  }
+}

extensions/ql-vscode/src/cli-version.ts

@@ -18,7 +18,7 @@ export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): P
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.
-    logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
+    void logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
     return undefined;
   }
 }

extensions/ql-vscode/src/cli.ts

@@ -1,27 +1,34 @@
-/* eslint-disable @typescript-eslint/camelcase */
 import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
-import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
+import { SemVer } from 'semver';
 import { Readable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as tk from 'tree-kill';
-import * as util from 'util';
-import { CancellationToken, Disposable } from 'vscode';
+import { promisify } from 'util';
+import { CancellationToken, Disposable, Uri } from 'vscode';
 
-import { BQRSInfo, DecodedBqrsChunk } from './bqrs-cli-types';
+import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
 import { CliConfig } from './config';
-import { DistributionProvider } from './distribution';
-import { assertNever } from './helpers-pure';
-import { QueryMetadata, SortDirection } from './interface-types';
+import { DistributionProvider, FindDistributionResultKind } from './distribution';
+import { assertNever } from './pure/helpers-pure';
+import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
+import { CompilationMessage } from './pure/messages';
+import { sarifParser } from './sarif-parser';
+import { dbSchemeToLanguage } from './helpers';
 
 /**
  * The version of the SARIF format that we are using.
  */
 const SARIF_FORMAT = 'sarifv2.1.0';
 
+/**
+ * The string used to specify CSV format.
+ */
+const CSV_FORMAT = 'csv';
+
 /**
  * Flags to pass to all cli commands.
  */
@@ -37,6 +44,16 @@ export interface QuerySetup {
   compilationCache?: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries --format bylanguage`.
+ */
+export interface QueryInfoByLanguage {
+  // Using `unknown` as a placeholder. For now, the value is only ever an empty object.
+  byLanguage: Record<string, Record<string, unknown>>;
+  noDeclaredLanguage: Record<string, unknown>;
+  multipleDeclaredLanguages: Record<string, unknown>;
+}
+
 /**
  * The expected output of `codeql resolve database`.
  */
@@ -48,6 +65,7 @@ export interface DbInfo {
   sourceArchiveRoot: string;
   datasetFolder: string;
   logsFolder: string;
+  languages: string[];
 }
 
 /**
@@ -56,6 +74,7 @@ export interface DbInfo {
 export interface UpgradesInfo {
   scripts: string[];
   finalDbscheme: string;
+  matchesTarget?: boolean;
 }
 
 /**
@@ -63,6 +82,25 @@ export interface UpgradesInfo {
  */
 export type QlpacksInfo = { [name: string]: string[] };
 
+/**
+ * The expected output of `codeql resolve languages`.
+ */
+export type LanguagesInfo = { [name: string]: string[] };
+
+/** Information about an ML model, as resolved by `codeql resolve ml-models`. */
+export type MlModelInfo = {
+  checksum: string;
+  path: string;
+};
+
+/** The expected output of `codeql resolve ml-models`. */
+export type MlModelsInfo = { models: MlModelInfo[] };
+
+/**
+ * The expected output of `codeql resolve qlref`.
+ */
+export type QlrefInfo = { resolvedPath: string };
+
 // `codeql bqrs interpret` requires both of these to be present or
 // both absent.
 export interface SourceInfo {
@@ -89,10 +127,13 @@ export interface TestRunOptions {
 export interface TestCompleted {
   test: string;
   pass: boolean;
-  messages: string[];
+  messages: CompilationMessage[];
   compilationMs: number;
   evaluationMs: number;
   expected: string;
+  diff: string[] | undefined;
+  failureDescription?: string;
+  failureStage?: string;
 }
 
 /**
@@ -115,6 +156,7 @@ interface BqrsDecodeOptions {
  */
 export class CodeQLCliServer implements Disposable {
 
+
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
   /** Queue of future commands*/
@@ -124,10 +166,28 @@ export class CodeQLCliServer implements Disposable {
   /**  A buffer with a single null byte. */
   nullBuffer: Buffer;
 
+  /** Version of current cli, lazily computed by the `getVersion()` method */
+  private _version: SemVer | undefined;
+
+  /**
+   * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
+   */
+  private _supportedLanguages: string[] | undefined;
+
+  /** Path to current codeQL executable, or undefined if not running yet. */
+  codeQlPath: string | undefined;
+
+  cliConstraints = new CliVersionConstraint(this);
+
+  /**
+   * When set to true, ignore some modal popups and assume user has clicked "yes".
+   */
+  public quiet = false;
+
   constructor(
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    private logger: Logger
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -135,16 +195,19 @@ export class CodeQLCliServer implements Disposable {
     if (this.distributionProvider.onDidChangeDistribution) {
       this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
+        this._version = undefined;
+        this._supportedLanguages = undefined;
       });
     }
     if (this.cliConfig.onDidChangeConfiguration) {
       this.cliConfig.onDidChangeConfiguration(() => {
         this.restartCliServer();
+        this._version = undefined;
+        this._supportedLanguages = undefined;
       });
     }
   }
 
-
   dispose(): void {
     this.killProcessIfRunning();
   }
@@ -152,15 +215,15 @@ export class CodeQLCliServer implements Disposable {
   killProcessIfRunning(): void {
     if (this.process) {
       // Tell the Java CLI server process to shut down.
-      this.logger.log('Sending shutdown request');
+      void this.logger.log('Sending shutdown request');
       try {
         this.process.stdin.write(JSON.stringify(['shutdown']), 'utf8');
         this.process.stdin.write(this.nullBuffer);
-        this.logger.log('Sent shutdown request');
+        void this.logger.log('Sent shutdown request');
       } catch (e) {
         // We are probably fine here, the process has already closed stdin.
-        this.logger.log(`Shutdown request failed: process stdin may have already closed. The error was ${e}`);
-        this.logger.log('Stopping the process anyway.');
+        void this.logger.log(`Shutdown request failed: process stdin may have already closed. The error was ${e}`);
+        void this.logger.log('Stopping the process anyway.');
       }
       // Close the stdin and stdout streams.
       // This is important on Windows where the child process may not die cleanly.
@@ -210,12 +273,17 @@ export class CodeQLCliServer implements Disposable {
    * Launch the cli server
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
-    const config = await this.getCodeQlPath();
-    return spawnServer(
-      config,
+    const codeQlPath = await this.getCodeQlPath();
+    const args = [];
+    if (shouldDebugCliServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9012,server=n,suspend=y,quiet=y');
+    }
+
+    return await spawnServer(
+      codeQlPath,
       'CodeQL CLI Server',
       ['execute', 'cli-server'],
-      [],
+      args,
       this.logger,
       _data => { /**/ }
     );
@@ -240,9 +308,9 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(' ');
-      this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
-        await new Promise((resolve, reject) => {
+        await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
           process.stdout.addListener('data', (newData: Buffer) => {
             stdoutBuffers.push(newData);
@@ -267,7 +335,7 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString('utf8', 0, fullBuffer.length - 1);
-        this.logger.log('CLI command succeeded.');
+        void this.logger.log('CLI command succeeded.');
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
@@ -280,7 +348,7 @@ export class CodeQLCliServer implements Disposable {
         newError.stack += (err.stack || '');
         throw newError;
       } finally {
-        this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
+        void this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
         // Remove the listeners we set up.
         process.stdout.removeAllListeners('data');
         process.stderr.removeAllListeners('data');
@@ -340,7 +408,7 @@ export class CodeQLCliServer implements Disposable {
       }
       if (logger !== undefined) {
         // The human-readable output goes to stderr.
-        logStream(child.stderr!, logger);
+        void logStream(child.stderr!, logger);
       }
 
       for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
@@ -421,12 +489,15 @@ export class CodeQLCliServer implements Disposable {
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
    * @param commandArgs The arguments to pass to the `codeql` command.
    * @param description Description of the action being run, to be shown in log and error messages.
+   * @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
    * @param progressReporter Used to output progress messages, e.g. to the status bar.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
-  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, progressReporter?: ProgressReporter): Promise<OutputType> {
-    // Add format argument first, in case commandArgs contains positional parameters.
-    const args = ['--format', 'json'].concat(commandArgs);
+  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, addFormat = true, progressReporter?: ProgressReporter): Promise<OutputType> {
+    let args: string[] = [];
+    if (addFormat) // Add format argument first, in case commandArgs contains positional parameters.
+      args = args.concat(['--format', 'json']);
+    args = args.concat(commandArgs);
     const result = await this.runCodeQlCliCommand(command, args, description, progressReporter);
     try {
       return JSON.parse(result) as OutputType;
@@ -449,6 +520,20 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, 'Resolving library paths');
   }
 
+  /**
+   * Resolves the language for a query.
+   * @param queryUri The URI of the query
+   */
+  async resolveQueryByLanguage(workspaces: string[], queryUri: Uri): Promise<QueryInfoByLanguage> {
+    const subcommandArgs = [
+      '--format', 'bylanguage',
+      queryUri.fsPath,
+      '--additional-packs',
+      workspaces.join(path.delimiter)
+    ];
+    return JSON.parse(await this.runCodeQlCliCommand(['resolve', 'queries'], subcommandArgs, 'Resolving query by language'));
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.
@@ -458,7 +543,23 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       testPath
     ];
-    return await this.runJsonCodeQlCliCommand<ResolvedTests>(['resolve', 'tests'], subcommandArgs, 'Resolving tests');
+    return await this.runJsonCodeQlCliCommand<ResolvedTests>(
+      ['resolve', 'tests', '--strict-test-discovery'],
+      subcommandArgs,
+      'Resolving tests'
+    );
+  }
+
+  public async resolveQlref(qlref: string): Promise<QlrefInfo> {
+    const subcommandArgs = [
+      qlref
+    ];
+    return await this.runJsonCodeQlCliCommand<QlrefInfo>(
+      ['resolve', 'qlref'],
+      subcommandArgs,
+      'Resolving qlref',
+      false
+    );
   }
 
   /**
@@ -471,12 +572,12 @@ export class CodeQLCliServer implements Disposable {
     testPaths: string[], workspaces: string[], options: TestRunOptions
   ): AsyncGenerator<TestCompleted, void, unknown> {
 
-    const subcommandArgs = [
+    const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
       '--additional-packs', workspaces.join(path.delimiter),
       '--threads',
       this.cliConfig.numberTestThreads.toString(),
       ...testPaths
-    ];
+    ]);
 
     for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
       subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
@@ -492,6 +593,12 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<QueryMetadata>(['resolve', 'metadata'], [queryPath], 'Resolving query metadata');
   }
 
+  /** Resolves the ML models that should be available when evaluating a query. */
+  async resolveMlModels(additionalPacks: string[]): Promise<MlModelsInfo> {
+    return await this.runJsonCodeQlCliCommand<MlModelsInfo>(['resolve', 'ml-models'], ['--additional-packs',
+      additionalPacks.join(path.delimiter)], 'Resolving ML models', false);
+  }
+
   /**
    * Gets the RAM setting for the query server.
    * @param queryMemoryMb The maximum amount of RAM to use, in MB.
@@ -505,7 +612,7 @@ export class CodeQLCliServer implements Disposable {
     if (queryMemoryMb !== undefined) {
       args.push('--ram', queryMemoryMb.toString());
     }
-    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', progressReporter);
+    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', true, progressReporter);
   }
   /**
    * Gets the headers (and optionally pagination info) of a bqrs.
@@ -521,6 +628,29 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, 'Reading bqrs header');
   }
 
+  async databaseUnbundle(archivePath: string, target: string, name?: string): Promise<string> {
+    const subcommandArgs = [];
+    if (target) subcommandArgs.push('--target', target);
+    if (name) subcommandArgs.push('--name', name);
+    subcommandArgs.push(archivePath);
+
+    return await this.runCodeQlCliCommand(['database', 'unbundle'], subcommandArgs, `Extracting ${archivePath} to directory ${target}`);
+  }
+
+  /**
+   * Uses a .qhelp file to generate Query Help documentation in a specified format.
+   * @param pathToQhelp The path to the .qhelp file
+   * @param format The format in which the query help should be generated {@link https://codeql.github.com/docs/codeql-cli/manual/generate-query-help/#cmdoption-codeql-generate-query-help-format}
+   * @param outputDirectory The output directory for the generated file
+   */
+  async generateQueryHelp(pathToQhelp: string, outputDirectory?: string): Promise<string> {
+    const subcommandArgs = ['--format=markdown'];
+    if (outputDirectory) subcommandArgs.push('--output', outputDirectory);
+    subcommandArgs.push(pathToQhelp);
+
+    return await this.runCodeQlCliCommand(['generate', 'query-help'], subcommandArgs, `Generating qhelp in markdown format at ${outputDirectory}`);
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
@@ -544,40 +674,49 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async interpretBqrs(metadata: { kind: string; id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
-      `-t=kind=${metadata.kind}`,
-      `-t=id=${metadata.id}`,
       '--output', interpretedResultsPath,
-      '--format', SARIF_FORMAT,
+      '--format', format,
+      // Forward all of the query metadata.
+      ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`)
+    ];
+    if (format == SARIF_FORMAT) {
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      '--no-group-results',
-    ];
+      args.push('--no-group-results');
+    }
     if (sourceInfo !== undefined) {
       args.push(
         '--source-archive', sourceInfo.sourceArchive,
         '--source-location-prefix', sourceInfo.sourceLocationPrefix
       );
     }
+
+    args.push(
+      '--threads',
+      this.cliConfig.numberThreads.toString(),
+    );
+
+    args.push(
+      '--max-paths',
+      this.cliConfig.maxPaths.toString(),
+    );
+
     args.push(resultsPath);
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
-
-    let output: string;
-    try {
-      output = await fs.readFile(interpretedResultsPath, 'utf8');
-    } catch (err) {
-      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
-    }
-    try {
-      return JSON.parse(output) as sarif.Log;
-    } catch (err) {
-      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
-    }
   }
 
+  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
+    return await sarifParser(interpretedResultsPath);
+  }
+
+  async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
+    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+  }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
     const sortDirectionStrings = sortDirections.map(direction => {
@@ -617,12 +756,19 @@ export class CodeQLCliServer implements Disposable {
    * Gets information necessary for upgrading a database.
    * @param dbScheme the path to the dbscheme of the database to be upgraded.
    * @param searchPath A list of directories to search for upgrade scripts.
+   * @param allowDowngradesIfPossible Whether we should try and include downgrades of we can.
+   * @param targetDbScheme The dbscheme to try to upgrade to.
    * @returns A list of database upgrade script directories
    */
-  resolveUpgrades(dbScheme: string, searchPath: string[]): Promise<UpgradesInfo> {
+  async resolveUpgrades(dbScheme: string, searchPath: string[], allowDowngradesIfPossible: boolean, targetDbScheme?: string): Promise<UpgradesInfo> {
     const args = ['--additional-packs', searchPath.join(path.delimiter), '--dbscheme', dbScheme];
-
-    return this.runJsonCodeQlCliCommand<UpgradesInfo>(
+    if (targetDbScheme) {
+      args.push('--target-dbscheme', targetDbScheme);
+      if (allowDowngradesIfPossible && await this.cliConstraints.supportsDowngrades()) {
+        args.push('--allow-downgrades');
+      }
+    }
+    return await this.runJsonCodeQlCliCommand<UpgradesInfo>(
       ['resolve', 'upgrades'],
       args,
       'Resolving database upgrade scripts',
@@ -649,6 +795,31 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Gets information about the available languages.
+   * @returns A dictionary mapping language name to the directory it comes from
+   */
+  async resolveLanguages(): Promise<LanguagesInfo> {
+    return await this.runJsonCodeQlCliCommand<LanguagesInfo>(['resolve', 'languages'], [], 'Resolving languages');
+  }
+
+  /**
+   * Gets the list of available languages. Refines the result of `resolveLanguages()`, by excluding
+   * extra things like "xml" and "properties".
+   *
+   * @returns An array of languages that are supported by the current version of the CodeQL CLI.
+   */
+  public async getSupportedLanguages(): Promise<string[]> {
+    if (!this._supportedLanguages) {
+      // Get the intersection of resolveLanguages with the list of hardcoded languages in dbSchemeToLanguage.
+      const resolvedLanguages = Object.keys(await this.resolveLanguages());
+      const hardcodedLanguages = Object.values(dbSchemeToLanguage);
+
+      this._supportedLanguages = resolvedLanguages.filter(lang => hardcodedLanguages.includes(lang));
+    }
+    return this._supportedLanguages;
+  }
+
   /**
    * Gets information about queries in a query suite.
    * @param suite The suite to resolve.
@@ -657,11 +828,15 @@ export class CodeQLCliServer implements Disposable {
    *   the default CLI search path is used.
    * @returns A list of query files found.
    */
-  resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
+  async resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
     const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
     if (searchPath !== undefined) {
       args.push('--search-path', path.join(...searchPath));
     }
+    if (await this.cliConstraints.supportsAllowLibraryPacksInResolveQueries()) {
+      // All of our usage of `codeql resolve queries` needs to handle library packs.
+      args.push('--allow-library-packs');
+    }
     args.push(suite);
     return this.runJsonCodeQlCliCommand<string[]>(
       ['resolve', 'queries'],
@@ -670,13 +845,71 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  async packInstall(dir: string) {
+    return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
+  }
+
+  async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
+    const args = [
+      '-o',
+      outputPath,
+      dir,
+      '--additional-packs',
+      workspaceFolders.join(path.delimiter)
+    ];
+    if (!precompile && await this.cliConstraints.supportsNoPrecompile()) {
+      args.push('--no-precompile');
+    }
+
+    return this.runJsonCodeQlCliCommand(['pack', 'bundle'], args, 'Bundling pack');
+  }
+
+  async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
+    const args = includeQueries ? [dir] : ['--no-include-queries', dir];
+    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
+    // previous versions return a list of packs.
+    const results: { paths: string[] } | string[] = await this.runJsonCodeQlCliCommand(['pack', 'packlist'], args, 'Generating the pack list');
+
+    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
+    if ('paths' in results) {
+      return results.paths;
+    } else {
+      return results;
+    }
+  }
+
   async generateDil(qloFile: string, outFile: string): Promise<void> {
+    const extraArgs = await this.cliConstraints.supportsDecompileDil()
+      ? ['--kind', 'dil', '-o', outFile, qloFile]
+      : ['-o', outFile, qloFile];
     await this.runCodeQlCliCommand(
       ['query', 'decompile'],
-      ['-o', outFile, qloFile],
+      extraArgs,
       'Generating DIL',
     );
   }
+
+  public async getVersion() {
+    if (!this._version) {
+      this._version = await this.refreshVersion();
+    }
+    return this._version;
+  }
+
+  private async refreshVersion() {
+    const distribution = await this.distributionProvider.getDistribution();
+    switch (distribution.kind) {
+      case FindDistributionResultKind.CompatibleDistribution:
+      // eslint-disable-next-line no-fallthrough
+      case FindDistributionResultKind.IncompatibleDistribution:
+        return distribution.version;
+
+      default:
+        // We should not get here because if no distributions are available, then
+        // the cli class is never instantiated.
+        throw new Error('No distribution found');
+    }
+  }
 }
 
 /**
@@ -712,7 +945,7 @@ export function spawnServer(
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Starting ${name}` });
   }
-  logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`);
+  void logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`);
   const child = child_process.spawn(base, args);
   if (!child || !child.pid) {
     throw new Error(`Failed to start ${name} using command ${base} ${argsString}.`);
@@ -728,13 +961,13 @@ export function spawnServer(
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Started ${name}` });
   }
-  logger.log(`${name} started on PID: ${child.pid}`);
+  void logger.log(`${name} started on PID: ${child.pid}`);
   return child;
 }
 
 /**
  * Runs a CodeQL CLI command without invoking the CLI server, returning the output as a string.
- * @param config The configuration containing the path to the CLI.
+ * @param codeQlPath The path to the CLI.
  * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
  * @param commandArgs The arguments to pass to the `codeql` command.
  * @param description Description of the action being run, to be shown in log and error messages.
@@ -742,7 +975,14 @@ export function spawnServer(
  * @param progressReporter Used to output progress messages, e.g. to the status bar.
  * @returns The contents of the command's stdout, if the command succeeded.
  */
-export async function runCodeQlCliCommand(codeQlPath: string, command: string[], commandArgs: string[], description: string, logger: Logger, progressReporter?: ProgressReporter): Promise<string> {
+export async function runCodeQlCliCommand(
+  codeQlPath: string,
+  command: string[],
+  commandArgs: string[],
+  description: string,
+  logger: Logger,
+  progressReporter?: ProgressReporter
+): Promise<string> {
   // Add logging arguments first, in case commandArgs contains positional parameters.
   const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
   const argsString = args.join(' ');
@@ -750,10 +990,10 @@ export async function runCodeQlCliCommand(codeQlPath: string, command: string[],
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
     }
-    logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
-    const result = await util.promisify(child_process.execFile)(codeQlPath, args);
-    logger.log(result.stderr);
-    logger.log('CLI command succeeded.');
+    void logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
+    const result = await promisify(child_process.execFile)(codeQlPath, args);
+    void logger.log(result.stderr);
+    void logger.log('CLI command succeeded.');
     return result.stdout;
   } catch (err) {
     throw new Error(`${description} failed: ${err.stderr || err}`);
@@ -789,6 +1029,20 @@ class SplitBuffer {
     this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
   }
 
+  /**
+   * A version of startsWith that isn't overriden by a broken version of ms-python.
+   *
+   * The definition comes from
+   * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
+   * which is CC0/public domain
+   *
+   * See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
+   */
+  private static startsWith(s: string, searchString: string, position: number): boolean {
+    const pos = position > 0 ? position | 0 : 0;
+    return s.substring(pos, pos + searchString.length) === searchString;
+  }
+
   /**
    * Extract the next full line from the buffer, if one is available.
    * @returns The text of the next available full line (without the separator), or `undefined` if no
@@ -797,7 +1051,7 @@ class SplitBuffer {
   public getNextLine(): string | undefined {
     while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
       for (const separator of this.separators) {
-        if (this.buffer.startsWith(separator, this.searchIndex)) {
+        if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
           const line = this.buffer.substr(0, this.searchIndex);
           this.buffer = this.buffer.substr(this.searchIndex + separator.length);
           this.searchIndex = 0;
@@ -854,6 +1108,138 @@ const lineEndings = ['\r\n', '\r', '\n'];
  */
 async function logStream(stream: Readable, logger: Logger): Promise<void> {
   for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
-    logger.log(line);
+    // Await the result of log here in order to ensure the logs are written in the correct order.
+    await logger.log(line);
+  }
+}
+
+
+export function shouldDebugIdeServer() {
+  return 'IDE_SERVER_JAVA_DEBUG' in process.env
+    && process.env.IDE_SERVER_JAVA_DEBUG !== '0'
+    && process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}
+
+export function shouldDebugQueryServer() {
+  return 'QUERY_SERVER_JAVA_DEBUG' in process.env
+    && process.env.QUERY_SERVER_JAVA_DEBUG !== '0'
+    && process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}
+
+export function shouldDebugCliServer() {
+  return 'CLI_SERVER_JAVA_DEBUG' in process.env
+    && process.env.CLI_SERVER_JAVA_DEBUG !== '0'
+    && process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}
+
+export class CliVersionConstraint {
+
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  public static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
+
+  /**
+   * CLI version where languages are exposed during a `codeql resolve database` command.
+   */
+  public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
+
+  /**
+   * CLI version where `codeql resolve upgrades` supports
+   * the `--allow-downgrades` flag
+   */
+  public static CLI_VERSION_WITH_DOWNGRADES = new SemVer('2.4.4');
+
+  /**
+   * CLI version where the `codeql resolve qlref` command is available.
+   */
+  public static CLI_VERSION_WITH_RESOLVE_QLREF = new SemVer('2.5.1');
+
+  /**
+   * CLI version where database registration was introduced
+  */
+  public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
+
+  /**
+   * CLI version where the `--allow-library-packs` option to `codeql resolve queries` was
+   * introduced.
+   */
+  public static CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES = new SemVer('2.6.1');
+
+  /**
+   * CLI version where the `database unbundle` subcommand was introduced.
+   */
+  public static CLI_VERSION_WITH_DATABASE_UNBUNDLE = new SemVer('2.6.0');
+
+  /**
+   * CLI version where the `--no-precompile` option for pack creation was introduced.
+   */
+  public static CLI_VERSION_WITH_NO_PRECOMPILE = new SemVer('2.7.1');
+
+  /**
+   * CLI version where remote queries are supported.
+   */
+  public static CLI_VERSION_REMOTE_QUERIES = new SemVer('2.6.3');
+
+  /**
+   * CLI version where the `resolve ml-models` subcommand was introduced.
+   */
+  public static CLI_VERSION_WITH_RESOLVE_ML_MODELS = new SemVer('2.7.3');
+
+  /**
+   * CLI version where the `--old-eval-stats` option to the query server was introduced.
+   */
+  public static CLI_VERSION_WITH_OLD_EVAL_STATS = new SemVer('2.7.4');
+
+  constructor(private readonly cli: CodeQLCliServer) {
+    /**/
+  }
+
+  private async isVersionAtLeast(v: SemVer) {
+    return (await this.cli.getVersion()).compare(v) >= 0;
+  }
+
+  public async supportsDecompileDil() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DECOMPILE_KIND_DIL);
+  }
+
+  public async supportsLanguageName() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
+  }
+
+  public async supportsDowngrades() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
+  }
+
+  public async supportsResolveQlref() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF);
+  }
+
+  public async supportsAllowLibraryPacksInResolveQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES);
+  }
+
+  async supportsDatabaseRegistration() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
+  }
+
+  async supportsDatabaseUnbundle() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
+  }
+
+  async supportsNoPrecompile() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NO_PRECOMPILE);
+  }
+
+  async supportsRemoteQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES);
+  }
+
+  async supportsResolveMlModels() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_ML_MODELS);
+  }
+
+  async supportsOldEvalStats() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_OLD_EVAL_STATS);
   }
 }

extensions/ql-vscode/src/commandRunner.ts

@@ -0,0 +1,241 @@
+import {
+  CancellationToken,
+  ProgressOptions,
+  window as Window,
+  commands,
+  Disposable,
+  ProgressLocation
+} from 'vscode';
+import { showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
+import { logger } from './logging';
+import { telemetryListener } from './telemetry';
+
+export class UserCancellationException extends Error {
+  /**
+   * @param message The error message
+   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
+   */
+  constructor(message?: string, public readonly silent = false) {
+    super(message);
+  }
+}
+
+export interface ProgressUpdate {
+  /**
+   * The current step
+   */
+  step: number;
+  /**
+   * The maximum step. This *should* be constant for a single job.
+   */
+  maxStep: number;
+  /**
+   * The current progress message
+   */
+  message: string;
+}
+
+export type ProgressCallback = (p: ProgressUpdate) => void;
+
+/**
+ * A task that handles command invocations from `commandRunner`
+ * and includes a progress monitor.
+ *
+ *
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `ProgressTask` instance.
+ *
+ * @param progress a progress handler function. Call this
+ * function with a `ProgressUpdate` instance in order to
+ * denote some progress being achieved on this task.
+ * @param token a cencellation token
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+export type ProgressTask<R> = (
+  progress: ProgressCallback,
+  token: CancellationToken,
+  ...args: any[]
+) => Thenable<R>;
+
+/**
+ * A task that handles command invocations from `commandRunner`.
+ * Arguments passed to the command handler are passed along,
+ * untouched to this `NoProgressTask` instance.
+ *
+ * @param args arguments passed to this task passed on from
+ * `commands.registerCommand`.
+ */
+type NoProgressTask = ((...args: any[]) => Promise<any>);
+
+/**
+ * This mediates between the kind of progress callbacks we want to
+ * write (where we *set* current progress position and give
+ * `maxSteps`) and the kind vscode progress api expects us to write
+ * (which increment progress by a certain amount out of 100%).
+ *
+ * Where possible, the `commandRunner` function below should be used
+ * instead of this function. The commandRunner is meant for wrapping
+ * top-level commands and provides error handling and other support
+ * automatically.
+ *
+ * Only use this function if you need a progress monitor and the
+ * control flow does not always come from a command (eg- during
+ * extension activation, or from an internal language server
+ * request).
+ */
+export function withProgress<R>(
+  options: ProgressOptions,
+  task: ProgressTask<R>,
+  ...args: any[]
+): Thenable<R> {
+  let progressAchieved = 0;
+  return Window.withProgress(options,
+    (progress, token) => {
+      return task(p => {
+        const { message, step, maxStep } = p;
+        const increment = 100 * (step - progressAchieved) / maxStep;
+        progressAchieved = step;
+        progress.report({ message, increment });
+      }, token, ...args);
+    });
+}
+
+/**
+ * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
+ *
+ * In this variant of the command runner, no progress monitor is used.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task.
+ */
+export function commandRunner(
+  commandId: string,
+  task: NoProgressTask,
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+
+    try {
+      return await task(...args);
+    } catch (e) {
+      error = e;
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          void logger.log(errorMessage);
+        } else {
+          void showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        void showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
+      }
+      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
+    }
+  });
+}
+
+/**
+ * A generic wrapper for command registration.  This wrapper adds uniform error handling,
+ * progress monitoring, and cancellation for commands.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task after the progress callback
+ * and cancellation token.
+ * @param progressOptions Progress options to be sent to the progress monitor.
+ */
+export function commandRunnerWithProgress<R>(
+  commandId: string,
+  task: ProgressTask<R>,
+  progressOptions: Partial<ProgressOptions>
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+    const progressOptionsWithDefaults = {
+      location: ProgressLocation.Notification,
+      ...progressOptions
+    };
+    try {
+      return await withProgress(progressOptionsWithDefaults, task, ...args);
+    } catch (e) {
+      error = e;
+      const errorMessage = `${e.message || e} (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          void logger.log(errorMessage);
+        } else {
+          void showAndLogWarningMessage(errorMessage);
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const fullMessage = e.stack
+          ? `${errorMessage}\n${e.stack}`
+          : errorMessage;
+        void showAndLogErrorMessage(errorMessage, {
+          fullMessage
+        });
+      }
+      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener.sendCommandUsage(commandId, executionTime, error);
+    }
+  });
+}
+
+/**
+ * Displays a progress monitor that indicates how much progess has been made
+ * reading from a stream.
+ *
+ * @param readable The stream to read progress from
+ * @param messagePrefix A prefix for displaying the message
+ * @param totalNumBytes Total number of bytes in this stream
+ * @param progress The progress callback used to set messages
+ */
+export function reportStreamProgress(
+  readable: NodeJS.ReadableStream,
+  messagePrefix: string,
+  totalNumBytes?: number,
+  progress?: ProgressCallback
+) {
+  if (progress && totalNumBytes) {
+    let numBytesDownloaded = 0;
+    const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+    const updateProgress = () => {
+      progress({
+        step: numBytesDownloaded,
+        maxStep: totalNumBytes,
+        message: `${messagePrefix} [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
+      });
+    };
+
+    // Display the progress straight away rather than waiting for the first chunk.
+    updateProgress();
+
+    readable.on('data', data => {
+      numBytesDownloaded += data.length;
+      updateProgress();
+    });
+  } else if (progress) {
+    progress({
+      step: 1,
+      maxStep: 2,
+      message: `${messagePrefix} (Size unknown)`,
+    });
+  }
+}

extensions/ql-vscode/src/compare/compare-interface.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from '../vscode-utils/disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import {
   WebviewPanel,
   ExtensionContext,
@@ -14,12 +14,12 @@ import {
   FromCompareViewMessage,
   ToCompareViewMessage,
   QueryCompareResult,
-} from '../interface-types';
+} from '../pure/interface-types';
 import { Logger } from '../logging';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
 import { getHtmlForWebview, jumpToLocation } from '../interface-utils';
-import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../bqrs-cli-types';
+import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../pure/bqrs-cli-types';
 import resultsDiff from './resultsDiff';
 
 interface ComparePair {
@@ -135,7 +135,7 @@ export class CompareInterfaceManager extends DisposableObject {
       );
 
       const stylesheetPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/resultsView.css')
+        ctx.asAbsolutePath('out/view/resultsView.css')
       );
 
       panel.webview.html = getHtmlForWebview(
@@ -173,7 +173,7 @@ export class CompareInterfaceManager extends DisposableObject {
         break;
 
       case 'changeCompare':
-        this.changeTable(msg.newResultSetName);
+        await this.changeTable(msg.newResultSetName);
         break;
 
       case 'viewSourceFile':
@@ -267,11 +267,11 @@ export class CompareInterfaceManager extends DisposableObject {
     return resultsDiff(fromResults, toResults);
   }
 
-  private openQuery(kind: 'from' | 'to') {
+  private async openQuery(kind: 'from' | 'to') {
     const toOpen =
       kind === 'from' ? this.comparePair?.from : this.comparePair?.to;
     if (toOpen) {
-      this.showQueryResultsCallback(toOpen);
+      await this.showQueryResultsCallback(toOpen);
     }
   }
 }

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -1,5 +1,5 @@
-import { RawResultSet } from '../bqrs-cli-types';
-import { QueryCompareResult } from '../interface-types';
+import { RawResultSet } from '../pure/bqrs-cli-types';
+import { QueryCompareResult } from '../pure/interface-types';
 
 /**
  * Compare the rows of two queries. Use deep equality to determine if

extensions/ql-vscode/src/compare/view/Compare.tsx

@@ -5,7 +5,7 @@ import * as Rdom from 'react-dom';
 import {
   ToCompareViewMessage,
   SetComparisonsMessage,
-} from '../../interface-types';
+} from '../../pure/interface-types';
 import CompareSelector from './CompareSelector';
 import { vscode } from '../../view/vscode-api';
 import CompareTable from './CompareTable';
@@ -21,7 +21,7 @@ const emptyComparison: SetComparisonsMessage = {
   message: 'Empty comparison'
 };
 
-export function Compare(_: {}): JSX.Element {
+export function Compare(_: Record<string, never>): JSX.Element {
   const [comparison, setComparison] = useState<SetComparisonsMessage>(
     emptyComparison
   );
@@ -38,7 +38,9 @@ export function Compare(_: {}): JSX.Element {
             setComparison(msg);
         }
       } else {
-        console.error(`Invalid event origin ${evt.origin}`);
+        // sanitize origin
+        const origin = evt.origin.replace(/\n|\r/g, '');
+        console.error(`Invalid event origin ${origin}`);
       }
     });
   });
@@ -64,8 +66,8 @@ export function Compare(_: {}): JSX.Element {
         {hasRows ? (
           <CompareTable comparison={comparison}></CompareTable>
         ) : (
-            <div className="vscode-codeql__compare-message">{message}</div>
-          )}
+          <div className="vscode-codeql__compare-message">{message}</div>
+        )}
       </>
     );
   } catch (err) {

extensions/ql-vscode/src/compare/view/CompareTable.tsx

@@ -1,9 +1,9 @@
 import * as React from 'react';
 
-import { SetComparisonsMessage } from '../../interface-types';
+import { SetComparisonsMessage } from '../../pure/interface-types';
 import RawTableHeader from '../../view/RawTableHeader';
 import { className } from '../../view/result-table-utils';
-import { ResultRow } from '../../bqrs-cli-types';
+import { ResultRow } from '../../pure/bqrs-cli-types';
 import RawTableRow from '../../view/RawTableRow';
 import { vscode } from '../../view/vscode-api';
 

extensions/ql-vscode/src/config.ts

@@ -1,10 +1,10 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
 
 /** Helper class to look up a labelled (and possibly nested) setting. */
-class Setting {
+export class Setting {
   name: string;
   parent?: Setting;
 
@@ -39,10 +39,19 @@ class Setting {
 
 const ROOT_SETTING = new Setting('codeQL');
 
-// Distribution configuration
+// Global configuration
+const TELEMETRY_SETTING = new Setting('telemetry', ROOT_SETTING);
+const AST_VIEWER_SETTING = new Setting('astViewer', ROOT_SETTING);
+const GLOBAL_TELEMETRY_SETTING = new Setting('telemetry');
 
+export const LOG_TELEMETRY = new Setting('logTelemetry', TELEMETRY_SETTING);
+export const ENABLE_TELEMETRY = new Setting('enableTelemetry', TELEMETRY_SETTING);
+
+export const GLOBAL_ENABLE_TELEMETRY = new Setting('enableTelemetry', GLOBAL_TELEMETRY_SETTING);
+
+// Distribution configuration
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
-const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
+export const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
 const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
@@ -52,7 +61,8 @@ const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
 
 export interface DistributionConfig {
-  customCodeQlPath?: string;
+  readonly customCodeQlPath?: string;
+  updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
   ownerName?: string;
@@ -64,24 +74,34 @@ export interface DistributionConfig {
 
 const RUNNING_QUERIES_SETTING = new Setting('runningQueries', ROOT_SETTING);
 const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES_SETTING);
+const SAVE_CACHE_SETTING = new Setting('saveCache', RUNNING_QUERIES_SETTING);
+const CACHE_SIZE_SETTING = new Setting('cacheSize', RUNNING_QUERIES_SETTING);
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
+const MAX_PATHS = new Setting('maxPaths', RUNNING_QUERIES_SETTING);
 const RUNNING_TESTS_SETTING = new Setting('runningTests', ROOT_SETTING);
+const RESULTS_DISPLAY_SETTING = new Setting('resultsDisplay', ROOT_SETTING);
 
+export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting('additionalTestArguments', RUNNING_TESTS_SETTING);
 export const NUMBER_OF_TEST_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_TESTS_SETTING);
 export const MAX_QUERIES = new Setting('maxQueries', RUNNING_QUERIES_SETTING);
 export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
+export const PAGE_SIZE = new Setting('pageSize', RESULTS_DISPLAY_SETTING);
+const CUSTOM_LOG_DIRECTORY_SETTING = new Setting('customLogDirectory', RUNNING_QUERIES_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
-const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
+const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING, DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING];
 
 export interface QueryServerConfig {
   codeQlPath: string;
   debug: boolean;
   numThreads: number;
+  saveCache: boolean;
+  cacheSize: number;
   queryMemoryMb?: number;
   timeoutSecs: number;
+  customLogDirectory?: string;
   onDidChangeConfiguration?: Event<void>;
 }
 
@@ -93,15 +113,18 @@ export interface QueryHistoryConfig {
   onDidChangeConfiguration: Event<void>;
 }
 
-const CLI_SETTINGS = [NUMBER_OF_TEST_THREADS_SETTING];
+const CLI_SETTINGS = [ADDITIONAL_TEST_ARGUMENTS_SETTING, NUMBER_OF_TEST_THREADS_SETTING, NUMBER_OF_THREADS_SETTING, MAX_PATHS];
 
 export interface CliConfig {
+  additionalTestArguments: string[];
   numberTestThreads: number;
+  numberThreads: number;
+  maxPaths: number;
   onDidChangeConfiguration?: Event<void>;
 }
 
 
-abstract class ConfigListener extends DisposableObject {
+export abstract class ConfigListener extends DisposableObject {
   protected readonly _onDidChangeConfiguration = this.push(new EventEmitter<void>());
 
   constructor() {
@@ -126,7 +149,7 @@ abstract class ConfigListener extends DisposableObject {
 
   protected abstract handleDidChangeConfiguration(e: ConfigurationChangeEvent): void;
   private updateConfiguration(): void {
-    this._onDidChangeConfiguration.fire();
+    this._onDidChangeConfiguration.fire(undefined);
   }
 
   public get onDidChangeConfiguration(): Event<void> {
@@ -147,6 +170,10 @@ export class DistributionConfigListener extends ConfigListener implements Distri
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
+  public async updateCustomCodeQlPath(newPath: string | undefined) {
+    await CUSTOM_CODEQL_PATH_SETTING.updateValue(newPath, ConfigurationTarget.Global);
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(DISTRIBUTION_CHANGE_SETTINGS, e);
   }
@@ -164,7 +191,7 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
       config.push(distributionManager.onDidChangeDistribution(async () => {
         const codeQlPath = await distributionManager.getCodeQlPathWithoutVersionCheck();
         config._codeQlPath = codeQlPath!;
-        config._onDidChangeConfiguration.fire();
+        config._onDidChangeConfiguration.fire(undefined);
       }));
     }
     return config;
@@ -174,10 +201,22 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     return this._codeQlPath;
   }
 
+  public get customLogDirectory(): string | undefined {
+    return CUSTOM_LOG_DIRECTORY_SETTING.getValue<string>() || undefined;
+  }
+
   public get numThreads(): number {
     return NUMBER_OF_THREADS_SETTING.getValue<number>();
   }
 
+  public get saveCache(): boolean {
+    return SAVE_CACHE_SETTING.getValue<boolean>();
+  }
+
+  public get cacheSize(): number {
+    return CACHE_SIZE_SETTING.getValue<number | null>() || 0;
+  }
+
   /** Gets the configured query timeout, in seconds. This looks up the setting at the time of access. */
   public get timeoutSecs(): number {
     return TIMEOUT_SETTING.getValue<number | null>() || 0;
@@ -189,7 +228,7 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
       return undefined;
     }
     if (memory == 0 || typeof (memory) !== 'number') {
-      logger.log(`Ignoring value '${memory}' for setting ${MEMORY_SETTING.qualifiedName}`);
+      void logger.log(`Ignoring value '${memory}' for setting ${MEMORY_SETTING.qualifiedName}`);
       return undefined;
     }
     return memory;
@@ -215,16 +254,37 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
 }
 
 export class CliConfigListener extends ConfigListener implements CliConfig {
+  public get additionalTestArguments(): string[] {
+    return ADDITIONAL_TEST_ARGUMENTS_SETTING.getValue();
+  }
 
   public get numberTestThreads(): number {
     return NUMBER_OF_TEST_THREADS_SETTING.getValue();
   }
 
+  public get numberThreads(): number {
+    return NUMBER_OF_THREADS_SETTING.getValue<number>();
+  }
+
+  public get maxPaths(): number {
+    return MAX_PATHS.getValue<number>();
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
   }
 }
 
+/**
+ * Whether to enable CodeLens for the 'Quick Evaluation' command.
+ */
+const QUICK_EVAL_CODELENS_SETTING = new Setting('quickEvalCodelens', RUNNING_QUERIES_SETTING);
+
+export function isQuickEvalCodelensEnabled() {
+  return QUICK_EVAL_CODELENS_SETTING.getValue<boolean>();
+}
+
+
 // Enable experimental features
 
 /**
@@ -233,3 +293,65 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
  * want to enable experimental features, they can add them directly in
  * their vscode settings json file.
  */
+
+/**
+ * Enables canary features of this extension. Recommended for all internal users.
+ */
+export const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);
+
+export function isCanary() {
+  return !!CANARY_FEATURES.getValue<boolean>();
+}
+
+/**
+ * Avoids caching in the AST viewer if the user is also a canary user.
+ */
+export const NO_CACHE_AST_VIEWER = new Setting('disableCache', AST_VIEWER_SETTING);
+
+// Settings for remote queries
+const REMOTE_QUERIES_SETTING = new Setting('remoteQueries', ROOT_SETTING);
+
+/**
+ * Lists of GitHub repositories that you want to query remotely via the "Run Remote query" command.
+ * Note: This command is only available for internal users.
+ *
+ * This setting should be a JSON object where each key is a user-specified name (string),
+ * and the value is an array of GitHub repositories (of the form `<owner>/<repo>`).
+ */
+const REMOTE_REPO_LISTS = new Setting('repositoryLists', REMOTE_QUERIES_SETTING);
+
+export function getRemoteRepositoryLists(): Record<string, string[]> | undefined {
+  return REMOTE_REPO_LISTS.getValue<Record<string, string[]>>() || undefined;
+}
+
+export async function setRemoteRepositoryLists(lists: Record<string, string[]> | undefined) {
+  await REMOTE_REPO_LISTS.updateValue(lists, ConfigurationTarget.Global);
+}
+
+/**
+ * The name of the "controller" repository that you want to use with the "Run Remote query" command.
+ * Note: This command is only available for internal users.
+ *
+ * This setting should be a GitHub repository of the form `<owner>/<repo>`.
+ */
+const REMOTE_CONTROLLER_REPO = new Setting('controllerRepo', REMOTE_QUERIES_SETTING);
+
+export function getRemoteControllerRepo(): string | undefined {
+  return REMOTE_CONTROLLER_REPO.getValue<string>() || undefined;
+}
+
+export async function setRemoteControllerRepo(repo: string | undefined) {
+  await REMOTE_CONTROLLER_REPO.updateValue(repo, ConfigurationTarget.Global);
+}
+
+/**
+ * Whether to insecurely load ML models from CodeQL packs.
+ *
+ * This setting is for internal users only.
+ */
+const SHOULD_INSECURELY_LOAD_MODELS_FROM_PACKS =
+  new Setting('shouldInsecurelyLoadModelsFromPacks', RUNNING_QUERIES_SETTING);
+
+export function shouldInsecurelyLoadMlModelsFromPacks(): boolean {
+  return SHOULD_INSECURELY_LOAD_MODELS_FROM_PACKS.getValue<boolean>();
+}

extensions/ql-vscode/src/contextual/astBuilder.ts

@@ -1,6 +1,6 @@
 import { QueryWithResults } from '../run-queries';
 import { CodeQLCliServer } from '../cli';
-import { DecodedBqrsChunk, BqrsId, EntityValue } from '../bqrs-cli-types';
+import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
 import { DatabaseItem } from '../databases';
 import { ChildAstItem, AstItem } from '../astViewer';
 import fileRangeFromURI from './fileRangeFromURI';
@@ -45,17 +45,18 @@ export default class AstBuilder {
     const parentToChildren = new Map<BqrsId, BqrsId[]>();
     const childToParent = new Map<BqrsId, BqrsId>();
     const astOrder = new Map<BqrsId, number>();
+    const edgeLabels = new Map<BqrsId, string>();
     const roots = [];
 
     // Build up the parent-child relationships
     edgeTuples.tuples.forEach(tuple => {
-      const [source, target, tupleType, orderValue] = tuple as [EntityValue, EntityValue, string, string];
+      const [source, target, tupleType, value] = tuple as [EntityValue, EntityValue, string, string];
       const sourceId = source.id!;
       const targetId = target.id!;
 
       switch (tupleType) {
         case 'semmle.order':
-          astOrder.set(targetId, Number(orderValue));
+          astOrder.set(targetId, Number(value));
           break;
 
         case 'semmle.label': {
@@ -65,6 +66,11 @@ export default class AstBuilder {
             parentToChildren.set(sourceId, children = []);
           }
           children.push(targetId);
+
+          // ignore values that indicate a numeric order.
+          if (!Number.isFinite(Number(value))) {
+            edgeLabels.set(targetId, value);
+          }
           break;
         }
 
@@ -75,18 +81,22 @@ export default class AstBuilder {
 
     // populate parents and children
     nodeTuples.tuples.forEach(tuple => {
-      const [entity, tupleType, orderValue] = tuple as [EntityValue, string, string];
+      const [entity, tupleType, value] = tuple as [EntityValue, string, string];
       const id = entity.id!;
 
       switch (tupleType) {
         case 'semmle.order':
-          astOrder.set(id, Number(orderValue));
+          astOrder.set(id, Number(value));
           break;
 
         case 'semmle.label': {
+          // If an edge label exists, include it and separate from the node label using ':'
+          const nodeLabel = value ?? entity.label;
+          const edgeLabel = edgeLabels.get(id);
+          const label = [edgeLabel, nodeLabel].filter(e => e).join(': ');
           const item = {
             id,
-            label: entity.label,
+            label,
             location: entity.url,
             fileLocation: fileRangeFromURI(entity.url, this.db),
             children: [] as ChildAstItem[],

extensions/ql-vscode/src/contextual/fileRangeFromURI.ts

@@ -1,7 +1,7 @@
 import * as vscode from 'vscode';
 
-import { UrlValue, LineColumnLocation } from '../bqrs-cli-types';
-import { isEmptyPath } from '../bqrs-utils';
+import { UrlValue, LineColumnLocation } from '../pure/bqrs-cli-types';
+import { isEmptyPath } from '../pure/bqrs-utils';
 import { DatabaseItem } from '../databases';
 
 
@@ -20,9 +20,8 @@ export default function fileRangeFromURI(uri: UrlValue | undefined, db: Database
       Math.max(0, (loc.endLine || 0) - 1),
       Math.max(0, (loc.endColumn || 0)));
     try {
-      const parsed = vscode.Uri.parse(uri.uri, true);
-      if (parsed.scheme === 'file') {
-        return new vscode.Location(db.resolveSourceFile(parsed.fsPath), range);
+      if (uri.uri.startsWith('file:')) {
+        return new vscode.Location(db.resolveSourceFile(uri.uri), range);
       }
       return undefined;
     } catch (e) {

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -1,18 +1,18 @@
 import * as vscode from 'vscode';
 
 import { decodeSourceArchiveUri, encodeArchiveBasePath } from '../archive-filesystem-provider';
-import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from '../bqrs-cli-types';
+import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from '../pure/bqrs-cli-types';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager, DatabaseItem } from '../databases';
 import fileRangeFromURI from './fileRangeFromURI';
-import * as messages from '../messages';
+import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { QueryWithResults, compileAndRunQueryAgainstDatabase } from '../run-queries';
-import { ProgressCallback } from '../helpers';
+import { ProgressCallback } from '../commandRunner';
 import { KeyType } from './keyType';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
 
-const SELECT_QUERY_NAME = '#select';
+export const SELECT_QUERY_NAME = '#select';
 export const TEMPLATE_NAME = 'selectedSourceFile';
 
 export interface FullLocationLink extends vscode.LocationLink {
@@ -43,7 +43,7 @@ export async function getLocationsForUriString(
   token: vscode.CancellationToken,
   filter: (src: string, dest: string) => boolean
 ): Promise<FullLocationLink[]> {
-  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString));
+  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString, true));
   const sourceArchiveUri = encodeArchiveBasePath(uri.sourceArchiveZipPath);
 
   const db = dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
@@ -52,9 +52,6 @@ export async function getLocationsForUriString(
   }
 
   const qlpack = await qlpackOfDatabase(cli, db);
-  if (qlpack === undefined) {
-    throw new Error('Can\'t infer qlpack from database source archive');
-  }
   const templates = createTemplates(uri.pathWithinSourceArchive);
 
   const links: FullLocationLink[] = [];

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -11,37 +11,96 @@ import {
 } from './keyType';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseItem } from '../databases';
+import { QlPacksForLanguage } from '../helpers';
 
-export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string | undefined> {
-  if (db.contents === undefined)
-    return undefined;
+export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<QlPacksForLanguage> {
+  if (db.contents === undefined) {
+    throw new Error('Database is invalid and cannot infer QLPack.');
+  }
   const datasetPath = db.contents.datasetUri.fsPath;
-  const { qlpack } = await helpers.resolveDatasetFolder(cli, datasetPath);
-  return qlpack;
+  const dbscheme = await helpers.getPrimaryDbscheme(datasetPath);
+  return await helpers.getQlPackForDbscheme(cli, dbscheme);
 }
 
-
-export async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
+/**
+ * Finds the contextual queries with the specified key in a list of CodeQL packs.
+ *
+ * @param cli The CLI instance to use.
+ * @param qlpacks The list of packs to search.
+ * @param keyType The contextual query key of the query to search for.
+ * @returns The found queries from the first pack in which any matching queries were found.
+ */
+async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[], keyType: KeyType): Promise<string[]> {
   const suiteFile = (await tmp.file({
     postfix: '.qls'
   })).path;
-  const suiteYaml = {
-    qlpack,
-    include: {
-      kind: kindOfKeyType(keyType),
-      'tags contain': tagOfKeyType(keyType)
-    }
-  };
+  const suiteYaml = [];
+  for (const qlpack of qlpacks) {
+    suiteYaml.push({
+      from: qlpack,
+      queries: '.',
+      include: {
+        kind: kindOfKeyType(keyType),
+        'tags contain': tagOfKeyType(keyType)
+      }
+    });
+  }
   await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
 
   const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
-  if (queries.length === 0) {
-    helpers.showAndLogErrorMessage(
-      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
-      Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
-      for this language.`
-    );
-    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
-  }
   return queries;
 }
+
+export async function resolveQueries(cli: CodeQLCliServer, qlpacks: QlPacksForLanguage, keyType: KeyType): Promise<string[]> {
+  const cliCanHandleLibraryPack = await cli.cliConstraints.supportsAllowLibraryPacksInResolveQueries();
+  const packsToSearch: string[] = [];
+  let blameCli: boolean;
+
+  if (cliCanHandleLibraryPack) {
+    // The CLI can handle both library packs and query packs, so search both packs in order.
+    packsToSearch.push(qlpacks.dbschemePack);
+    if (qlpacks.queryPack !== undefined) {
+      packsToSearch.push(qlpacks.queryPack);
+    }
+    // If we don't find the query, it's because it's not there, not because the CLI was unable to
+    // search the pack.
+    blameCli = false;
+  } else {
+    // Older CLIs can't handle `codeql resolve queries` with a suite that references a library pack.
+    if (qlpacks.dbschemePackIsLibraryPack) {
+      if (qlpacks.queryPack !== undefined) {
+        // Just search the query pack, because some older library/query releases still had the
+        // contextual queries in the query pack.
+        packsToSearch.push(qlpacks.queryPack);
+      }
+      // If we don't find it, it's because the CLI was unable to search the library pack that
+      // actually contains the query. Blame any failure on the CLI, not the packs.
+      blameCli = true;
+    } else {
+      // We have an old CLI, but the dbscheme pack is old enough that it's still a unified pack with
+      // both libraries and queries. Just search that pack.
+      packsToSearch.push(qlpacks.dbschemePack);
+      // Any CLI should be able to search the single query pack, so if we don't find it, it's
+      // because the language doesn't support it.
+      blameCli = false;
+    }
+  }
+
+  const queries = await resolveQueriesFromPacks(cli, packsToSearch, keyType);
+  if (queries.length > 0) {
+    return queries;
+  }
+
+  // No queries found. Determine the correct error message for the various scenarios.
+  const errorMessage = blameCli ?
+    `Your current version of the CodeQL CLI, '${(await cli.getVersion()).version}', \
+    is unable to use contextual queries from recent versions of the standard CodeQL libraries. \
+    Please upgrade to the latest version of the CodeQL CLI.`
+    :
+    `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
+    Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
+    for this language.`;
+
+  void helpers.showAndLogErrorMessage(errorMessage);
+  throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} in any of the following packs: ${packsToSearch.join(', ')}.`);
+}

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -1,10 +1,22 @@
-import * as vscode from 'vscode';
+import {
+  CancellationToken,
+  DefinitionProvider,
+  Location,
+  LocationLink,
+  Position,
+  ProgressLocation,
+  ReferenceContext,
+  ReferenceProvider,
+  TextDocument,
+  Uri
+} from 'vscode';
 
 import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
-import { CachedOperation, ProgressCallback, withProgress } from '../helpers';
-import * as messages from '../messages';
+import { CachedOperation } from '../helpers';
+import { ProgressCallback, withProgress } from '../commandRunner';
+import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';
 import AstBuilder from './astBuilder';
@@ -13,6 +25,7 @@ import {
 } from './keyType';
 import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
 
 /**
  * Run templated CodeQL queries to find definitions and references in
@@ -21,20 +34,20 @@ import { qlpackOfDatabase, resolveQueries } from './queryResolver';
  * or from a selected identifier.
  */
 
-export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
-  private cache: CachedOperation<vscode.LocationLink[]>;
+export class TemplateQueryDefinitionProvider implements DefinitionProvider {
+  private cache: CachedOperation<LocationLink[]>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
   ) {
-    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+    this.cache = new CachedOperation<LocationLink[]>(this.getDefinitions.bind(this));
   }
 
-  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+  async provideDefinition(document: TextDocument, position: Position, _token: CancellationToken): Promise<LocationLink[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.LocationLink[] = [];
+    const locLinks: LocationLink[] = [];
     for (const link of fileLinks) {
       if (link.originSelectionRange!.contains(position)) {
         locLinks.push(link);
@@ -43,9 +56,9 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
     return locLinks;
   }
 
-  private async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+  private async getDefinitions(uriString: string): Promise<LocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding definitions'
     }, async (progress, token) => {
@@ -63,7 +76,7 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
   }
 }
 
-export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+export class TemplateQueryReferenceProvider implements ReferenceProvider {
   private cache: CachedOperation<FullLocationLink[]>;
 
   constructor(
@@ -75,13 +88,13 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
   }
 
   async provideReferences(
-    document: vscode.TextDocument,
-    position: vscode.Position,
-    _context: vscode.ReferenceContext,
-    _token: vscode.CancellationToken
-  ): Promise<vscode.Location[]> {
+    document: TextDocument,
+    position: Position,
+    _context: ReferenceContext,
+    _token: CancellationToken
+  ): Promise<Location[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.Location[] = [];
+    const locLinks: Location[] = [];
     for (const link of fileLinks) {
       if (link.targetRange!.contains(position)) {
         locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
@@ -92,7 +105,7 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 
   private async getReferences(uriString: string): Promise<FullLocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding references'
     }, async (progress, token) => {
@@ -111,40 +124,47 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 }
 
 export class TemplatePrintAstProvider {
-  private cache: CachedOperation<QueryWithResults | undefined>;
+  private cache: CachedOperation<QueryWithResults>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
-
-    // Note: progress and token are only used if a cached value is not available
-    private progress: ProgressCallback,
-    private token: vscode.CancellationToken
   ) {
-    this.cache = new CachedOperation<QueryWithResults | undefined>(this.getAst.bind(this));
+    this.cache = new CachedOperation<QueryWithResults>(this.getAst.bind(this));
   }
 
-  async provideAst(document?: vscode.TextDocument): Promise<AstBuilder | undefined> {
+  async provideAst(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    document?: TextDocument
+  ): Promise<AstBuilder | undefined> {
     if (!document) {
-      return;
-    }
-    const queryResults = await this.cache.get(document.uri.toString());
-    if (!queryResults) {
-      return;
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
+    const queryResults = this.shouldCache()
+      ? await this.cache.get(document.uri.toString(), progress, token)
+      : await this.getAst(document.uri.toString(), progress, token);
 
     return new AstBuilder(
       queryResults, this.cli,
-      this.dbm.findDatabaseItem(vscode.Uri.parse(queryResults.database.databaseUri!))!,
+      this.dbm.findDatabaseItem(Uri.parse(queryResults.database.databaseUri!, true))!,
       document.fileName
     );
   }
 
-  private async getAst(uriString: string): Promise<QueryWithResults> {
-    const uri = vscode.Uri.parse(uriString, true);
+  private shouldCache() {
+    return !(isCanary() && NO_CACHE_AST_VIEWER.getValue<boolean>());
+  }
+
+  private async getAst(
+    uriString: string,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<QueryWithResults> {
+    const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
-      throw new Error('AST Viewing is only available for databases with zipped source archives.');
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
 
     const zippedArchive = decodeSourceArchiveUri(uri);
@@ -155,11 +175,8 @@ export class TemplatePrintAstProvider {
       throw new Error('Can\'t infer database from the provided source.');
     }
 
-    const qlpack = await qlpackOfDatabase(this.cli, db);
-    if (!qlpack) {
-      throw new Error('Can\'t infer qlpack from database source archive');
-    }
-    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintAstQuery);
+    const qlpacks = await qlpackOfDatabase(this.cli, db);
+    const queries = await resolveQueries(this.cli, qlpacks, KeyType.PrintAstQuery);
     if (queries.length > 1) {
       throw new Error('Found multiple Print AST queries. Can\'t continue');
     }
@@ -183,9 +200,9 @@ export class TemplatePrintAstProvider {
       this.qs,
       db,
       false,
-      vscode.Uri.file(query),
-      this.progress,
-      this.token,
+      Uri.file(query),
+      progress,
+      token,
       templates
     );
   }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -1,33 +1,39 @@
 import fetch, { Response } from 'node-fetch';
-import * as unzipper from 'unzipper';
 import { zip } from 'zip-a-folder';
+import * as unzipper from 'unzipper';
 import {
   Uri,
   CancellationToken,
   commands,
   window,
 } from 'vscode';
+import { CodeQLCliServer } from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
 import { DatabaseManager, DatabaseItem } from './databases';
 import {
-  ProgressCallback,
   showAndLogInformationMessage,
 } from './helpers';
+import {
+  reportStreamProgress,
+  ProgressCallback,
+} from './commandRunner';
 import { logger } from './logging';
+import { tmpDir } from './run-queries';
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
  *
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportInternetDatabase(
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  _: CancellationToken,
+  token: CancellationToken,
+  cli?: CodeQLCliServer
 ): Promise<DatabaseItem | undefined> {
   const databaseUrl = await window.showInputBox({
     prompt: 'Enter URL of zipfile of database to download',
@@ -40,14 +46,16 @@ export async function promptImportInternetDatabase(
 
   const item = await databaseArchiveFetcher(
     databaseUrl,
-    databasesManager,
+    databaseManager,
     storagePath,
-    progress
+    progress,
+    token,
+    cli
   );
 
   if (item) {
-    commands.executeCommand('codeQLDatabases.focus');
-    showAndLogInformationMessage('Database downloaded and imported successfully.');
+    await commands.executeCommand('codeQLDatabases.focus');
+    void showAndLogInformationMessage('Database downloaded and imported successfully.');
   }
   return item;
 
@@ -58,35 +66,43 @@ export async function promptImportInternetDatabase(
  * User enters a project url and then the user is asked which language
  * to download (if there is more than one)
  *
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportLgtmDatabase(
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  _: CancellationToken
+  token: CancellationToken,
+  cli?: CodeQLCliServer
 ): Promise<DatabaseItem | undefined> {
+  progress({
+    message: 'Choose project',
+    step: 1,
+    maxStep: 2
+  });
   const lgtmUrl = await window.showInputBox({
     prompt:
-      'Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)',
+      'Enter the project slug or URL on LGTM (e.g., g/github/codeql or https://lgtm.com/projects/g/github/codeql)',
   });
   if (!lgtmUrl) {
     return;
   }
 
   if (looksLikeLgtmUrl(lgtmUrl)) {
-    const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
+    const databaseUrl = await convertToDatabaseUrl(lgtmUrl, progress);
     if (databaseUrl) {
       const item = await databaseArchiveFetcher(
         databaseUrl,
-        databasesManager,
+        databaseManager,
         storagePath,
-        progress
+        progress,
+        token,
+        cli
       );
       if (item) {
-        commands.executeCommand('codeQLDatabases.focus');
-        showAndLogInformationMessage('Database downloaded and imported successfully.');
+        await commands.executeCommand('codeQLDatabases.focus');
+        void showAndLogInformationMessage('Database downloaded and imported successfully.');
       }
       return item;
     }
@@ -96,30 +112,43 @@ export async function promptImportLgtmDatabase(
   return;
 }
 
+export async function retrieveCanonicalRepoName(lgtmUrl: string) {
+  const givenRepoName = extractProjectSlug(lgtmUrl);
+  const response = await checkForFailingResponse(await fetch(`https://api.github.com/repos/${givenRepoName}`), 'Failed to locate the repository on github');
+  const repo = await response.json();
+  if (!repo || !repo.full_name) {
+    return;
+  }
+  return repo.full_name;
+}
+
 /**
  * Imports a database from a local archive.
  *
  * @param databaseUrl the file url of the archive to import
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
 export async function importArchiveDatabase(
   databaseUrl: string,
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  _: CancellationToken,
+  token: CancellationToken,
+  cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
   try {
     const item = await databaseArchiveFetcher(
       databaseUrl,
-      databasesManager,
+      databaseManager,
       storagePath,
-      progress
+      progress,
+      token,
+      cli
     );
     if (item) {
-      commands.executeCommand('codeQLDatabases.focus');
-      showAndLogInformationMessage('Database unzipped and imported successfully.');
+      await commands.executeCommand('codeQLDatabases.focus');
+      void showAndLogInformationMessage('Database unzipped and imported successfully.');
     }
     return item;
   } catch (e) {
@@ -137,17 +166,20 @@ export async function importArchiveDatabase(
  * or in the local filesystem.
  *
  * @param databaseUrl URL from which to grab the database
- * @param databasesManager the DatabaseManager
+ * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
- * @param progressCallback optional callback to send progress messages to
+ * @param progress callback to send progress messages to
+ * @param token cancellation token
  */
 async function databaseArchiveFetcher(
   databaseUrl: string,
-  databasesManager: DatabaseManager,
+  databaseManager: DatabaseManager,
   storagePath: string,
-  progressCallback?: ProgressCallback
+  progress: ProgressCallback,
+  token: CancellationToken,
+  cli?: CodeQLCliServer,
 ): Promise<DatabaseItem> {
-  progressCallback?.({
+  progress({
     message: 'Getting database',
     step: 1,
     maxStep: 4,
@@ -159,12 +191,12 @@ async function databaseArchiveFetcher(
   const unzipPath = await getStorageFolder(storagePath, databaseUrl);
 
   if (isFile(databaseUrl)) {
-    await readAndUnzip(databaseUrl, unzipPath);
+    await readAndUnzip(databaseUrl, unzipPath, cli, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, progressCallback);
+    await fetchAndUnzip(databaseUrl, unzipPath, cli, progress);
   }
 
-  progressCallback?.({
+  progress({
     message: 'Opening database',
     step: 3,
     maxStep: 4,
@@ -177,15 +209,15 @@ async function databaseArchiveFetcher(
     'codeql-database.yml'
   );
   if (dbPath) {
-    progressCallback?.({
+    progress({
       message: 'Validating and fixing source location',
       step: 4,
       maxStep: 4,
     });
     await ensureZippedSourceLocation(dbPath);
 
-    const item = await databasesManager.openDatabase(Uri.file(dbPath));
-    databasesManager.setCurrentDatabaseItem(item);
+    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath));
+    await databaseManager.setCurrentDatabaseItem(item);
     return item;
   } else {
     throw new Error('Database not found in archive.');
@@ -232,48 +264,75 @@ function validateHttpsUrl(databaseUrl: string) {
   }
 }
 
-async function readAndUnzip(databaseUrl: string, unzipPath: string) {
-  const databaseFile = Uri.parse(databaseUrl).fsPath;
-  const directory = await unzipper.Open.file(databaseFile);
-  await directory.extract({ path: unzipPath });
+async function readAndUnzip(
+  zipUrl: string,
+  unzipPath: string,
+  cli?: CodeQLCliServer,
+  progress?: ProgressCallback
+) {
+  // TODO: Providing progress as the file is unzipped is currently blocked
+  // on https://github.com/ZJONSSON/node-unzipper/issues/222
+  const zipFile = Uri.parse(zipUrl).fsPath;
+  progress?.({
+    maxStep: 10,
+    step: 9,
+    message: `Unzipping into ${path.basename(unzipPath)}`
+  });
+  if (cli && await cli.cliConstraints.supportsDatabaseUnbundle()) {
+    // Use the `database unbundle` command if the installed cli version supports it
+    await cli.databaseUnbundle(zipFile, unzipPath);
+  } else {
+    // Must get the zip central directory since streaming the
+    // zip contents may not have correct local file headers.
+    // Instead, we can only rely on the central directory.
+    const directory = await unzipper.Open.file(zipFile);
+    await directory.extract({ path: unzipPath });
+  }
 }
 
 async function fetchAndUnzip(
   databaseUrl: string,
   unzipPath: string,
-  progressCallback?: ProgressCallback
+  cli?: CodeQLCliServer,
+  progress?: ProgressCallback
 ) {
-  const response = await fetch(databaseUrl);
+  // Although it is possible to download and stream directly to an unzipped directory,
+  // we need to avoid this for two reasons. The central directory is located at the
+  // end of the zip file. It is the source of truth of the content locations. Individual
+  // file headers may be incorrect. Additionally, saving to file first will reduce memory
+  // pressure compared with unzipping while downloading the archive.
 
-  await checkForFailingResponse(response);
+  const archivePath = path.join(tmpDir.name, `archive-${Date.now()}.zip`);
 
-  const unzipStream = unzipper.Extract({
-    path: unzipPath,
-  });
-
-  progressCallback?.({
+  progress?.({
     maxStep: 3,
-    message: 'Unzipping database',
-    step: 2,
-  });
-  await new Promise((resolve, reject) => {
-    const handler = (err: Error) => {
-      if (err.message.startsWith('invalid signature')) {
-        reject(new Error('Not a valid archive.'));
-      } else {
-        reject(err);
-      }
-    };
-    response.body.on('error', handler);
-    unzipStream.on('error', handler);
-    unzipStream.on('close', resolve);
-    response.body.pipe(unzipStream);
+    message: 'Downloading database',
+    step: 1,
   });
+
+  const response = await checkForFailingResponse(await fetch(databaseUrl), 'Error downloading database');
+  const archiveFileStream = fs.createWriteStream(archivePath);
+
+  const contentLength = response.headers.get('content-length');
+  const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
+  reportStreamProgress(response.body, 'Downloading database', totalNumBytes, progress);
+
+  await new Promise((resolve, reject) =>
+    response.body.pipe(archiveFileStream)
+      .on('finish', resolve)
+      .on('error', reject)
+  );
+
+  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, cli, progress);
+
+
+  // remove archivePath eagerly since these archives can be large.
+  await fs.remove(archivePath);
 }
 
-async function checkForFailingResponse(response: Response): Promise<void | never> {
+async function checkForFailingResponse(response: Response, errorMessage: string): Promise<Response | never> {
   if (response.ok) {
-    return;
+    return response;
   }
 
   // An error downloading the database. Attempt to extract the resaon behind it.
@@ -285,7 +344,7 @@ async function checkForFailingResponse(response: Response): Promise<void | never
   } catch (e) {
     msg = text;
   }
-  throw new Error(`Error downloading database.\n\nReason: ${msg}`);
+  throw new Error(`${errorMessage}.\n\nReason: ${msg}`);
 }
 
 function isFile(databaseUrl: string) {
@@ -324,13 +383,14 @@ export async function findDirWithFile(
 
 /**
  * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
- * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * There are several possibilities for the provider: in addition to GitHub.com (g),
  * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
  *
- * After the {provider}/{org}/{name} path components, there may be the components
- * related to sub pages.
+ * This function accepts any url that matches the pattern above. It also accepts the
+ * raw project slug, e.g., `g/myorg/myproject`
  *
- * This function accepts any url that matches the patter above
+ * After the `{provider}/{org}/{name}` path components, there may be the components
+ * related to sub pages.
  *
  * @param lgtmUrl The URL to the lgtm project
  *
@@ -342,6 +402,10 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
     return false;
   }
 
+  if (convertRawLgtmSlug(lgtmUrl)) {
+    return true;
+  }
+
   try {
     const uri = Uri.parse(lgtmUrl, true);
     if (uri.scheme !== 'https') {
@@ -359,22 +423,51 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
   }
 }
 
+function convertRawLgtmSlug(maybeSlug: string): string | undefined {
+  if (!maybeSlug) {
+    return;
+  }
+  const segments = maybeSlug.split('/');
+  const providers = ['g', 'gl', 'b', 'git'];
+  if (segments.length === 3 && providers.includes(segments[0])) {
+    return `https://lgtm.com/projects/${maybeSlug}`;
+  }
+  return;
+}
+ 
+function extractProjectSlug(lgtmUrl: string): string | undefined {
+  // Only matches the '/g/' provider (github)
+  const re = new RegExp('https://lgtm.com/projects/g/(.*[^/])');
+  const match = lgtmUrl.match(re);
+  if (!match) {
+    return;
+  }
+  return match[1];
+}
+
 // exported for testing
-export async function convertToDatabaseUrl(lgtmUrl: string) {
+export async function convertToDatabaseUrl(
+  lgtmUrl: string,
+  progress: ProgressCallback) {
   try {
-    const uri = Uri.parse(lgtmUrl, true);
-    const paths = ['api', 'v1.0'].concat(
-      uri.path.split('/').filter((segment) => segment)
-    ).slice(0, 6);
-    const projectUrl = `https://lgtm.com/${paths.join('/')}`;
-    const projectResponse = await fetch(projectUrl);
-    const projectJson = await projectResponse.json();
+    lgtmUrl = convertRawLgtmSlug(lgtmUrl) || lgtmUrl;
+    let projectJson = await downloadLgtmProjectMetadata(lgtmUrl);
 
     if (projectJson.code === 404) {
-      throw new Error();
+      // fallback check for github repositories with same name but different case
+      // will fail for other providers
+      let canonicalName = await retrieveCanonicalRepoName(lgtmUrl);
+      if (!canonicalName) {
+        throw new Error(`Project was not found at ${lgtmUrl}.`);
+      }
+      canonicalName = convertRawLgtmSlug(`g/${canonicalName}`);
+      projectJson = await downloadLgtmProjectMetadata(canonicalName);
+      if (projectJson.code === 404) {
+        throw new Error('Failed to download project from LGTM.');
+      }
     }
 
-    const language = await promptForLanguage(projectJson);
+    const language = await promptForLanguage(projectJson, progress);
     if (!language) {
       return;
     }
@@ -386,14 +479,30 @@ export async function convertToDatabaseUrl(lgtmUrl: string) {
       language,
     ].join('/')}`;
   } catch (e) {
-    logger.log(`Error: ${e.message}`);
+    void logger.log(`Error: ${e.message}`);
     throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
   }
 }
 
+async function downloadLgtmProjectMetadata(lgtmUrl: string): Promise<any> {
+  const uri = Uri.parse(lgtmUrl, true);
+  const paths = ['api', 'v1.0'].concat(
+    uri.path.split('/').filter((segment) => segment)
+  ).slice(0, 6);
+  const projectUrl = `https://lgtm.com/${paths.join('/')}`;
+  const projectResponse = await fetch(projectUrl);
+  return projectResponse.json();
+}
+
 async function promptForLanguage(
-  projectJson: any
+  projectJson: any,
+  progress: ProgressCallback
 ): Promise<string | undefined> {
+  progress({
+    message: 'Choose language',
+    step: 2,
+    maxStep: 2
+  });
   if (!projectJson?.languages?.length) {
     return;
   }

extensions/ql-vscode/src/databases-ui.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import {
   Event,
   EventEmitter,
@@ -8,34 +8,36 @@ import {
   TreeItem,
   Uri,
   window,
-  env
+  env,
 } from 'vscode';
 import * as fs from 'fs-extra';
 
-import * as cli from './cli';
 import {
   DatabaseChangedEvent,
   DatabaseItem,
   DatabaseManager,
-  getUpgradesDirectories,
 } from './databases';
 import {
   commandRunner,
   commandRunnerWithProgress,
-  getOnDiskWorkspaceFolders,
   ProgressCallback,
+} from './commandRunner';
+import {
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder,
   showAndLogErrorMessage
 } from './helpers';
 import { logger } from './logging';
 import { clearCacheInDatabase } from './run-queries';
 import * as qsClient from './queryserver-client';
-import { upgradeDatabase } from './upgrades';
+import { upgradeDatabaseExplicit } from './upgrades';
 import {
   importArchiveDatabase,
   promptImportInternetDatabase,
   promptImportLgtmDatabase,
 } from './databaseFetcher';
-import { CancellationToken } from 'vscode-jsonrpc';
+import { CancellationToken } from 'vscode';
+import { asyncFilter } from './pure/helpers-pure';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -78,7 +80,7 @@ class DatabaseTreeDataProvider extends DisposableObject
   implements TreeDataProvider<DatabaseItem> {
   private _sortOrder = SortOrder.NameAsc;
 
-  private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
+  private readonly _onDidChangeTreeData = this.push(new EventEmitter<DatabaseItem | undefined>());
   private currentDatabaseItem: DatabaseItem | undefined;
 
   constructor(
@@ -106,7 +108,7 @@ class DatabaseTreeDataProvider extends DisposableObject
   }
 
   private handleDidChangeDatabaseItem = (event: DatabaseChangedEvent): void => {
-    // Note that events from the databse manager are instances of DatabaseChangedEvent
+    // Note that events from the database manager are instances of DatabaseChangedEvent
     // and events fired by the UI are instances of DatabaseItem
 
     // When event.item is undefined, then the entire tree is refreshed.
@@ -133,6 +135,7 @@ class DatabaseTreeDataProvider extends DisposableObject
         this.extensionPath,
         SELECTED_DATABASE_ICON
       );
+      item.contextValue = 'currentDatabase';
     } else if (element.error !== undefined) {
       item.iconPath = joinThemableIconPath(
         this.extensionPath,
@@ -140,6 +143,7 @@ class DatabaseTreeDataProvider extends DisposableObject
       );
     }
     item.tooltip = element.databaseUri.fsPath;
+    item.description = element.language;
     return item;
   }
 
@@ -176,7 +180,7 @@ class DatabaseTreeDataProvider extends DisposableObject
 
   public set sortOrder(newSortOrder: SortOrder) {
     this._sortOrder = newSortOrder;
-    this._onDidChangeTreeData.fire();
+    this._onDidChangeTreeData.fire(undefined);
   }
 }
 
@@ -212,7 +216,6 @@ export class DatabaseUI extends DisposableObject {
   private treeDataProvider: DatabaseTreeDataProvider;
 
   public constructor(
-    private cliserver: cli.CodeQLCliServer,
     private databaseManager: DatabaseManager,
     private readonly queryServer: qsClient.QueryServerClient | undefined,
     private readonly storagePath: string,
@@ -229,8 +232,10 @@ export class DatabaseUI extends DisposableObject {
         canSelectMany: true,
       })
     );
+  }
 
-    logger.log('Registering database panel commands.');
+  init() {
+    void logger.log('Registering database panel commands.');
     this.push(
       commandRunnerWithProgress(
         'codeQL.setCurrentDatabase',
@@ -313,9 +318,13 @@ export class DatabaseUI extends DisposableObject {
       )
     );
     this.push(
-      commandRunner(
+      commandRunnerWithProgress(
         'codeQLDatabases.removeDatabase',
-        this.handleRemoveDatabase
+        this.handleRemoveDatabase,
+        {
+          title: 'Removing database',
+          cancellable: false
+        }
       )
     );
     this.push(
@@ -340,6 +349,18 @@ export class DatabaseUI extends DisposableObject {
         this.handleOpenFolder
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.addDatabaseSource',
+        this.handleAddSource
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.removeOrphanedDatabases',
+        this.handleRemoveOrphanedDatabases
+      )
+    );
   }
 
   private handleMakeCurrentDatabase = async (
@@ -355,11 +376,67 @@ export class DatabaseUI extends DisposableObject {
     try {
       return await this.chooseAndSetDatabase(true, progress, token);
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
       return undefined;
     }
   };
 
+  handleRemoveOrphanedDatabases = async (): Promise<void> => {
+    void logger.log('Removing orphaned databases from workspace storage.');
+    let dbDirs = undefined;
+
+    if (
+      !(await fs.pathExists(this.storagePath)) ||
+      !(await fs.stat(this.storagePath)).isDirectory()
+    ) {
+      void logger.log('Missing or invalid storage directory. Not trying to remove orphaned databases.');
+      return;
+    }
+
+    dbDirs =
+      // read directory
+      (await fs.readdir(this.storagePath, { withFileTypes: true }))
+        // remove non-directories
+        .filter(dirent => dirent.isDirectory())
+        // get the full path
+        .map(dirent => path.join(this.storagePath, dirent.name))
+        // remove databases still in workspace
+        .filter(dbDir => {
+          const dbUri = Uri.file(dbDir);
+          return this.databaseManager.databaseItems.every(item => item.databaseUri.fsPath !== dbUri.fsPath);
+        });
+
+    // remove non-databases
+    dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
+
+    if (!dbDirs.length) {
+      void logger.log('No orphaned databases found.');
+      return;
+    }
+
+    // delete
+    const failures = [] as string[];
+    await Promise.all(
+      dbDirs.map(async dbDir => {
+        try {
+          void logger.log(`Deleting orphaned database '${dbDir}'.`);
+          await fs.remove(dbDir);
+        } catch (e) {
+          failures.push(`${path.basename(dbDir)}`);
+        }
+      })
+    );
+
+    if (failures.length) {
+      const dirname = path.dirname(failures[0]);
+      void showAndLogErrorMessage(
+        `Failed to delete unused databases (${failures.join(', ')
+        }).\nTo delete unused databases, please remove them manually from the storage folder ${dirname}.`
+      );
+    }
+  };
+
+
   handleChooseDatabaseArchive = async (
     progress: ProgressCallback,
     token: CancellationToken
@@ -367,7 +444,7 @@ export class DatabaseUI extends DisposableObject {
     try {
       return await this.chooseAndSetDatabase(false, progress, token);
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
       return undefined;
     }
   };
@@ -375,14 +452,13 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseInternet = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<
-    DatabaseItem | undefined
-  > => {
+  ): Promise<DatabaseItem | undefined> => {
     return await promptImportInternetDatabase(
       this.databaseManager,
       this.storagePath,
       progress,
-      token
+      token,
+      this.queryServer?.cliServer
     );
   };
 
@@ -394,7 +470,8 @@ export class DatabaseUI extends DisposableObject {
       this.databaseManager,
       this.storagePath,
       progress,
-      token
+      token,
+      this.queryServer?.cliServer
     );
   };
 
@@ -465,25 +542,10 @@ export class DatabaseUI extends DisposableObject {
     }
 
     // Search for upgrade scripts in any workspace folders available
-    const searchPath: string[] = getOnDiskWorkspaceFolders();
 
-    const upgradeInfo = await this.cliserver.resolveUpgrades(
-      databaseItem.contents.dbSchemeUri.fsPath,
-      searchPath
-    );
-
-    const { scripts, finalDbscheme } = upgradeInfo;
-
-    if (finalDbscheme === undefined) {
-      throw new Error('Could not determine target dbscheme to upgrade to.');
-    }
-    const targetDbSchemeUri = Uri.file(finalDbscheme);
-
-    await upgradeDatabase(
+    await upgradeDatabaseExplicit(
       this.queryServer,
       databaseItem,
-      targetDbSchemeUri,
-      getUpgradesDirectories(scripts),
       progress,
       token
     );
@@ -519,31 +581,33 @@ export class DatabaseUI extends DisposableObject {
           this.databaseManager,
           this.storagePath,
           progress,
-          token
+          token,
+          this.queryServer?.cliServer
         );
       } else {
-        await this.setCurrentDatabase(uri);
+        await this.setCurrentDatabase(progress, token, uri);
       }
     } catch (e) {
       // rethrow and let this be handled by default error handling.
       throw new Error(
-        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${
-        e.message
+        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${e.message
         }`
       );
     }
   };
 
   private handleRemoveDatabase = async (
+    progress: ProgressCallback,
+    token: CancellationToken,
     databaseItem: DatabaseItem,
     multiSelect: DatabaseItem[] | undefined
   ): Promise<void> => {
     if (multiSelect?.length) {
-      multiSelect.forEach((dbItem) =>
-        this.databaseManager.removeDatabaseItem(dbItem)
-      );
+      await Promise.all(multiSelect.map((dbItem) =>
+        this.databaseManager.removeDatabaseItem(progress, token, dbItem)
+      ));
     } else {
-      this.databaseManager.removeDatabaseItem(databaseItem);
+      await this.databaseManager.removeDatabaseItem(progress, token, databaseItem);
     }
   };
 
@@ -559,7 +623,7 @@ export class DatabaseUI extends DisposableObject {
     });
 
     if (newName) {
-      this.databaseManager.renameDatabaseItem(databaseItem, newName);
+      await this.databaseManager.renameDatabaseItem(databaseItem, newName);
     }
   };
 
@@ -576,6 +640,24 @@ export class DatabaseUI extends DisposableObject {
     }
   };
 
+  /**
+   * Adds the source folder of a CodeQL database to the workspace.
+   * When a database is first added in the "Databases" view, its source folder is added to the workspace.
+   * If the source folder is removed from the workspace for some reason, we want to be able to re-add it if need be.
+   */
+  private handleAddSource = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      for (const dbItem of multiSelect) {
+        await this.databaseManager.addDatabaseSourceArchiveFolder(dbItem);
+      }
+    } else {
+      await this.databaseManager.addDatabaseSourceArchiveFolder(databaseItem);
+    }
+  };
+
   /**
    * Return the current database directory. If we don't already have a
    * current database, ask the user for one, and return that, or
@@ -593,11 +675,13 @@ export class DatabaseUI extends DisposableObject {
   }
 
   private async setCurrentDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken,
     uri: Uri
   ): Promise<DatabaseItem | undefined> {
     let databaseItem = this.databaseManager.findDatabaseItem(uri);
     if (databaseItem === undefined) {
-      databaseItem = await this.databaseManager.openDatabase(uri);
+      databaseItem = await this.databaseManager.openDatabase(progress, token, uri);
     }
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
 
@@ -614,7 +698,6 @@ export class DatabaseUI extends DisposableObject {
     token: CancellationToken,
   ): Promise<DatabaseItem | undefined> {
     const uri = await chooseDatabaseDir(byFolder);
-
     if (!uri) {
       return undefined;
     }
@@ -622,7 +705,7 @@ export class DatabaseUI extends DisposableObject {
     if (byFolder) {
       const fixedUri = await this.fixDbUri(uri);
       // we are selecting a database folder
-      return await this.setCurrentDatabase(fixedUri);
+      return await this.setCurrentDatabase(progress, token, fixedUri);
     } else {
       // we are selecting a database archive. Must unzip into a workspace-controlled area
       // before importing.
@@ -631,7 +714,8 @@ export class DatabaseUI extends DisposableObject {
         this.databaseManager,
         this.storagePath,
         progress,
-        token
+        token,
+        this.queryServer?.cliServer
       );
     }
   }
@@ -643,7 +727,7 @@ export class DatabaseUI extends DisposableObject {
    * 2. If the selected URI is a directory matching db-*, choose the containing directory
    * 3. choose the current directory
    *
-   * @param uri a URI that is a datbase folder or inside it
+   * @param uri a URI that is a database folder or inside it
    *
    * @return the actual database folder found by using the heuristics above.
    */
@@ -653,7 +737,7 @@ export class DatabaseUI extends DisposableObject {
       dbPath = path.dirname(dbPath);
     }
 
-    if (isLikelyDbFolder(dbPath)) {
+    if (isLikelyDbLanguageFolder(dbPath)) {
       dbPath = path.dirname(dbPath);
     }
     return Uri.file(dbPath);
@@ -668,9 +752,3 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 }
-
-// TODO: Get the list of supported languages from a list that will be auto-updated.
-const dbRegeEx = /^db-(javascript|go|cpp|java|python|csharp)$/;
-function isLikelyDbFolder(dbPath: string) {
-  return path.basename(dbPath).match(dbRegeEx);
-}

extensions/ql-vscode/src/databases.ts

@@ -4,11 +4,21 @@ import * as path from 'path';
 import * as vscode from 'vscode';
 import * as cli from './cli';
 import { ExtensionContext } from 'vscode';
-import { showAndLogErrorMessage, showAndLogWarningMessage, showAndLogInformationMessage } from './helpers';
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  showAndLogInformationMessage,
+  isLikelyDatabaseRoot
+} from './helpers';
+import {
+  ProgressCallback,
+  withProgress
+} from './commandRunner';
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { QueryServerConfig } from './config';
+import { DisposableObject } from './pure/disposable-object';
 import { Logger, logger } from './logging';
+import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
+import { QueryServerClient } from './queryserver-client';
 
 /**
  * databases.ts
@@ -36,11 +46,13 @@ export interface DatabaseOptions {
   displayName?: string;
   ignoreSourceArchive?: boolean;
   dateAdded?: number | undefined;
+  language?: string;
 }
 
-interface FullDatabaseOptions extends DatabaseOptions {
+export interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
   dateAdded: number | undefined;
+  language: string | undefined;
 }
 
 interface PersistedDatabaseItem {
@@ -103,30 +115,31 @@ async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
 
   const dbAbsolutePath = path.join(parentDirectory, dbRelativePaths[0]);
   if (dbRelativePaths.length > 1) {
-    showAndLogWarningMessage(`Found multiple dataset directories in database, using '${dbAbsolutePath}'.`);
+    void showAndLogWarningMessage(`Found multiple dataset directories in database, using '${dbAbsolutePath}'.`);
   }
 
   return vscode.Uri.file(dbAbsolutePath);
 }
 
-async function findSourceArchive(
+// exported for testing
+export async function findSourceArchive(
   databasePath: string, silent = false
 ): Promise<vscode.Uri | undefined> {
-
   const relativePaths = ['src', 'output/src_archive'];
 
   for (const relativePath of relativePaths) {
     const basePath = path.join(databasePath, relativePath);
     const zipPath = basePath + '.zip';
 
-    if (await fs.pathExists(basePath)) {
-      return vscode.Uri.file(basePath);
-    } else if (await fs.pathExists(zipPath)) {
+    // Prefer using a zip archive over a directory.
+    if (await fs.pathExists(zipPath)) {
       return encodeArchiveBasePath(zipPath);
+    } else if (await fs.pathExists(basePath)) {
+      return vscode.Uri.file(basePath);
     }
   }
   if (!silent) {
-    showAndLogInformationMessage(
+    void showAndLogInformationMessage(
       `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`
     );
   }
@@ -149,7 +162,6 @@ async function resolveDatabase(
     datasetUri,
     sourceArchiveUri
   };
-
 }
 
 /** Gets the relative paths of all `.dbscheme` files in the given directory. */
@@ -193,6 +205,9 @@ export interface DatabaseItem {
   readonly databaseUri: vscode.Uri;
   /** The name of the database to be displayed in the UI */
   name: string;
+
+  /** The primary language of the database or empty string if unknown */
+  readonly language: string;
   /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
   readonly sourceArchive: vscode.Uri | undefined;
   /**
@@ -243,12 +258,27 @@ export interface DatabaseItem {
    * Returns the root uri of the virtual filesystem for this database's source archive,
    * as displayed in the filesystem explorer.
    */
-  getSourceArchiveExplorerUri(): vscode.Uri | undefined;
+  getSourceArchiveExplorerUri(): vscode.Uri;
 
   /**
    * Holds if `uri` belongs to this database's source archive.
    */
   belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
+
+  /**
+   * Whether the database may be affected by test execution for the given path.
+   */
+  isAffectedByTest(testPath: string): Promise<boolean>;
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  getPersistedState(): PersistedDatabaseItem;
+
+  /**
+   * Verifies that this database item has a zipped source folder. Returns an error message if it does not.
+   */
+  verifyZippedSources(): string | undefined;
 }
 
 export enum DatabaseEventKind {
@@ -341,28 +371,29 @@ export class DatabaseItemImpl implements DatabaseItem {
     }
   }
 
-  public resolveSourceFile(uri: string | undefined): vscode.Uri {
+  public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
-    // Sometimes, we are passed a path, sometimes a file URI.
-    // We need to convert this to a file path that is probably inside of a zip file.
-    const file = uri?.replace(/file:/, '');
+    const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
+    if (uri && uri.scheme !== 'file') {
+      throw new Error(`Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`);
+    }
     if (!sourceArchive) {
-      if (file) {
-        // Treat it as an absolute path.
-        return vscode.Uri.file(file);
+      if (uri) {
+        return uri;
       } else {
         return this.databaseUri;
       }
     }
 
-    if (file) {
-      const absoluteFilePath = file.replace(':', '_');
-      // Strip any leading slashes from the file path, and replace `:` with `_`.
-      const relativeFilePath = absoluteFilePath.replace(/^\/*/, '').replace(':', '_');
+    if (uri) {
+      const relativeFilePath = decodeURI(uri.path).replace(':', '_').replace(/^\/*/, '');
       if (sourceArchive.scheme === zipArchiveScheme) {
         const zipRef = decodeSourceArchiveUri(sourceArchive);
+        const pathWithinSourceArchive = zipRef.pathWithinSourceArchive === '/'
+          ? relativeFilePath
+          : zipRef.pathWithinSourceArchive + '/' + relativeFilePath;
         return encodeSourceArchiveUri({
-          pathWithinSourceArchive: zipRef.pathWithinSourceArchive + '/' + absoluteFilePath,
+          pathWithinSourceArchive,
           sourceArchiveZipPath: zipRef.sourceArchiveZipPath,
         });
 
@@ -396,10 +427,7 @@ export class DatabaseItemImpl implements DatabaseItem {
    * Holds if the database item refers to an exported snapshot
    */
   public async hasMetadataFile(): Promise<boolean> {
-    return (await Promise.all([
-      fs.pathExists(path.join(this.databaseUri.fsPath, '.dbinfo')),
-      fs.pathExists(path.join(this.databaseUri.fsPath, 'codeql-database.yml'))
-    ])).some(x => x);
+    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
   }
 
   /**
@@ -429,16 +457,33 @@ export class DatabaseItemImpl implements DatabaseItem {
     return dbInfo.datasetFolder;
   }
 
+  public get language() {
+    return this.options.language || '';
+  }
+
   /**
    * Returns the root uri of the virtual filesystem for this database's source archive.
    */
-  public getSourceArchiveExplorerUri(): vscode.Uri | undefined {
+  public getSourceArchiveExplorerUri(): vscode.Uri {
     const sourceArchive = this.sourceArchive;
-    if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith('.zip'))
-      return undefined;
+    if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith('.zip')) {
+      throw new Error(this.verifyZippedSources());
+    }
     return encodeArchiveBasePath(sourceArchive.fsPath);
   }
 
+  public verifyZippedSources(): string | undefined {
+    const sourceArchive = this.sourceArchive;
+    if (sourceArchive === undefined) {
+      return `${this.name} has no source archive.`;
+    }
+
+    if (!sourceArchive.fsPath.endsWith('.zip')) {
+      return `${this.name} has a source folder that is unzipped.`;
+    }
+    return;
+  }
+
   /**
    * Holds if `uri` belongs to this database's source archive.
    */
@@ -448,6 +493,27 @@ export class DatabaseItemImpl implements DatabaseItem {
     return uri.scheme === zipArchiveScheme &&
       decodeSourceArchiveUri(uri).sourceArchiveZipPath === this.sourceArchive.fsPath;
   }
+
+  public async isAffectedByTest(testPath: string): Promise<boolean> {
+    const databasePath = this.databaseUri.fsPath;
+    if (!databasePath.endsWith('.testproj')) {
+      return false;
+    }
+    try {
+      const stats = await fs.stat(testPath);
+      if (stats.isDirectory()) {
+        return !path.relative(testPath, databasePath).startsWith('..');
+      } else {
+        // database for /one/two/three/test.ql is at /one/two/three/three.testproj
+        const testdir = path.dirname(testPath);
+        const testdirbase = path.basename(testdir);
+        return databasePath == path.join(testdir, testdirbase + '.testproj');
+      }
+    } catch {
+      // No information available for test path - assume database is unaffected.
+      return false;
+    }
+  }
 }
 
 /**
@@ -458,7 +524,7 @@ export class DatabaseItemImpl implements DatabaseItem {
 function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | undefined> {
   return new Promise((res, _rej) => {
     const timeout = setTimeout(() => {
-      logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
+      void logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
       res(undefined);
       dispose();
     }, timeoutMs);
@@ -481,43 +547,65 @@ export class DatabaseManager extends DisposableObject {
   private readonly _onDidChangeCurrentDatabaseItem = this.push(new vscode.EventEmitter<DatabaseChangedEvent>());
   readonly onDidChangeCurrentDatabaseItem = this._onDidChangeCurrentDatabaseItem.event;
 
-  private readonly _databaseItems: DatabaseItemImpl[] = [];
+  private readonly _databaseItems: DatabaseItem[] = [];
   private _currentDatabaseItem: DatabaseItem | undefined = undefined;
 
   constructor(
-    private ctx: ExtensionContext,
-    public config: QueryServerConfig,
+    private readonly ctx: ExtensionContext,
+    private readonly qs: QueryServerClient,
+    private readonly cli: cli.CodeQLCliServer,
     public logger: Logger
   ) {
     super();
 
-    this.loadPersistedState();  // Let this run async.
+    qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
+
+    // Let this run async.
+    void this.loadPersistedState();
   }
 
   public async openDatabase(
-    uri: vscode.Uri, options?: DatabaseOptions
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    uri: vscode.Uri,
   ): Promise<DatabaseItem> {
-
     const contents = await resolveDatabaseContents(uri);
-    const realOptions = options || {};
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj';
     const fullOptions: FullDatabaseOptions = {
-      ignoreSourceArchive: (realOptions.ignoreSourceArchive !== undefined) ?
-        realOptions.ignoreSourceArchive : isQLTestDatabase,
-      displayName: realOptions.displayName,
-      dateAdded: realOptions.dateAdded || Date.now()
+      ignoreSourceArchive: isQLTestDatabase,
+      // displayName is only set if a user explicitly renames a database
+      displayName: undefined,
+      dateAdded: Date.now(),
+      language: await this.getPrimaryLanguage(uri.fsPath)
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => {
       this._onDidChangeDatabaseItem.fire(event);
     });
-    await this.addDatabaseItem(databaseItem);
+
+    await this.addDatabaseItem(progress, token, databaseItem);
     await this.addDatabaseSourceArchiveFolder(databaseItem);
 
     return databaseItem;
   }
 
-  private async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
+  private async reregisterDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken
+  ) {
+    let completed = 0;
+    await Promise.all(this._databaseItems.map(async (databaseItem) => {
+      await this.registerDatabase(progress, token, databaseItem);
+      completed++;
+      progress({
+        maxStep: this._databaseItems.length,
+        step: completed,
+        message: 'Re-registering databases'
+      });
+    }));
+  }
+
+  public async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
     // The folder may already be in workspace state from a previous
     // session. If not, add it.
     const index = this.getDatabaseWorkspaceFolderIndex(item);
@@ -533,36 +621,41 @@ export class DatabaseManager extends DisposableObject {
       // This is undesirable, as we might be adding and removing many
       // workspace folders as the user adds and removes databases.
       const end = (vscode.workspace.workspaceFolders || []).length;
+
+      const msg = item.verifyZippedSources();
+      if (msg) {
+        void logger.log(`Could not add source folder because ${msg}`);
+        return;
+      }
+
       const uri = item.getSourceArchiveExplorerUri();
-      if (uri === undefined) {
-        logger.log(`Couldn't obtain file explorer uri for ${item.name}`);
-      }
-      else {
-        logger.log(`Adding workspace folder for ${item.name} source archive at index ${end}`);
-        if ((vscode.workspace.workspaceFolders || []).length < 2) {
-          // Adding this workspace folder makes the workspace
-          // multi-root, which may surprise the user. Let them know
-          // we're doing this.
-          vscode.window.showInformationMessage(`Adding workspace folder for source archive of database ${item.name}.`);
-        }
-        vscode.workspace.updateWorkspaceFolders(end, 0, {
-          name: `[${item.name} source archive]`,
-          uri,
-        });
-        // vscode api documentation says we must to wait for this event
-        // between multiple `updateWorkspaceFolders` calls.
-        await eventFired(vscode.workspace.onDidChangeWorkspaceFolders);
+      void logger.log(`Adding workspace folder for ${item.name} source archive at index ${end}`);
+      if ((vscode.workspace.workspaceFolders || []).length < 2) {
+        // Adding this workspace folder makes the workspace
+        // multi-root, which may surprise the user. Let them know
+        // we're doing this.
+        void vscode.window.showInformationMessage(`Adding workspace folder for source archive of database ${item.name}.`);
       }
+      vscode.workspace.updateWorkspaceFolders(end, 0, {
+        name: `[${item.name} source archive]`,
+        uri,
+      });
+      // vscode api documentation says we must to wait for this event
+      // between multiple `updateWorkspaceFolders` calls.
+      await eventFired(vscode.workspace.onDidChangeWorkspaceFolders);
     }
   }
 
   private async createDatabaseItemFromPersistedState(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
     state: PersistedDatabaseItem
   ): Promise<DatabaseItem> {
 
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
     let dateAdded = undefined;
+    let language = undefined;
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -573,43 +666,69 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.dateAdded === 'number') {
         dateAdded = state.options.dateAdded;
       }
+      language = state.options.language;
     }
+
+    const dbBaseUri = vscode.Uri.parse(state.uri, true);
+    if (language === undefined) {
+      // we haven't been successful yet at getting the language. try again
+      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
+    }
+
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive,
       displayName,
-      dateAdded
+      dateAdded,
+      language
     };
-    const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri), undefined, fullOptions,
+    const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions,
       (event) => {
         this._onDidChangeDatabaseItem.fire(event);
       });
-    await this.addDatabaseItem(item);
 
+    await this.addDatabaseItem(progress, token, item);
     return item;
   }
 
   private async loadPersistedState(): Promise<void> {
-    const currentDatabaseUri = this.ctx.workspaceState.get<string>(CURRENT_DB);
-    const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(DB_LIST, []);
-
-    try {
-      for (const database of databases) {
-        const databaseItem = await this.createDatabaseItemFromPersistedState(database);
+    return withProgress({
+      location: vscode.ProgressLocation.Notification
+    },
+      async (progress, token) => {
+        const currentDatabaseUri = this.ctx.workspaceState.get<string>(CURRENT_DB);
+        const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(DB_LIST, []);
+        let step = 0;
+        progress({
+          maxStep: databases.length,
+          message: 'Loading persisted databases',
+          step
+        });
         try {
-          await databaseItem.refresh();
-          if (currentDatabaseUri === database.uri) {
-            this.setCurrentDatabaseItem(databaseItem, true);
+          for (const database of databases) {
+            progress({
+              maxStep: databases.length,
+              message: `Loading ${database.options?.displayName || 'databases'}`,
+              step: ++step
+            });
+
+            const databaseItem = await this.createDatabaseItemFromPersistedState(progress, token, database);
+            try {
+              await databaseItem.refresh();
+              await this.registerDatabase(progress, token, databaseItem);
+              if (currentDatabaseUri === database.uri) {
+                await this.setCurrentDatabaseItem(databaseItem, true);
+              }
+            }
+            catch (e) {
+              // When loading from persisted state, leave invalid databases in the list. They will be
+              // marked as invalid, and cannot be set as the current database.
+            }
           }
+        } catch (e) {
+          // database list had an unexpected type - nothing to be done?
+          void showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
         }
-        catch (e) {
-          // When loading from persisted state, leave invalid databases in the list. They will be
-          // marked as invalid, and cannot be set as the current database.
-        }
-      }
-    } catch (e) {
-      // database list had an unexpected type - nothing to be done?
-      showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
-    }
+      });
   }
 
   public get databaseItems(): readonly DatabaseItem[] {
@@ -620,8 +739,10 @@ export class DatabaseManager extends DisposableObject {
     return this._currentDatabaseItem;
   }
 
-  public async setCurrentDatabaseItem(item: DatabaseItem | undefined,
-    skipRefresh = false): Promise<void> {
+  public async setCurrentDatabaseItem(
+    item: DatabaseItem | undefined,
+    skipRefresh = false
+  ): Promise<void> {
 
     if (!skipRefresh && (item !== undefined)) {
       await item.refresh();  // Will throw on invalid database.
@@ -629,6 +750,9 @@ export class DatabaseManager extends DisposableObject {
     if (this._currentDatabaseItem !== item) {
       this._currentDatabaseItem = item;
       this.updatePersistedCurrentDatabaseItem();
+
+      await vscode.commands.executeCommand('setContext', 'codeQL.currentDatabaseItem', item?.name);
+
       this._onDidChangeCurrentDatabaseItem.fire({
         item,
         kind: DatabaseEventKind.Change
@@ -655,9 +779,20 @@ export class DatabaseManager extends DisposableObject {
     return this._databaseItems.find(item => item.sourceArchive && item.sourceArchive.toString(true) === uriString);
   }
 
-  private async addDatabaseItem(item: DatabaseItemImpl) {
+  private async addDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem
+  ) {
     this._databaseItems.push(item);
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
+
+    // Add this database item to the allow-list
+    // Database items reconstituted from persisted state
+    // will not have their contents yet.
+    if (item.contents?.datasetUri) {
+      await this.registerDatabase(progress, token, item);
+    }
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
       item: undefined,
@@ -667,35 +802,46 @@ export class DatabaseManager extends DisposableObject {
 
   public async renameDatabaseItem(item: DatabaseItem, newName: string) {
     item.name = newName;
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
     this._onDidChangeDatabaseItem.fire({
-      item,
+      // pass undefined so that the entire tree is rebuilt in order to re-sort
+      item: undefined,
       kind: DatabaseEventKind.Rename
     });
   }
 
-  public removeDatabaseItem(item: DatabaseItem) {
-    if (this._currentDatabaseItem == item)
+  public async removeDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem
+  ) {
+    if (this._currentDatabaseItem == item) {
       this._currentDatabaseItem = undefined;
+    }
     const index = this.databaseItems.findIndex(searchItem => searchItem === item);
     if (index >= 0) {
       this._databaseItems.splice(index, 1);
     }
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
 
     // Delete folder from workspace, if it is still there
-    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(folder => item.belongsToSourceArchiveExplorerUri(folder.uri));
-    if (index >= 0) {
-      logger.log(`Removing workspace folder at index ${folderIndex}`);
+    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
+      folder => item.belongsToSourceArchiveExplorerUri(folder.uri)
+    );
+    if (folderIndex >= 0) {
+      void logger.log(`Removing workspace folder at index ${folderIndex}`);
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
+    // Remove this database item from the allow-list
+    await this.deregisterDatabase(progress, token, item);
+
     // Delete folder from file system only if it is controlled by the extension
     if (this.isExtensionControlledLocation(item.databaseUri)) {
-      logger.log('Deleting database from filesystem.');
-      fs.remove(item.databaseUri.path).then(
-        () => logger.log(`Deleted '${item.databaseUri.path}'`),
-        e => logger.log(`Failed to delete '${item.databaseUri.path}'. Reason: ${e.message}`));
+      void logger.log('Deleting database from filesystem.');
+      fs.remove(item.databaseUri.fsPath).then(
+        () => void logger.log(`Deleted '${item.databaseUri.fsPath}'`),
+        e => void logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
     }
 
     // note that we use undefined as the item in order to reset the entire tree
@@ -705,18 +851,63 @@ export class DatabaseManager extends DisposableObject {
     });
   }
 
+  private async deregisterDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
+    }
+  }
+
+  private async registerDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
+    }
+  }
+
   private updatePersistedCurrentDatabaseItem(): void {
-    this.ctx.workspaceState.update(CURRENT_DB, this._currentDatabaseItem ?
+    void this.ctx.workspaceState.update(CURRENT_DB, this._currentDatabaseItem ?
       this._currentDatabaseItem.databaseUri.toString(true) : undefined);
   }
 
-  private updatePersistedDatabaseList(): void {
-    this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
+  private async updatePersistedDatabaseList(): Promise<void> {
+    await this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
   }
 
   private isExtensionControlledLocation(uri: vscode.Uri) {
     const storagePath = this.ctx.storagePath || this.ctx.globalStoragePath;
-    return uri.path.startsWith(storagePath);
+    // the uri.fsPath function on windows returns a lowercase drive letter,
+    // but storagePath will have an uppercase drive letter. Be sure to compare
+    // URIs to URIs only
+    if (storagePath) {
+      return uri.fsPath.startsWith(vscode.Uri.file(storagePath).fsPath);
+    }
+    return false;
+  }
+
+  private async getPrimaryLanguage(dbPath: string) {
+    if (!(await this.cli.cliConstraints.supportsLanguageName())) {
+      // return undefined so that we recalculate on restart until the cli is at a version that
+      // supports this feature. This recalculation is cheap since we avoid calling into the cli
+      // unless we know it can return the langauges property.
+      return undefined;
+    }
+    const dbInfo = await this.cli.resolveDatabase(dbPath);
+    return dbInfo.languages?.[0] || '';
   }
 }
 

extensions/ql-vscode/src/discovery.ts

@@ -1,5 +1,5 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { showAndLogErrorMessage } from './helpers';
+import { DisposableObject } from './pure/disposable-object';
+import { logger } from './logging';
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
@@ -59,23 +59,23 @@ export abstract class Discovery<T> extends DisposableObject {
         this.discoveryInProgress = false;
         this.update(results);
       }
-    });
+    })
 
-    discoveryPromise.catch(err => {
-      showAndLogErrorMessage(`${this.name} failed. Reason: ${err.message}`);
-    });
+      .catch(err => {
+        void logger.log(`${this.name} failed. Reason: ${err.message}`);
+      })
 
-    discoveryPromise.finally(() => {
-      if (this.retry) {
-        // Another refresh request came in while we were still running a previous discovery
-        // operation. Since the discovery results we just computed are now stale, we'll launch
-        // another discovery operation instead of updating.
-        // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-        // initial discovery operation failed.
-        this.retry = false;
-        this.launchDiscovery();
-      }
-    });
+      .finally(() => {
+        if (this.retry) {
+          // Another refresh request came in while we were still running a previous discovery
+          // operation. Since the discovery results we just computed are now stale, we'll launch
+          // another discovery operation instead of updating.
+          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+          // initial discovery operation failed.
+          this.retry = false;
+          this.launchDiscovery();
+        }
+      });
   }
 
   /**

extensions/ql-vscode/src/distribution.ts

@@ -7,10 +7,15 @@ import * as unzipper from 'unzipper';
 import * as url from 'url';
 import { ExtensionContext, Event } from 'vscode';
 import { DistributionConfig } from './config';
-import { InvocationRateLimiter, InvocationRateLimiterResultKind, showAndLogErrorMessage } from './helpers';
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage
+} from './helpers';
 import { logger } from './logging';
-import * as helpers from './helpers';
 import { getCodeQlCliVersion } from './cli-version';
+import { ProgressCallback, reportStreamProgress } from './commandRunner';
 
 /**
  * distribution.ts
@@ -45,19 +50,40 @@ export const DEFAULT_DISTRIBUTION_VERSION_RANGE: semver.Range = new semver.Range
 export interface DistributionProvider {
   getCodeQlPathWithoutVersionCheck(): Promise<string | undefined>;
   onDidChangeDistribution?: Event<void>;
+  getDistribution(): Promise<FindDistributionResult>;
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
-    this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
+
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  public static getRequiredAssetName(): string {
+    switch (os.platform()) {
+      case 'linux':
+        return 'codeql-linux64.zip';
+      case 'darwin':
+        return 'codeql-osx64.zip';
+      case 'win32':
+        return 'codeql-win64.zip';
+      default:
+        return 'codeql.zip';
+    }
+  }
+
+  constructor(
+    public readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    extensionContext: ExtensionContext
+  ) {
     this._onDidChangeDistribution = config.onDidChangeConfiguration;
-    this._updateCheckRateLimiter = new InvocationRateLimiter(
+    this.extensionSpecificDistributionManager =
+      new ExtensionSpecificDistributionManager(config, versionRange, extensionContext);
+    this.updateCheckRateLimiter = new InvocationRateLimiter(
       extensionContext,
       'extensionSpecificDistributionUpdateCheck',
-      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+      () => this.extensionSpecificDistributionManager.checkForUpdatesToDistribution()
     );
-    this._versionRange = versionRange;
   }
 
   /**
@@ -94,9 +120,9 @@ export class DistributionManager implements DistributionProvider {
      * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
      * includePrerelease config option is set.
      */
-    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this.config.includePrerelease;
 
-    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+    if (!semver.satisfies(version, this.versionRange, { includePrerelease })) {
       return {
         distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
@@ -125,9 +151,9 @@ export class DistributionManager implements DistributionProvider {
    */
   async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath) {
-      if (!await fs.pathExists(this._config.customCodeQlPath)) {
-        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
+    if (this.config.customCodeQlPath) {
+      if (!await fs.pathExists(this.config.customCodeQlPath)) {
+        void showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
           'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
           'that a CodeQL executable exists at the specified path or remove the setting.');
         return undefined;
@@ -136,18 +162,18 @@ export class DistributionManager implements DistributionProvider {
       // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
       if (
         deprecatedCodeQlLauncherName() &&
-        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        this.config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
         await this.hasNewLauncherName()
       ) {
         warnDeprecatedLauncher();
       }
       return {
-        codeQlPath: this._config.customCodeQlPath,
+        codeQlPath: this.config.customCodeQlPath,
         kind: DistributionKind.CustomPathConfig
       };
     }
 
-    const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionSpecificCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
       return {
         codeQlPath: extensionSpecificCodeQlPath,
@@ -165,7 +191,7 @@ export class DistributionManager implements DistributionProvider {
           };
         }
       }
-      logger.log('INFO: Could not find CodeQL on path.');
+      void logger.log('INFO: Could not find CodeQL on path.');
     }
 
     return undefined;
@@ -180,12 +206,12 @@ export class DistributionManager implements DistributionProvider {
   public async checkForUpdatesToExtensionManagedDistribution(
     minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
     const distribution = await this.getDistributionWithoutVersionCheck();
-    const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionManagedCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
       return createInvalidLocationResult();
     }
-    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    const updateCheckResult = await this.updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
     switch (updateCheckResult.kind) {
       case InvocationRateLimiterResultKind.Invoked:
         return updateCheckResult.result;
@@ -199,9 +225,11 @@ export class DistributionManager implements DistributionProvider {
    *
    * Returns a failed promise if an unexpected error occurs during installation.
    */
-  public installExtensionManagedDistributionRelease(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
-    return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
+  public installExtensionManagedDistributionRelease(
+    release: Release,
+    progressCallback?: ProgressCallback
+  ): Promise<void> {
+    return this.extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
   }
 
   public get onDidChangeDistribution(): Event<void> | undefined {
@@ -214,27 +242,27 @@ export class DistributionManager implements DistributionProvider {
    * installation. False otherwise.
    */
   private async hasNewLauncherName(): Promise<boolean> {
-    if (!this._config.customCodeQlPath) {
+    if (!this.config.customCodeQlPath) {
       // not managed externally
       return false;
     }
-    const dir = path.dirname(this._config.customCodeQlPath);
+    const dir = path.dirname(this.config.customCodeQlPath);
     const newLaunderPath = path.join(dir, codeQlLauncherName());
     return await fs.pathExists(newLaunderPath);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
-  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
+  private readonly extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionRange: semver.Range;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
-    this._extensionContext = extensionContext;
-    this._config = config;
-    this._versionRange = versionRange;
+  constructor(
+    private readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    private readonly extensionContext: ExtensionContext
+  ) {
+    /**/
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@@ -248,7 +276,7 @@ class ExtensionSpecificDistributionManager {
       try {
         await this.removeDistribution();
       } catch (e) {
-        logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
+        void logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
           `${this.getDistributionStoragePath()} but encountered an error: ${e}.`);
       }
     }
@@ -282,29 +310,29 @@ class ExtensionSpecificDistributionManager {
    * Returns a failed promise if an unexpected error occurs during installation.
    */
   public async installDistributionRelease(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     await this.downloadDistribution(release, progressCallback);
     // Store the installed release within the global extension state.
-    this.storeInstalledRelease(release);
+    await this.storeInstalledRelease(release);
   }
 
   private async downloadDistribution(release: Release,
-    progressCallback?: helpers.ProgressCallback): Promise<void> {
+    progressCallback?: ProgressCallback): Promise<void> {
     try {
       await this.removeDistribution();
     } catch (e) {
-      logger.log(`Tried to clean up old version of CLI at ${this.getDistributionStoragePath()} ` +
+      void logger.log(`Tried to clean up old version of CLI at ${this.getDistributionStoragePath()} ` +
         `but encountered an error: ${e}.`);
     }
 
     // Filter assets to the unique one that we require.
-    const requiredAssetName = this.getRequiredAssetName();
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
     const assets = release.assets.filter(asset => asset.name === requiredAssetName);
     if (assets.length === 0) {
       throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
     }
     if (assets.length > 1) {
-      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+      void logger.log('WARNING: chose a release with more than one asset to install, found ' +
         assets.map(asset => asset.name).join(', '));
     }
 
@@ -316,27 +344,8 @@ class ExtensionSpecificDistributionManager {
       const archiveFile = fs.createWriteStream(archivePath);
 
       const contentLength = assetStream.headers.get('content-length');
-      let numBytesDownloaded = 0;
-
-      if (progressCallback && contentLength !== null) {
-        const totalNumBytes = parseInt(contentLength, 10);
-        const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-        const updateProgress = (): void => {
-          progressCallback({
-            step: numBytesDownloaded,
-            maxStep: totalNumBytes,
-            message: `Downloading CodeQL CLI… [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
-          });
-        };
-
-        // Display the progress straight away rather than waiting for the first chunk.
-        updateProgress();
-
-        assetStream.body.on('data', data => {
-          numBytesDownloaded += data.length;
-          updateProgress();
-        });
-      }
+      const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
+      reportStreamProgress(assetStream.body, `Downloading CodeQL CLI ${release.name}…`, totalNumBytes, progressCallback);
 
       await new Promise((resolve, reject) =>
         assetStream.body.pipe(archiveFile)
@@ -346,7 +355,7 @@ class ExtensionSpecificDistributionManager {
 
       await this.bumpDistributionFolderIndex();
 
-      logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
+      void logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
       await extractZipArchive(archivePath, this.getDistributionStoragePath());
     } finally {
       await fs.remove(tmpDirectory);
@@ -359,37 +368,27 @@ class ExtensionSpecificDistributionManager {
    * This should not be called for a distribution that is currently in use, as remove may fail.
    */
   private async removeDistribution(): Promise<void> {
-    this.storeInstalledRelease(undefined);
+    await this.storeInstalledRelease(undefined);
     if (await fs.pathExists(this.getDistributionStoragePath())) {
       await fs.remove(this.getDistributionStoragePath());
     }
   }
 
-  /**
-   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
-   */
-  private getRequiredAssetName(): string {
-    if (os.platform() === 'linux') return 'codeql-linux64.zip';
-    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
-    if (os.platform() === 'win32') return 'codeql-win64.zip';
-    return 'codeql.zip';
-  }
-
   private async getLatestRelease(): Promise<Release> {
-    const requiredAssetName = this.getRequiredAssetName();
-    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
+    void logger.log(`Searching for latest release including ${requiredAssetName}.`);
     return this.createReleasesApiConsumer().getLatestRelease(
-      this._versionRange,
-      this._config.includePrerelease,
+      this.versionRange,
+      this.config.includePrerelease,
       release => {
         const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
         if (matchingAssets.length === 0) {
           // For example, this could be a release with no platform-specific assets.
-          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          void logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
           return false;
         }
         if (matchingAssets.length > 1) {
-          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          void logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
           return false;
         }
         return true;
@@ -398,23 +397,23 @@ class ExtensionSpecificDistributionManager {
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
-    const ownerName = this._config.ownerName ? this._config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
-    const repositoryName = this._config.repositoryName ? this._config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
-    return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
+    const ownerName = this.config.ownerName ? this.config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
+    const repositoryName = this.config.repositoryName ? this.config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
+    return new ReleasesApiConsumer(ownerName, repositoryName, this.config.personalAccessToken);
   }
 
   private async bumpDistributionFolderIndex(): Promise<void> {
-    const index = this._extensionContext.globalState.get(
+    const index = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    await this._extensionContext.globalState.update(
+    await this.extensionContext.globalState.update(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
   }
 
   private getDistributionStoragePath(): string {
     // Use an empty string for the initial distribution for backwards compatibility.
-    const distributionFolderIndex = this._extensionContext.globalState.get(
+    const distributionFolderIndex = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || '';
-    return path.join(this._extensionContext.globalStoragePath,
+    return path.join(this.extensionContext.globalStoragePath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName + distributionFolderIndex);
   }
 
@@ -424,17 +423,13 @@ class ExtensionSpecificDistributionManager {
   }
 
   private getInstalledRelease(): Release | undefined {
-    return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
+    return this.extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
   }
 
   private async storeInstalledRelease(release: Release | undefined): Promise<void> {
-    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+    await this.extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionContext: ExtensionContext;
-  private readonly _versionRange: semver.Range;
-
   private static readonly _currentDistributionFolderBaseName = 'distribution';
   private static readonly _currentDistributionFolderIndexStateKey = 'distributionFolderIndex';
   private static readonly _installedReleaseStateKey = 'distributionRelease';
@@ -575,7 +570,7 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-function codeQlLauncherName(): string {
+export function codeQlLauncherName(): string {
   return (os.platform() === 'win32') ? 'codeql.exe' : 'codeql';
 }
 
@@ -712,14 +707,14 @@ export async function getExecutableFromDirectory(directory: string, warnWhenNotF
     return alternateExpectedLauncherPath;
   }
   if (warnWhenNotFound) {
-    logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+    void logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
       'Will try PATH.');
   }
   return undefined;
 }
 
 function warnDeprecatedLauncher() {
-  helpers.showAndLogWarningMessage(
+  void showAndLogWarningMessage(
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
     `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
   );

extensions/ql-vscode/src/extension.ts

@@ -10,24 +10,32 @@ import {
   Uri,
   window as Window,
   env,
-  window
+  window,
+  QuickPickItem,
+  Range,
+  workspace,
+  ProviderResult
 } from 'vscode';
 import { LanguageClient } from 'vscode-languageclient';
+import * as os from 'os';
 import * as path from 'path';
+import * as tmp from 'tmp-promise';
 import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
 
 import { AstViewer } from './astViewer';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { CodeQLCliServer } from './cli';
+import QuickEvalCodeLensProvider from './quickEvalCodeLensProvider';
+import { CodeQLCliServer, CliVersionConstraint } from './cli';
 import {
   CliConfigListener,
   DistributionConfigListener,
+  isCanary,
   MAX_QUERIES,
   QueryHistoryConfigListener,
   QueryServerConfigListener
 } from './config';
 import * as languageSupport from './languageSupport';
-import { DatabaseManager } from './databases';
+import { DatabaseItem, DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
 import {
   TemplateQueryDefinitionProvider,
@@ -45,7 +53,7 @@ import {
   GithubRateLimitedError
 } from './distribution';
 import * as helpers from './helpers';
-import { assertNever } from './helpers-pure';
+import { assertNever } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
 import { InterfaceManager } from './interface';
 import { WebviewReveal } from './interface-utils';
@@ -58,7 +66,21 @@ import { compileAndRunQueryAgainstDatabase, tmpDirDisposal } from './run-queries
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
 import { CompareInterfaceManager } from './compare/compare-interface';
-import { gatherQlFiles } from './files';
+import { gatherQlFiles } from './pure/files';
+import { initializeTelemetry } from './telemetry';
+import {
+  commandRunner,
+  commandRunnerWithProgress,
+  ProgressCallback,
+  withProgress,
+  ProgressUpdate
+} from './commandRunner';
+import { CodeQlStatusBarHandler } from './status-bar';
+
+import { Credentials } from './authentication';
+import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
+import { RemoteQuery } from './remote-queries/remote-query';
+import { URLSearchParams } from 'url';
 
 /**
  * extension.ts
@@ -87,6 +109,9 @@ const errorStubs: Disposable[] = [];
  */
 let isInstallingOrUpdatingDistribution = false;
 
+const extensionId = 'GitHub.vscode-codeql';
+const extension = extensions.getExtension(extensionId);
+
 /**
  * If the user tries to execute vscode commands after extension activation is failed, give
  * a sensible error message.
@@ -97,8 +122,6 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
-  const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
-  const extension = extensions.getExtension(extensionId);
   if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
   }
@@ -108,26 +131,58 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 
   stubbedCommands.forEach(command => {
     if (excludedCommands.indexOf(command) === -1) {
-      errorStubs.push(helpers.commandRunner(command, stubGenerator(command)));
+      errorStubs.push(commandRunner(command, stubGenerator(command)));
     }
   });
 }
 
-export async function activate(ctx: ExtensionContext): Promise<void> {
-  logger.log('Starting CodeQL extension');
+/**
+ * The publicly available interface for this extension. This is to
+ * be used in our tests.
+ */
+export interface CodeQLExtensionInterface {
+  readonly ctx: ExtensionContext;
+  readonly cliServer: CodeQLCliServer;
+  readonly qs: qsClient.QueryServerClient;
+  readonly distributionManager: DistributionManager;
+  readonly databaseManager: DatabaseManager;
+  readonly databaseUI: DatabaseUI;
+  readonly dispose: () => void;
+}
 
-  initializeLogging(ctx);
-  languageSupport.install();
+/**
+ * Returns the CodeQLExtensionInterface, or an empty object if the interface is not
+ * available after activation is complete. This will happen if there is no cli
+ * installed when the extension starts. Downloading and installing the cli
+ * will happen at a later time.
+ *
+ * @param ctx The extension context
+ *
+ * @returns CodeQLExtensionInterface
+ */
+export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | Record<string, never>> {
+
+  void logger.log(`Starting ${extensionId} extension`);
+  if (extension === undefined) {
+    throw new Error(`Can't find extension ${extensionId}`);
+  }
 
   const distributionConfigListener = new DistributionConfigListener();
+  await initializeLogging(ctx);
+  await initializeTelemetry(extension, ctx);
+  languageSupport.install();
+
+  const codelensProvider = new QuickEvalCodeLensProvider();
+  languages.registerCodeLensProvider({ scheme: 'file', language: 'ql' }, codelensProvider);
+
   ctx.subscriptions.push(distributionConfigListener);
   const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
+  const distributionManager = new DistributionManager(distributionConfigListener, codeQlVersionRange, ctx);
 
   const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
   registerErrorStubs([checkForUpdatesCommand], command => (async () => {
-    helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
+    void helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
   }));
 
   interface DistributionUpdateConfig {
@@ -139,7 +194,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
   async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
     const minSecondsSinceLastUpdateCheck = config.isUserInitiated ? 0 : 86400;
     const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
-      helpers.showAndLogInformationMessage : async (message: string) => logger.log(message);
+      helpers.showAndLogInformationMessage : async (message: string) => void logger.log(message);
     const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
 
     // We do want to auto update if there is no distribution at all
@@ -147,7 +202,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
     switch (result.kind) {
       case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
-        logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
+        void logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
           `${minSecondsSinceLastUpdateCheck} seconds.`);
         break;
       case DistributionUpdateCheckResultKind.AlreadyUpToDate:
@@ -170,11 +225,11 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
             location: ProgressLocation.Notification,
           };
 
-          await helpers.withProgress(progressOptions, progress =>
+          await withProgress(progressOptions, progress =>
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
-          helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
+          void helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
         }
         break;
       default:
@@ -206,12 +261,12 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
         codeQlInstalled ? 'check for updates to' : 'install') + ' CodeQL CLI';
 
       if (e instanceof GithubRateLimitedError) {
-        alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
+        void alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
           `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString(env.language)}.`);
       } else if (e instanceof GithubApiError) {
-        alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
+        void alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
       }
-      alertFunction(`Unable to ${taskDescription}. ` + e);
+      void alertFunction(`Unable to ${taskDescription}. ` + e);
     } finally {
       isInstallingOrUpdatingDistribution = false;
     }
@@ -221,7 +276,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
+        void logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
       case FindDistributionResultKind.IncompatibleDistribution: {
         const fixGuidanceMessage = (() => {
@@ -236,16 +291,20 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
           }
         })();
 
-        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
-          'is incompatible with this extension. ' + fixGuidanceMessage);
+        void helpers.showAndLogWarningMessage(
+          `The current version of the CodeQL CLI (${result.version.raw}) ` +
+          `is incompatible with this extension. ${fixGuidanceMessage}`
+        );
         break;
       }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
-        helpers.showAndLogWarningMessage('Compatibility with the configured CodeQL CLI could not be determined. ' +
-          'You may experience problems using the extension.');
+        void helpers.showAndLogWarningMessage(
+          'Compatibility with the configured CodeQL CLI could not be determined. ' +
+          'You may experience problems using the extension.'
+        );
         break;
       case FindDistributionResultKind.NoDistribution:
-        helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
+        void helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
         break;
       default:
         assertNever(result);
@@ -253,22 +312,30 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
+  async function installOrUpdateThenTryActivate(
+    config: DistributionUpdateConfig
+  ): Promise<CodeQLExtensionInterface | Record<string, never>> {
+
     await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
-
+    let extensionInterface: CodeQLExtensionInterface | Record<string, never> = {};
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(
+        ctx,
+        distributionManager,
+        distributionConfigListener
+      );
+
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = 'Install CodeQL CLI';
-        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+        const chosenAction = await void helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
           items: [installActionName]
         });
         if (chosenAction === installActionName) {
-          installOrUpdateThenTryActivate({
+          await installOrUpdateThenTryActivate({
             isUserInitiated: true,
             shouldDisplayMessageWhenNoUpdates: false,
             allowAutoUpdating: true
@@ -276,6 +343,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
         }
       });
     }
+    return extensionInterface;
   }
 
   ctx.subscriptions.push(distributionConfigListener.onDidChangeConfiguration(() => installOrUpdateThenTryActivate({
@@ -283,13 +351,13 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     shouldDisplayMessageWhenNoUpdates: false,
     allowAutoUpdating: true
   })));
-  ctx.subscriptions.push(helpers.commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
+  ctx.subscriptions.push(commandRunner(checkForUpdatesCommand, () => installOrUpdateThenTryActivate({
     isUserInitiated: true,
     shouldDisplayMessageWhenNoUpdates: true,
     allowAutoUpdating: true
   })));
 
-  await installOrUpdateThenTryActivate({
+  return await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
 
@@ -301,20 +369,21 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
-  distributionManager: DistributionManager
-): Promise<void> {
+  distributionManager: DistributionManager,
+  distributionConfigListener: DistributionConfigListener
+): Promise<CodeQLExtensionInterface> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
   errorStubs.forEach((stub) => stub.dispose());
 
-  logger.log('Initializing configuration listener...');
+  void logger.log('Initializing configuration listener...');
   const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(
     distributionManager
   );
   ctx.subscriptions.push(qlConfigurationListener);
 
-  logger.log('Initializing CodeQL cli server...');
+  void logger.log('Initializing CodeQL cli server...');
   const cliServer = new CodeQLCliServer(
     distributionManager,
     new CliConfigListener(),
@@ -322,12 +391,16 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cliServer);
 
-  logger.log('Initializing query server client.');
+  const statusBar = new CodeQlStatusBarHandler(cliServer, distributionConfigListener);
+  ctx.subscriptions.push(statusBar);
+
+  void logger.log('Initializing query server client.');
   const qs = new qsClient.QueryServerClient(
     qlConfigurationListener,
     cliServer,
     {
       logger: queryServerLogger,
+      contextStoragePath: getContextStoragePath(ctx),
     },
     (task) =>
       Window.withProgress(
@@ -338,21 +411,22 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
-  logger.log('Initializing database manager.');
-  const dbm = new DatabaseManager(ctx, qlConfigurationListener, logger);
+  void logger.log('Initializing database manager.');
+  const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
   ctx.subscriptions.push(dbm);
-  logger.log('Initializing database panel.');
+  void logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(
-    cliServer,
     dbm,
     qs,
     getContextStoragePath(ctx),
     ctx.extensionPath
   );
+  databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
-  logger.log('Initializing query history manager.');
+  void logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  ctx.subscriptions.push(queryHistoryConfigurationListener);
   const showResults = async (item: CompletedQuery) =>
     showResultsForCompletedQuery(item, WebviewReveal.Forced);
 
@@ -365,11 +439,11 @@ async function activateWithInstalledDistribution(
       showResultsForComparison(from, to),
   );
   ctx.subscriptions.push(qhm);
-  logger.log('Initializing results panel interface.');
+  void logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
 
-  logger.log('Initializing compare panel interface.');
+  void logger.log('Initializing compare panel interface.');
   const cmpm = new CompareInterfaceManager(
     ctx,
     dbm,
@@ -379,7 +453,7 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cmpm);
 
-  logger.log('Initializing source archive filesystem provider.');
+  void logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
   async function showResultsForComparison(
@@ -389,7 +463,7 @@ async function activateWithInstalledDistribution(
     try {
       await cmpm.showResults(from, to);
     } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+      void helpers.showAndLogErrorMessage(e.message);
     }
   }
 
@@ -403,35 +477,85 @@ async function activateWithInstalledDistribution(
   async function compileAndRunQuery(
     quickEval: boolean,
     selectedQuery: Uri | undefined,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
+    databaseItem: DatabaseItem | undefined,
+    range?: Range
   ): Promise<void> {
     if (qs !== undefined) {
-      const dbItem = await databaseUI.getDatabaseItem(progress, token);
-      if (dbItem === undefined) {
+      // If no databaseItem is specified, use the database currently selected in the Databases UI
+      databaseItem = databaseItem || await databaseUI.getDatabaseItem(progress, token);
+      if (databaseItem === undefined) {
         throw new Error('Can\'t run query without a selected database');
       }
       const info = await compileAndRunQueryAgainstDatabase(
         cliServer,
         qs,
-        dbItem,
+        databaseItem,
         quickEval,
         selectedQuery,
         progress,
-        token
+        token,
+        undefined,
+        range
       );
-      const item = qhm.addQuery(info);
+      const item = qhm.buildCompletedQuery(info);
       await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      // The call to showResults potentially creates SARIF file;
-      // Update the tree item context value to allow viewing that
-      // SARIF file from context menu.
-      await qhm.updateTreeItemContextValue(item);
+      // Note we must update the query history view after showing results as the
+      // display and sorting might depend on the number of results
+      await qhm.addCompletedQuery(item);
+    }
+  }
+
+  const qhelpTmpDir = tmp.dirSync({ prefix: 'qhelp_', keep: false, unsafeCleanup: true });
+  ctx.subscriptions.push({ dispose: qhelpTmpDir.removeCallback });
+
+  async function previewQueryHelp(
+    selectedQuery: Uri
+  ): Promise<void> {
+    // selectedQuery is unpopulated when executing through the command palette
+    const pathToQhelp = selectedQuery ? selectedQuery.fsPath : window.activeTextEditor?.document.uri.fsPath;
+    if (pathToQhelp) {
+      // Create temporary directory
+      const relativePathToMd = path.basename(pathToQhelp, '.qhelp') + '.md';
+      const absolutePathToMd = path.join(qhelpTmpDir.name, relativePathToMd);
+      const uri = Uri.file(absolutePathToMd);
+      try {
+        await cliServer.generateQueryHelp(pathToQhelp, absolutePathToMd);
+        await commands.executeCommand('markdown.showPreviewToSide', uri);
+      } catch (err) {
+        const errorMessage = err.message.includes('Generating qhelp in markdown') ? (
+          `Could not generate markdown from ${pathToQhelp}: Bad formatting in .qhelp file.`
+        ) : `Could not open a preview of the generated file (${absolutePathToMd}).`;
+        void helpers.showAndLogErrorMessage(errorMessage, { fullMessage: `${errorMessage}\n${err}` });
+      }
+    }
+
+  }
+
+  async function openReferencedFile(
+    selectedQuery: Uri
+  ): Promise<void> {
+    // If no file is selected, the path of the file in the editor is selected
+    const path = selectedQuery?.fsPath || window.activeTextEditor?.document.uri.fsPath;
+    if (qs !== undefined && path) {
+      if (await cliServer.cliConstraints.supportsResolveQlref()) {
+        const resolved = await cliServer.resolveQlref(path);
+        const uri = Uri.file(resolved.resolvedPath);
+        await window.showTextDocument(uri, { preview: false });
+      } else {
+        void helpers.showAndLogErrorMessage(
+          'Jumping from a .qlref file to the .ql file it references is not '
+          + 'supported with the CLI version you are running.\n'
+          + `Please upgrade your CLI to version ${CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF
+          } or later to use this feature.`);
+      }
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
-  logger.log('Initializing CodeQL language server.');
+  void logger.log('Initializing CodeQL language server.');
   const client = new LanguageClient(
     'CodeQL Language Server',
     () => spawnIdeServer(qlConfigurationListener),
@@ -449,39 +573,106 @@ async function activateWithInstalledDistribution(
     true
   );
 
-  logger.log('Initializing QLTest interface.');
+  void logger.log('Initializing QLTest interface.');
   const testExplorerExtension = extensions.getExtension<TestHub>(
     testExplorerExtensionId
   );
   if (testExplorerExtension) {
     const testHub = testExplorerExtension.exports;
-    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
+    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer, dbm);
     ctx.subscriptions.push(testAdapterFactory);
 
     const testUIService = new TestUIService(testHub);
     ctx.subscriptions.push(testUIService);
   }
 
-  logger.log('Registering top-level command palette commands.');
+  void logger.log('Registering top-level command palette commands.');
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.runQuery',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
-      ) => await compileAndRunQuery(false, uri, progress, token),
+      ) => await compileAndRunQuery(false, uri, progress, token, undefined),
       {
         title: 'Running query',
         cancellable: true
       }
     )
   );
+  interface DatabaseQuickPickItem extends QuickPickItem {
+    databaseItem: DatabaseItem;
+  }
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
+      'codeQL.runQueryOnMultipleDatabases',
+      async (
+        progress: ProgressCallback,
+        token: CancellationToken,
+        uri: Uri | undefined
+      ) => {
+        let filteredDBs = dbm.databaseItems;
+        if (filteredDBs.length === 0) {
+          void helpers.showAndLogErrorMessage('No databases found. Please add a suitable database to your workspace.');
+          return;
+        }
+        // If possible, only show databases with the right language (otherwise show all databases).
+        const queryLanguage = await helpers.findLanguage(cliServer, uri);
+        if (queryLanguage) {
+          filteredDBs = dbm.databaseItems.filter(db => db.language === queryLanguage);
+          if (filteredDBs.length === 0) {
+            void helpers.showAndLogErrorMessage(`No databases found for language ${queryLanguage}. Please add a suitable database to your workspace.`);
+            return;
+          }
+        }
+        const quickPickItems = filteredDBs.map<DatabaseQuickPickItem>(dbItem => (
+          {
+            databaseItem: dbItem,
+            label: dbItem.name,
+            description: dbItem.language,
+          }
+        ));
+        /**
+         * Databases that were selected in the quick pick menu.
+         */
+        const quickpick = await window.showQuickPick<DatabaseQuickPickItem>(
+          quickPickItems,
+          { canPickMany: true, ignoreFocusOut: true }
+        );
+        if (quickpick !== undefined) {
+          // Collect all skipped databases and display them at the end (instead of popping up individual errors)
+          const skippedDatabases = [];
+          const errors = [];
+          for (const item of quickpick) {
+            try {
+              await compileAndRunQuery(false, uri, progress, token, item.databaseItem);
+            } catch (error) {
+              skippedDatabases.push(item.label);
+              errors.push(error.message);
+            }
+          }
+          if (skippedDatabases.length > 0) {
+            void logger.log(`Errors:\n${errors.join('\n')}`);
+            void helpers.showAndLogWarningMessage(
+              `The following databases were skipped:\n${skippedDatabases.join('\n')}.\nFor details about the errors, see the logs.`
+            );
+          }
+        } else {
+          void helpers.showAndLogErrorMessage('No databases selected.');
+        }
+      },
+      {
+        title: 'Running query on selected databases',
+        cancellable: true
+      }
+    )
+  );
+  ctx.subscriptions.push(
+    commandRunnerWithProgress(
       'codeQL.runQueries',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         _: Uri | undefined,
         multi: Uri[]
@@ -506,7 +697,7 @@ async function activateWithInstalledDistribution(
 
         // Use a wrapped progress so that messages appear with the queries remaining in it.
         let queriesRemaining = queryUris.length;
-        function wrappedProgress(update: helpers.ProgressUpdate) {
+        function wrappedProgress(update: ProgressUpdate) {
           const message = queriesRemaining > 1
             ? `${queriesRemaining} remaining. ${update.message}`
             : update.message;
@@ -532,7 +723,7 @@ async function activateWithInstalledDistribution(
         });
 
         await Promise.all(queryUris.map(async uri =>
-          compileAndRunQuery(false, uri, wrappedProgress, token)
+          compileAndRunQuery(false, uri, wrappedProgress, token, undefined)
             .then(() => queriesRemaining--)
         ));
       },
@@ -542,21 +733,37 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress(
+    commandRunnerWithProgress(
       'codeQL.quickEval',
       async (
-        progress: helpers.ProgressCallback,
+        progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
-      ) => await compileAndRunQuery(true, uri, progress, token),
+      ) => await compileAndRunQuery(true, uri, progress, token, undefined),
       {
         title: 'Running query',
         cancellable: true
       })
   );
+
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.quickQuery', async (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress(
+      'codeQL.codeLensQuickEval',
+      async (
+        progress: ProgressCallback,
+        token: CancellationToken,
+        uri: Uri,
+        range: Range
+      ) => await compileAndRunQuery(true, uri, progress, token, undefined, range),
+      {
+        title: 'Running query',
+        cancellable: true
+      })
+  );
+
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.quickQuery', async (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       displayQuickQuery(ctx, cliServer, databaseUI, progress, token),
@@ -566,33 +773,94 @@ async function activateWithInstalledDistribution(
     )
   );
 
+  void logger.log('Initializing remote queries interface.');
+  const rqm = new RemoteQueriesManager(ctx, logger, cliServer);
+
+  registerRemoteQueryTextProvider();
+
+  // The "runRemoteQuery" command is internal-only.
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.restartQueryServer', async () => {
-      await qs.restartQueryServer();
-      helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
+    commandRunnerWithProgress('codeQL.runRemoteQuery', async (
+      progress: ProgressCallback,
+      token: CancellationToken,
+      uri: Uri | undefined
+    ) => {
+      if (isCanary()) {
+        progress({
+          maxStep: 5,
+          step: 0,
+          message: 'Getting credentials'
+        });
+        await rqm.runRemoteQuery(
+          uri || window.activeTextEditor?.document.uri,
+          progress,
+          token
+        );
+      } else {
+        throw new Error('Remote queries require the CodeQL Canary version to run.');
+      }
+    }, {
+      title: 'Run Remote Query',
+      cancellable: true
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.monitorRemoteQuery', async (
+      query: RemoteQuery,
+      token: CancellationToken) => {
+      await rqm.monitorRemoteQuery(query, token);
+    }));
+
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.openReferencedFile',
+      openReferencedFile
+    )
+  );
+
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.previewQueryHelp',
+      previewQueryHelp
+    )
+  );
+
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.restartQueryServer', async (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) => {
+      await qs.restartQueryServer(progress, token);
+      void helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
       });
+    }, {
+      title: 'Restarting Query Server'
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.chooseDatabaseFolder', (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) =>
+      databaseUI.handleChooseDatabaseFolder(progress, token), {
+      title: 'Choose a Database from a Folder'
     })
   );
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.chooseDatabaseFolder', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseArchive', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
-      databaseUI.handleChooseDatabaseFolder(progress, token)
-    )
+      databaseUI.handleChooseDatabaseArchive(progress, token), {
+      title: 'Choose a Database from an Archive'
+    })
   );
   ctx.subscriptions.push(
-    helpers.commandRunner('codeQL.chooseDatabaseArchive', (
-      progress: helpers.ProgressCallback,
-      token: CancellationToken
-    ) =>
-      databaseUI.handleChooseDatabaseArchive(progress, token)
-    )
-  );
-  ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseLgtm(progress, token),
@@ -601,8 +869,8 @@ async function activateWithInstalledDistribution(
       })
   );
   ctx.subscriptions.push(
-    helpers.commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
-      progress: helpers.ProgressCallback,
+    commandRunnerWithProgress('codeQL.chooseDatabaseInternet', (
+      progress: ProgressCallback,
       token: CancellationToken
     ) =>
       databaseUI.handleChooseDatabaseInternet(progress, token),
@@ -612,11 +880,49 @@ async function activateWithInstalledDistribution(
       })
   );
 
-  logger.log('Starting language server.');
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openDocumentation', async () =>
+      env.openExternal(Uri.parse('https://codeql.github.com/docs/'))));
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.copyVersion', async () => {
+      const text = `CodeQL extension version: ${extension?.packageJSON.version} \nCodeQL CLI version: ${await getCliVersion()} \nPlatform: ${os.platform()} ${os.arch()}`;
+      await env.clipboard.writeText(text);
+      void helpers.showAndLogInformationMessage(text);
+    }));
+
+  const getCliVersion = async () => {
+    try {
+      return await cliServer.getVersion();
+    } catch {
+      return '<missing>';
+    }
+  };
+
+  // The "authenticateToGitHub" command is internal-only.
+  ctx.subscriptions.push(
+    commandRunner('codeQL.authenticateToGitHub', async () => {
+      if (isCanary()) {
+        /**
+         * Credentials for authenticating to GitHub.
+         * These are used when making API calls.
+         */
+        const credentials = await Credentials.initialize(ctx);
+        const octokit = await credentials.getOctokit();
+        const userInfo = await octokit.users.getAuthenticated();
+        void helpers.showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
+      }
+    }));
+
+  commands.registerCommand('codeQL.showLogs', () => {
+    logger.show();
+  });
+
+  void logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
   // Jump-to-definition and find-references
-  logger.log('Registering jump-to-definition handlers.');
+  void logger.log('Registering jump-to-definition handlers.');
   languages.registerDefinitionProvider(
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
     new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
@@ -628,13 +934,18 @@ async function activateWithInstalledDistribution(
   );
 
   const astViewer = new AstViewer();
+  const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm);
+
   ctx.subscriptions.push(astViewer);
-  ctx.subscriptions.push(helpers.commandRunnerWithProgress('codeQL.viewAst', async (
-    progress: helpers.ProgressCallback,
+  ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
+    progress: ProgressCallback,
     token: CancellationToken
   ) => {
-    const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)
-      .provideAst(window.activeTextEditor?.document);
+    const ast = await templateProvider.provideAst(
+      progress,
+      token,
+      window.activeTextEditor?.document,
+    );
     if (ast) {
       astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
     }
@@ -643,21 +954,51 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
-  logger.log('Successfully finished extension initialization.');
+  await commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
+
+  void logger.log('Successfully finished extension initialization.');
+
+  return {
+    ctx,
+    cliServer,
+    qs,
+    distributionManager,
+    databaseManager: dbm,
+    databaseUI,
+    dispose: () => {
+      ctx.subscriptions.forEach(d => d.dispose());
+    }
+  };
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {
   return ctx.storagePath || ctx.globalStoragePath;
 }
 
-function initializeLogging(ctx: ExtensionContext): void {
+async function initializeLogging(ctx: ExtensionContext): Promise<void> {
   const storagePath = getContextStoragePath(ctx);
-  logger.init(storagePath);
-  queryServerLogger.init(storagePath);
-  ideServerLogger.init(storagePath);
+  await logger.setLogStoragePath(storagePath, false);
+  await ideServerLogger.setLogStoragePath(storagePath, false);
   ctx.subscriptions.push(logger);
   ctx.subscriptions.push(queryServerLogger);
   ctx.subscriptions.push(ideServerLogger);
 }
 
 const checkForUpdatesCommand = 'codeQL.checkForUpdatesToCLI';
+
+/**
+ * This text provider lets us open readonly files in the editor.
+ * 
+ * TODO: Consolidate this with the 'codeql' text provider in query-history.ts.
+ */
+function registerRemoteQueryTextProvider() {
+  workspace.registerTextDocumentContentProvider('remote-query', {
+    provideTextDocumentContent(
+      uri: Uri
+    ): ProviderResult<string> {
+      const params = new URLSearchParams(uri.query);
+
+      return params.get('queryText');
+    },
+  });
+}

extensions/ql-vscode/src/helpers.ts

@@ -3,176 +3,16 @@ import * as glob from 'glob-promise';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
 import {
-  CancellationToken,
   ExtensionContext,
-  ProgressOptions,
+  Uri,
   window as Window,
   workspace,
-  commands,
-  Disposable,
-  ProgressLocation
+  env
 } from 'vscode';
-import { CodeQLCliServer } from './cli';
+import { CodeQLCliServer, QlpacksInfo } from './cli';
+import { UserCancellationException } from './commandRunner';
 import { logger } from './logging';
-
-export class UserCancellationException extends Error {
-  /**
-   * @param message The error message
-   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
-   */
-  constructor(message?: string, public readonly silent = false) {
-    super(message);
-  }
-}
-
-export interface ProgressUpdate {
-  /**
-   * The current step
-   */
-  step: number;
-  /**
-   * The maximum step. This *should* be constant for a single job.
-   */
-  maxStep: number;
-  /**
-   * The current progress message
-   */
-  message: string;
-}
-
-export type ProgressCallback = (p: ProgressUpdate) => void;
-
-/**
- * A task that handles command invocations from `commandRunner`
- * and includes a progress monitor.
- *
- *
- * Arguments passed to the command handler are passed along,
- * untouched to this `ProgressTask` instance.
- *
- * @param progress a progress handler function. Call this
- * function with a `ProgressUpdate` instance in order to
- * denote some progress being achieved on this task.
- * @param token a cencellation token
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-export type ProgressTask<R> = (
-  progress: ProgressCallback,
-  token: CancellationToken,
-  ...args: any[]
-) => Thenable<R>;
-
-/**
- * A task that handles command invocations from `commandRunner`.
- * Arguments passed to the command handler are passed along,
- * untouched to this `NoProgressTask` instance.
- *
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-type NoProgressTask = ((...args: any[]) => Promise<any>);
-
-/**
- * This mediates between the kind of progress callbacks we want to
- * write (where we *set* current progress position and give
- * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%).
- *
- * Where possible, the `commandRunner` function below should be used
- * instead of this function. The commandRunner is meant for wrapping
- * top-level commands and provides error handling and other support
- * automatically.
- *
- * Only use this function if you need a progress monitor and the
- * control flow does not always come from a command (eg- during
- * extension activation, or from an internal language server
- * request).
- */
-export function withProgress<R>(
-  options: ProgressOptions,
-  task: ProgressTask<R>,
-  ...args: any[]
-): Thenable<R> {
-  let progressAchieved = 0;
-  return Window.withProgress(options,
-    (progress, token) => {
-      return task(p => {
-        const { message, step, maxStep } = p;
-        const increment = 100 * (step - progressAchieved) / maxStep;
-        progressAchieved = step;
-        progress.report({ message, increment });
-      }, token, ...args);
-    });
-}
-
-/**
- * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
- *
- * In this variant of the command runner, no progress monitor is used.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task.
- */
-export function commandRunner(
-  commandId: string,
-  task: NoProgressTask,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    try {
-      await task(...args);
-    } catch (e) {
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          logger.log(e.message);
-        } else {
-          showAndLogWarningMessage(e.message);
-        }
-      } else {
-        showAndLogErrorMessage(e.message || e);
-      }
-    }
-  });
-}
-
-/**
- * A generic wrapper for command registration.  This wrapper adds uniform error handling,
- * progress monitoring, and cancellation for commands.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task after the progress callback
- * and cancellation token.
- * @param progressOptions Progress options to be sent to the progress monitor.
- */
-export function commandRunnerWithProgress<R>(
-  commandId: string,
-  task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const progressOptionsWithDefaults = {
-      location: ProgressLocation.Notification,
-      ...progressOptions
-    };
-    try {
-      await withProgress(progressOptionsWithDefaults, task, ...args);
-    } catch (e) {
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          logger.log(e.message);
-        } else {
-          showAndLogWarningMessage(e.message);
-        }
-      } else {
-        showAndLogErrorMessage(e.message || e);
-      }
-    }
-  });
-}
+import { QueryMetadata } from './pure/interface-types';
 
 /**
  * Show an error message and log it to the console
@@ -180,15 +20,24 @@ export function commandRunnerWithProgress<R>(
  * @param message The message to show.
  * @param options.outputLogger The output logger that will receive the message
  * @param options.items A set of items that will be rendered as actions in the message.
+ * @param options.fullMessage An alternate message that is added to the log, but not displayed
+ *                           in the popup. This is useful for adding extra detail to the logs
+ *                           that would be too noisy for the popup.
  *
  * @return A promise that resolves to the selected item or undefined when being dismissed.
  */
 export async function showAndLogErrorMessage(message: string, {
   outputLogger = logger,
-  items = [] as string[]
+  items = [] as string[],
+  fullMessage = undefined as (string | undefined)
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage);
+  return internalShowAndLog(dropLinesExceptInitial(message), items, outputLogger, Window.showErrorMessage, fullMessage);
 }
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join('\n');
+}
+
 /**
  * Show a warning message and log it to the console
  *
@@ -222,10 +71,15 @@ export async function showAndLogInformationMessage(message: string, {
 
 type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
 
-async function internalShowAndLog(message: string, items: string[], outputLogger = logger,
-  fn: ShowMessageFn): Promise<string | undefined> {
+async function internalShowAndLog(
+  message: string,
+  items: string[],
+  outputLogger = logger,
+  fn: ShowMessageFn,
+  fullMessage?: string
+): Promise<string | undefined> {
   const label = 'Show Log';
-  outputLogger.log(message);
+  void outputLogger.log(fullMessage || message);
   const result = await fn(message, label, ...items);
   if (result === label) {
     outputLogger.show();
@@ -235,17 +89,61 @@ async function internalShowAndLog(message: string, items: string[], outputLogger
 
 /**
  * Opens a modal dialog for the user to make a yes/no choice.
- * @param message The message to show.
  *
- * @return `true` if the user clicks 'Yes', `false` if the user clicks 'No' or cancels the dialog.
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
  */
-export async function showBinaryChoiceDialog(message: string): Promise<boolean> {
+export async function showBinaryChoiceDialog(message: string, modal = true): Promise<boolean | undefined> {
   const yesItem = { title: 'Yes', isCloseAffordance: false };
   const noItem = { title: 'No', isCloseAffordance: true };
-  const chosenItem = await Window.showInformationMessage(message, { modal: true }, yesItem, noItem);
+  const chosenItem = await Window.showInformationMessage(message, { modal }, yesItem, noItem);
+  if (!chosenItem) {
+    return undefined;
+  }
   return chosenItem?.title === yesItem.title;
 }
 
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(message: string, url: string): Promise<boolean | undefined> {
+  const urlItem = { title: 'More Information', isCloseAffordance: false };
+  const yesItem = { title: 'Yes', isCloseAffordance: false };
+  const noItem = { title: 'No', isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await Window.showInformationMessage(message, { modal: true }, urlItem, yesItem, noItem);
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
 /**
  * Show an information message with a customisable action.
  * @param message The message to show.
@@ -358,40 +256,78 @@ function createRateLimitedResult(): RateLimitedResult {
   };
 }
 
-
-export type DatasetFolderInfo = {
-  dbscheme: string;
-  qlpack: string;
+export interface QlPacksForLanguage {
+  /** The name of the pack containing the dbscheme. */
+  dbschemePack: string;
+  /** `true` if `dbschemePack` is a library pack. */
+  dbschemePackIsLibraryPack: boolean;
+  /**
+   * The name of the corresponding standard query pack.
+   * Only defined if `dbschemePack` is a library pack.
+   */
+  queryPack?: string;
 }
 
-export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
-  const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
-  const packs: { packDir: string | undefined; packName: string }[] =
-    Object.entries(qlpacks).map(([packName, dirs]) => {
-      if (dirs.length < 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
-        return { packName, packDir: undefined };
-      }
-      if (dirs.length > 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
-      }
-      return {
-        packName,
-        packDir: dirs[0]
-      };
-    });
+interface QlPackWithPath {
+  packName: string;
+  packDir: string | undefined;
+}
+
+async function findDbschemePack(packs: QlPackWithPath[], dbschemePath: string): Promise<{ name: string; isLibraryPack: boolean; }> {
   for (const { packDir, packName } of packs) {
     if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme: string };
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
       if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
-        return packName;
+        return {
+          name: packName,
+          isLibraryPack: qlpack.library === true
+        };
       }
     }
   }
   throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
 }
 
-export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFolder: string): Promise<DatasetFolderInfo> {
+function findStandardQueryPack(qlpacks: QlpacksInfo, dbschemePackName: string): string | undefined {
+  const matches = dbschemePackName.match(/^codeql\/(?<language>[a-z]+)-all$/);
+  if (matches) {
+    const queryPackName = `codeql/${matches.groups!.language}-queries`;
+    if (qlpacks[queryPackName] !== undefined) {
+      return queryPackName;
+    }
+  }
+
+  // Either the dbscheme pack didn't look like one where the queries might be in the query pack, or
+  // no query pack was found in the search path. Either is OK.
+  return undefined;
+}
+
+export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<QlPacksForLanguage> {
+  const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
+  const packs: QlPackWithPath[] =
+    Object.entries(qlpacks).map(([packName, dirs]) => {
+      if (dirs.length < 1) {
+        void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
+        return { packName, packDir: undefined };
+      }
+      if (dirs.length > 1) {
+        void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
+      }
+      return {
+        packName,
+        packDir: dirs[0]
+      };
+    });
+  const dbschemePack = await findDbschemePack(packs, dbschemePath);
+  const queryPack = dbschemePack.isLibraryPack ? findStandardQueryPack(qlpacks, dbschemePack.name) : undefined;
+  return {
+    dbschemePack: dbschemePack.name,
+    dbschemePackIsLibraryPack: dbschemePack.isLibraryPack,
+    queryPack
+  };
+}
+
+export async function getPrimaryDbscheme(datasetFolder: string): Promise<string> {
   const dbschemes = await glob(path.join(datasetFolder, '*.dbscheme'));
 
   if (dbschemes.length < 1) {
@@ -400,31 +336,30 @@ export async function resolveDatasetFolder(cliServer: CodeQLCliServer, datasetFo
 
   dbschemes.sort();
   const dbscheme = dbschemes[0];
-  if (dbschemes.length > 1) {
-    Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
-  }
 
-  const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
-  return { dbscheme, qlpack };
+  if (dbschemes.length > 1) {
+    void Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
+  }
+  return dbscheme;
 }
 
 /**
  * A cached mapping from strings to value of type U.
  */
 export class CachedOperation<U> {
-  private readonly operation: (t: string) => Promise<U>;
+  private readonly operation: (t: string, ...args: any[]) => Promise<U>;
   private readonly cached: Map<string, U>;
   private readonly lru: string[];
   private readonly inProgressCallbacks: Map<string, [(u: U) => void, (reason?: any) => void][]>;
 
-  constructor(operation: (t: string) => Promise<U>, private cacheSize = 100) {
+  constructor(operation: (t: string, ...args: any[]) => Promise<U>, private cacheSize = 100) {
     this.operation = operation;
     this.lru = [];
     this.inProgressCallbacks = new Map<string, [(u: U) => void, (reason?: any) => void][]>();
     this.cached = new Map<string, U>();
   }
 
-  async get(t: string): Promise<U> {
+  async get(t: string, ...args: any[]): Promise<U> {
     // Try and retrieve from the cache
     const fromCache = this.cached.get(t);
     if (fromCache !== undefined) {
@@ -445,7 +380,7 @@ export class CachedOperation<U> {
     const callbacks: [(u: U) => void, (reason?: any) => void][] = [];
     this.inProgressCallbacks.set(t, callbacks);
     try {
-      const result = await this.operation(t);
+      const result = await this.operation(t, ...args);
       callbacks.forEach(f => f[0](result));
       this.inProgressCallbacks.delete(t);
       if (this.lru.length > this.cacheSize) {
@@ -464,3 +399,137 @@ export class CachedOperation<U> {
     }
   }
 }
+
+
+
+/**
+ * The following functions al heuristically determine metadata about databases.
+ */
+
+/**
+ * Note that this heuristic is only being used for backwards compatibility with
+ * CLI versions before the langauge name was introduced to dbInfo. Features
+ * that do not require backwards compatibility should call
+ * `cli.CodeQLCliServer.resolveDatabase` and use the first entry in the
+ * `languages` property.
+ *
+ * @see cli.CliVersionConstraint.supportsLanguageName
+ * @see cli.CodeQLCliServer.resolveDatabase
+ */
+export const dbSchemeToLanguage = {
+  'semmlecode.javascript.dbscheme': 'javascript',
+  'semmlecode.cpp.dbscheme': 'cpp',
+  'semmlecode.dbscheme': 'java',
+  'semmlecode.python.dbscheme': 'python',
+  'semmlecode.csharp.dbscheme': 'csharp',
+  'go.dbscheme': 'go',
+  'ruby.dbscheme': 'ruby'
+};
+
+export const languageToDbScheme = Object.entries(dbSchemeToLanguage).reduce((acc, [k, v]) => {
+  acc[v] = k;
+  return acc;
+}, {} as { [k: string]: string });
+
+
+/**
+ * Returns the initial contents for an empty query, based on the language of the selected
+ * databse.
+ *
+ * First try to use the given language name. If that doesn't exist, try to infer it based on
+ * dbscheme. Otherwise return no import statement.
+ *
+ * @param language the database language or empty string if unknown
+ * @param dbscheme path to the dbscheme file
+ *
+ * @returns an import and empty select statement appropriate for the selected language
+ */
+export function getInitialQueryContents(language: string, dbscheme: string) {
+  if (!language) {
+    const dbschemeBase = path.basename(dbscheme) as keyof typeof dbSchemeToLanguage;
+    language = dbSchemeToLanguage[dbschemeBase];
+  }
+
+  return language
+    ? `import ${language}\n\nselect ""`
+    : 'select ""';
+}
+
+/**
+ * Heuristically determines if the directory passed in corresponds
+ * to a database root.
+ *
+ * @param maybeRoot
+ */
+export async function isLikelyDatabaseRoot(maybeRoot: string) {
+  const [a, b, c] = (await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    fs.pathExists(path.join(maybeRoot, '.dbinfo')),
+    fs.pathExists(path.join(maybeRoot, 'codeql-database.yml')),
+
+    // they *must* have a db-{language} folder
+    glob('db-*/', { cwd: maybeRoot })
+  ]));
+
+  return !!((a || b) && c);
+}
+
+export function isLikelyDbLanguageFolder(dbPath: string) {
+  return !!path.basename(dbPath).startsWith('db-');
+}
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined
+): Promise<string | undefined> {
+  const uri = queryUri || Window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(getOnDiskWorkspaceFolders(), uri);
+      const language = (Object.keys(queryInfo.byLanguage))[0];
+      void logger.log(`Detected query language: ${language}`);
+      return language;
+    } catch (e) {
+      void logger.log('Could not autodetect query language. Select language manually.');
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(cliServer: CodeQLCliServer, throwOnEmpty = true): Promise<string | undefined> {
+  const language = await Window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    { placeHolder: 'Select target language for your query', ignoreFocusOut: true }
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException('Cancelled.');
+    } else {
+      void showAndLogErrorMessage('Language not found. Language must be specified manually.');
+    }
+  }
+  return language;
+}
+
+/**
+ * Gets metadata for a query, if it exists.
+ * @param cliServer The CLI server.
+ * @param queryPath The path to the query.
+ * @returns A promise that resolves to the query metadata, if available.
+ */
+export async function tryGetQueryMetadata(cliServer: CodeQLCliServer, queryPath: string): Promise<QueryMetadata | undefined> {
+  try {
+    return await cliServer.resolveMetadata(queryPath);
+  } catch (e) {
+    // Ignore errors and provide no metadata.
+    void logger.log(`Couldn't resolve metadata for ${queryPath}: ${e}`);
+    return;
+  }
+}

extensions/ql-vscode/src/ide-server.ts

@@ -12,7 +12,7 @@ import { ideServerLogger } from './logging';
 export async function spawnIdeServer(config: QueryServerConfig): Promise<StreamInfo> {
   return window.withProgress({ title: 'CodeQL language server', location: ProgressLocation.Window }, async (progressReporter, _) => {
     const args = ['--check-errors', 'ON_CHANGE'];
-    if (shouldDebug()) {
+    if (cli.shouldDebugIdeServer()) {
       args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9009,server=y,suspend=n,quiet=y');
     }
     const child = cli.spawnServer(
@@ -28,9 +28,3 @@ export async function spawnIdeServer(config: QueryServerConfig): Promise<StreamI
     return { writer: child.stdin!, reader: child.stdout! };
   });
 }
-
-function shouldDebug() {
-  return 'DEBUG_LANGUAGE_SERVER' in process.env
-    && process.env.DEBUG_LANGUAGE_SERVER !== '0'
-    && process.env.DEBUG_LANGUAGE_SERVER?.toLocaleLowerCase() !== 'false';
-}

extensions/ql-vscode/src/interface-utils.ts

@@ -15,11 +15,16 @@ import {
 import {
   tryGetResolvableLocation,
   isLineColumnLoc
-} from './bqrs-utils';
+} from './pure/bqrs-utils';
 import { DatabaseItem, DatabaseManager } from './databases';
-import { ViewSourceFileMsg } from './interface-types';
+import { ViewSourceFileMsg } from './pure/interface-types';
 import { Logger } from './logging';
-import { LineColumnLocation, WholeFileLocation, UrlValue, ResolvableLocationValue } from './bqrs-cli-types';
+import {
+  LineColumnLocation,
+  WholeFileLocation,
+  UrlValue,
+  ResolvableLocationValue
+} from './pure/bqrs-cli-types';
 
 /**
  * This module contains functions and types that are sharedd between
@@ -65,7 +70,7 @@ function resolveFivePartLocation(
     Math.max(0, loc.startLine - 1),
     Math.max(0, loc.startColumn - 1),
     Math.max(0, loc.endLine - 1),
-    Math.max(0, loc.endColumn)
+    Math.max(1, loc.endColumn)
   );
 
   return new Location(databaseItem.resolveSourceFile(loc.uri), range);
@@ -162,7 +167,13 @@ export async function showLocation(location?: Location) {
   const editor =
     editorsWithDoc.length > 0
       ? editorsWithDoc[0]
-      : await Window.showTextDocument(doc, ViewColumn.One);
+      : await Window.showTextDocument(
+        doc, {
+        // avoid preview mode so editor is sticky and will be added to navigation and search histories.
+        preview: false,
+        viewColumn: ViewColumn.One,
+      });
+
   const range = location.range;
   // When highlighting the range, vscode's occurrence-match and bracket-match highlighting will
   // trigger based on where we place the cursor/selection, and will compete for the user's attention.
@@ -213,14 +224,14 @@ export async function jumpToLocation(
     } catch (e) {
       if (e instanceof Error) {
         if (e.message.match(/File not found/)) {
-          Window.showErrorMessage(
+          void Window.showErrorMessage(
             'Original file of this result is not in the database\'s source archive.'
           );
         } else {
-          logger.log(`Unable to handleMsgFromView: ${e.message}`);
+          void logger.log(`Unable to handleMsgFromView: ${e.message}`);
         }
       } else {
-        logger.log(`Unable to handleMsgFromView: ${e}`);
+        void logger.log(`Unable to handleMsgFromView: ${e}`);
       }
     }
   }

extensions/ql-vscode/src/interface.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as Sarif from 'sarif';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as vscode from 'vscode';
 import {
   Diagnostic,
@@ -15,7 +15,7 @@ import * as cli from './cli';
 import { CodeQLCliServer } from './cli';
 import { DatabaseEventKind, DatabaseItem, DatabaseManager } from './databases';
 import { showAndLogErrorMessage } from './helpers';
-import { assertNever } from './helpers-pure';
+import { assertNever } from './pure/helpers-pure';
 import {
   FromResultsViewMsg,
   Interpretation,
@@ -26,17 +26,15 @@ import {
   SortedResultsMap,
   InterpretedResultsSortState,
   SortDirection,
-  RAW_RESULTS_PAGE_SIZE,
-  INTERPRETED_RESULTS_PAGE_SIZE,
   ALERTS_TABLE_NAME,
   RawResultsSortState,
-} from './interface-types';
+} from './pure/interface-types';
 import { Logger } from './logging';
-import { commandRunner } from './helpers';
-import * as messages from './messages';
+import * as messages from './pure/messages';
+import { commandRunner } from './commandRunner';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
-import { parseSarifLocation, parseSarifPlainTextMessage } from './sarif-utils';
+import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
 import {
   WebviewReveal,
   fileUriToWebviewUri,
@@ -46,8 +44,9 @@ import {
   shownLocationLineDecoration,
   jumpToLocation,
 } from './interface-utils';
-import { getDefaultResultSetName, ParsedResultSets } from './interface-types';
-import { RawResultSet, transformBqrsResultSet, ResultSetSchema } from './bqrs-cli-types';
+import { getDefaultResultSetName, ParsedResultSets } from './pure/interface-types';
+import { RawResultSet, transformBqrsResultSet, ResultSetSchema } from './pure/bqrs-cli-types';
+import { PAGE_SIZE } from './config';
 
 /**
  * interface.ts
@@ -89,11 +88,11 @@ function sortInterpretedResults(
 }
 
 function numPagesOfResultSet(resultSet: RawResultSet): number {
-  return Math.ceil(resultSet.schema.rows / RAW_RESULTS_PAGE_SIZE);
+  return Math.ceil(resultSet.schema.rows / PAGE_SIZE.getValue<number>());
 }
 
 function numInterpretedPages(interpretation: Interpretation | undefined): number {
-  return Math.ceil((interpretation?.sarif.runs[0].results?.length || 0) / INTERPRETED_RESULTS_PAGE_SIZE);
+  return Math.ceil((interpretation?.sarif.runs[0].results?.length || 0) / PAGE_SIZE.getValue<number>());
 }
 
 export class InterfaceManager extends DisposableObject {
@@ -120,7 +119,7 @@ export class InterfaceManager extends DisposableObject {
         this.handleSelectionChange.bind(this)
       )
     );
-    logger.log('Registering path-step navigation commands.');
+    void logger.log('Registering path-step navigation commands.');
     this.push(
       commandRunner(
         'codeQLQueryResults.nextPathStep',
@@ -138,16 +137,22 @@ export class InterfaceManager extends DisposableObject {
       this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
         if (kind === DatabaseEventKind.Remove) {
           this._diagnosticCollection.clear();
-          this.postMessage({
-            t: 'untoggleShowProblems'
-          });
+          if (this.isShowingPanel()) {
+            void this.postMessage({
+              t: 'untoggleShowProblems'
+            });
+          }
         }
       })
     );
   }
 
   async navigatePathStep(direction: number): Promise<void> {
-    this.postMessage({ t: 'navigatePath', direction });
+    await this.postMessage({ t: 'navigatePath', direction });
+  }
+
+  private isShowingPanel() {
+    return !!this._panel;
   }
 
   // Returns the webview panel, creating it if it doesn't already
@@ -155,10 +160,11 @@ export class InterfaceManager extends DisposableObject {
   getPanel(): vscode.WebviewPanel {
     if (this._panel == undefined) {
       const { ctx } = this;
+      const webViewColumn = this.chooseColumnForWebview();
       const panel = (this._panel = Window.createWebviewPanel(
         'resultsView', // internal name
         'CodeQL Query Results', // user-visible name
-        { viewColumn: vscode.ViewColumn.Beside, preserveFocus: true },
+        { viewColumn: webViewColumn, preserveFocus: true },
         {
           enableScripts: true,
           enableFindWidget: true,
@@ -169,6 +175,7 @@ export class InterfaceManager extends DisposableObject {
           ]
         }
       ));
+
       this._panel.onDidDispose(
         () => {
           this._panel = undefined;
@@ -181,7 +188,7 @@ export class InterfaceManager extends DisposableObject {
         ctx.asAbsolutePath('out/resultsView.js')
       );
       const stylesheetPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath('out/resultsView.css')
+        ctx.asAbsolutePath('out/view/resultsView.css')
       );
       panel.webview.html = getHtmlForWebview(
         panel.webview,
@@ -197,18 +204,41 @@ export class InterfaceManager extends DisposableObject {
     return this._panel;
   }
 
+  /**
+   * Choose where to open the webview.
+   *
+   * If there is a single view column, then open beside it.
+   * If there are multiple view columns, then open beside the active column,
+   * unless the active editor is the last column. In this case, open in the first column.
+   *
+   * The goal is to avoid opening new columns when there already are two columns open.
+   */
+  private chooseColumnForWebview(): vscode.ViewColumn {
+    // This is not a great way to determine the number of view columns, but I
+    // can't find a vscode API that does it any better.
+    // Here, iterate through all the visible editors and determine the max view column.
+    // This won't work if the largest view column is empty.
+    const colCount = Window.visibleTextEditors.reduce((maxVal, editor) =>
+      Math.max(maxVal, Number.parseInt(editor.viewColumn?.toFixed() || '0', 10)), 0);
+    if (colCount <= 1) {
+      return vscode.ViewColumn.Beside;
+    }
+    const activeViewColumnNum = Number.parseInt(Window.activeTextEditor?.viewColumn?.toFixed() || '0', 10);
+    return activeViewColumnNum === colCount ? vscode.ViewColumn.One : vscode.ViewColumn.Beside;
+  }
+
   private async changeInterpretedSortState(
     sortState: InterpretedResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         'Failed to sort results since evaluation info was unknown.'
       );
       return;
     }
     // Notify the webview that it should expect new results.
     await this.postMessage({ t: 'resultsUpdating' });
-    this._displayedQuery.updateInterpretedSortState(sortState);
+    await this._displayedQuery.updateInterpretedSortState(sortState);
     await this.showResults(this._displayedQuery, WebviewReveal.NotForced, true);
   }
 
@@ -217,7 +247,7 @@ export class InterfaceManager extends DisposableObject {
     sortState: RawResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         'Failed to sort results since evaluation info was unknown.'
       );
       return;
@@ -237,58 +267,67 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
-    switch (msg.t) {
-      case 'viewSourceFile': {
-        await jumpToLocation(msg, this.databaseManager, this.logger);
-        break;
-      }
-      case 'toggleDiagnostics': {
-        if (msg.visible) {
-          const databaseItem = this.databaseManager.findDatabaseItem(
-            Uri.parse(msg.databaseUri)
-          );
-          if (databaseItem !== undefined) {
-            await this.showResultsAsDiagnostics(
-              msg.origResultsPaths,
-              msg.metadata,
-              databaseItem
+    try {
+      switch (msg.t) {
+        case 'viewSourceFile': {
+          await jumpToLocation(msg, this.databaseManager, this.logger);
+          break;
+        }
+        case 'toggleDiagnostics': {
+          if (msg.visible) {
+            const databaseItem = this.databaseManager.findDatabaseItem(
+              Uri.parse(msg.databaseUri)
+            );
+            if (databaseItem !== undefined) {
+              await this.showResultsAsDiagnostics(
+                msg.origResultsPaths,
+                msg.metadata,
+                databaseItem
+              );
+            }
+          } else {
+            // TODO: Only clear diagnostics on the same database.
+            this._diagnosticCollection.clear();
+          }
+          break;
+        }
+        case 'resultViewLoaded':
+          this._panelLoaded = true;
+          this._panelLoadedCallBacks.forEach((cb) => cb());
+          this._panelLoadedCallBacks = [];
+          break;
+        case 'changeSort':
+          await this.changeRawSortState(msg.resultSetName, msg.sortState);
+          break;
+        case 'changeInterpretedSort':
+          await this.changeInterpretedSortState(msg.sortState);
+          break;
+        case 'changePage':
+          if (msg.selectedTable === ALERTS_TABLE_NAME) {
+            await this.showPageOfInterpretedResults(msg.pageNumber);
+          }
+          else {
+            await this.showPageOfRawResults(
+              msg.selectedTable,
+              msg.pageNumber,
+              // When we are in an unsorted state, we guarantee that
+              // sortedResultsInfo doesn't have an entry for the current
+              // result set. Use this to determine whether or not we use
+              // the sorted bqrs file.
+              this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
             );
           }
-        } else {
-          // TODO: Only clear diagnostics on the same database.
-          this._diagnosticCollection.clear();
-        }
-        break;
+          break;
+        case 'openFile':
+          await this.openFile(msg.filePath);
+          break;
+        default:
+          assertNever(msg);
       }
-      case 'resultViewLoaded':
-        this._panelLoaded = true;
-        this._panelLoadedCallBacks.forEach((cb) => cb());
-        this._panelLoadedCallBacks = [];
-        break;
-      case 'changeSort':
-        await this.changeRawSortState(msg.resultSetName, msg.sortState);
-        break;
-      case 'changeInterpretedSort':
-        await this.changeInterpretedSortState(msg.sortState);
-        break;
-      case 'changePage':
-        if (msg.selectedTable === ALERTS_TABLE_NAME) {
-          await this.showPageOfInterpretedResults(msg.pageNumber);
-        }
-        else {
-          await this.showPageOfRawResults(
-            msg.selectedTable,
-            msg.pageNumber,
-            // When we are in an unsorted state, we guarantee that
-            // sortedResultsInfo doesn't have an entry for the current
-            // result set. Use this to determine whether or not we use
-            // the sorted bqrs file.
-            this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
-          );
-        }
-        break;
-      default:
-        assertNever(msg);
+    } catch (e) {
+      void showAndLogErrorMessage(e.message, {
+        fullMessage: e.stack
+      });
     }
   }
 
@@ -350,14 +389,13 @@ export class InterfaceManager extends DisposableObject {
       const showButton = 'View Results';
       const queryName = results.queryName;
       const resultPromise = vscode.window.showInformationMessage(
-        `Finished running query ${
-        queryName.length > 0 ? ` "${queryName}"` : ''
+        `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ''
         }.`,
         showButton
       );
       // Address this click asynchronously so we still update the
       // query history immediately.
-      resultPromise.then((result) => {
+      void resultPromise.then((result) => {
         if (result === showButton) {
           panel.reveal();
         }
@@ -378,7 +416,7 @@ export class InterfaceManager extends DisposableObject {
     // Use sorted results path if it exists. This may happen if we are
     // reloading the results view after it has been sorted in the past.
     const resultsPath = results.getResultsPath(selectedTable);
-
+    const pageSize = PAGE_SIZE.getValue<number>();
     const chunk = await this.cliServer.bqrsDecode(
       resultsPath,
       schema.name,
@@ -388,12 +426,14 @@ export class InterfaceManager extends DisposableObject {
         // if there are interpreted results, but speculatively
         // send anyway.
         offset: schema.pagination?.offsets[0],
-        pageSize: RAW_RESULTS_PAGE_SIZE
+        pageSize
       }
     );
     const resultSet = transformBqrsResultSet(schema, chunk);
+    results.setResultCount(interpretationPage?.numTotalResults || resultSet.schema.rows);
     const parsedResultSets: ParsedResultSets = {
       pageNumber: 0,
+      pageSize,
       numPages: numPagesOfResultSet(resultSet),
       numInterpretedPages: numInterpretedPages(this._interpretation),
       resultSet: { ...resultSet, t: 'RawResultSet' },
@@ -413,6 +453,8 @@ export class InterfaceManager extends DisposableObject {
       database: results.database,
       shouldKeepOldResultsWhileRendering,
       metadata: results.query.metadata,
+      queryName: results.toString(),
+      queryPath: results.query.program.queryPath
     });
   }
 
@@ -442,7 +484,10 @@ export class InterfaceManager extends DisposableObject {
       metadata: this._displayedQuery.query.metadata,
       pageNumber,
       resultSetNames,
+      pageSize: PAGE_SIZE.getValue(),
       numPages: numInterpretedPages(this._interpretation),
+      queryName: this._displayedQuery.toString(),
+      queryPath: this._displayedQuery.query.program.queryPath
     });
   }
 
@@ -450,11 +495,16 @@ export class InterfaceManager extends DisposableObject {
     const resultsPath = results.getResultsPath(selectedTable);
     const schemas = await this.cliServer.bqrsInfo(
       resultsPath,
-      RAW_RESULTS_PAGE_SIZE
+      PAGE_SIZE.getValue()
     );
     return schemas['result-sets'];
   }
 
+  public async openFile(filePath: string) {
+    const textDocument = await vscode.workspace.openTextDocument(filePath);
+    await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+  }
+
   /**
    * Show a page of raw results from the chosen table.
    */
@@ -475,7 +525,12 @@ export class InterfaceManager extends DisposableObject {
     );
 
     const resultSetSchemas = await this.getResultSetSchemas(results, sorted ? selectedTable : '');
-    const resultSetNames = resultSetSchemas.map(schema => schema.name);
+
+    // If there is a specific sorted table selected, a different bqrs file is loaded that doesn't have all the result set names.
+    // Make sure that we load all result set names here.
+    // See https://github.com/github/vscode-codeql/issues/1005
+    const allResultSetSchemas = sorted ? await this.getResultSetSchemas(results, '') : resultSetSchemas;
+    const resultSetNames = allResultSetSchemas.map(schema => schema.name);
 
     const schema = resultSetSchemas.find(
       (resultSet) => resultSet.name == selectedTable
@@ -483,18 +538,20 @@ export class InterfaceManager extends DisposableObject {
     if (schema === undefined)
       throw new Error(`Query result set '${selectedTable}' not found.`);
 
+    const pageSize = PAGE_SIZE.getValue<number>();
     const chunk = await this.cliServer.bqrsDecode(
       results.getResultsPath(selectedTable, sorted),
       schema.name,
       {
         offset: schema.pagination?.offsets[pageNumber],
-        pageSize: RAW_RESULTS_PAGE_SIZE
+        pageSize
       }
     );
     const resultSet = transformBqrsResultSet(schema, chunk);
 
     const parsedResultSets: ParsedResultSets = {
       pageNumber,
+      pageSize,
       resultSet: { t: 'RawResultSet', ...resultSet },
       numPages: numPagesOfResultSet(resultSet),
       numInterpretedPages: numInterpretedPages(this._interpretation),
@@ -514,6 +571,8 @@ export class InterfaceManager extends DisposableObject {
       database: results.database,
       shouldKeepOldResultsWhileRendering: false,
       metadata: results.query.metadata,
+      queryName: results.toString(),
+      queryPath: results.query.program.queryPath
     });
   }
 
@@ -523,23 +582,26 @@ export class InterfaceManager extends DisposableObject {
     sourceInfo: cli.SourceInfo | undefined,
     sourceLocationPrefix: string,
     sortState: InterpretedResultsSortState | undefined
-  ): Promise<Interpretation> {
+  ): Promise<Interpretation | undefined> {
+    if (!resultsPaths) {
+      void this.logger.log('No results path. Cannot display interpreted results.');
+      return undefined;
+    }
+
     const sarif = await interpretResults(
       this.cliServer,
       metadata,
       resultsPaths,
       sourceInfo
     );
+
     sarif.runs.forEach(run => {
-      if (run.results !== undefined)
+      if (run.results !== undefined) {
         sortInterpretedResults(run.results, sortState);
+      }
     });
 
-    const numTotalResults = (() => {
-      if (sarif.runs.length === 0) return 0;
-      if (sarif.runs[0].results === undefined) return 0;
-      return sarif.runs[0].results.length;
-    })();
+    const numTotalResults = sarif.runs[0]?.results?.length || 0;
 
     const interpretation: Interpretation = {
       sarif,
@@ -559,8 +621,8 @@ export class InterfaceManager extends DisposableObject {
     function getPageOfRun(run: Sarif.Run): Sarif.Run {
       return {
         ...run, results: run.results?.slice(
-          INTERPRETED_RESULTS_PAGE_SIZE * pageNumber,
-          INTERPRETED_RESULTS_PAGE_SIZE * (pageNumber + 1)
+          PAGE_SIZE.getValue<number>() * pageNumber,
+          PAGE_SIZE.getValue<number>() * (pageNumber + 1)
         )
       };
     }
@@ -569,7 +631,7 @@ export class InterfaceManager extends DisposableObject {
       throw new Error('Tried to get interpreted results before interpretation finished');
     }
     if (this._interpretation.sarif.runs.length !== 1) {
-      this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
+      void this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
     }
     const interp = this._interpretation;
     return {
@@ -608,8 +670,8 @@ export class InterfaceManager extends DisposableObject {
       } catch (e) {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
-        this.logger.log(
-          `Exception during results interpretation: ${e.message}. Will show raw results instead.`
+        void showAndLogErrorMessage(
+          `Showing raw results instead of interpreted ones due to an error. ${e.message}`
         );
       }
     }
@@ -641,11 +703,15 @@ export class InterfaceManager extends DisposableObject {
       undefined
     );
 
+    if (!interpretation) {
+      return;
+    }
+
     try {
       await this.showProblemResultsAsDiagnostics(interpretation, database);
     } catch (e) {
       const msg = e instanceof Error ? e.message : e.toString();
-      this.logger.log(
+      void this.logger.log(
         `Exception while computing problem results as diagnostics: ${msg}`
       );
       this._diagnosticCollection.clear();
@@ -659,7 +725,7 @@ export class InterfaceManager extends DisposableObject {
     const { sarif, sourceLocationPrefix } = interpretation;
 
     if (!sarif.runs || !sarif.runs[0].results) {
-      this.logger.log(
+      void this.logger.log(
         'Didn\'t find a run in the sarif results. Error processing sarif?'
       );
       return;
@@ -670,11 +736,11 @@ export class InterfaceManager extends DisposableObject {
     for (const result of sarif.runs[0].results) {
       const message = result.message.text;
       if (message === undefined) {
-        this.logger.log('Sarif had result without plaintext message');
+        void this.logger.log('Sarif had result without plaintext message');
         continue;
       }
       if (!result.locations) {
-        this.logger.log('Sarif had result without location');
+        void this.logger.log('Sarif had result without location');
         continue;
       }
 
@@ -687,7 +753,7 @@ export class InterfaceManager extends DisposableObject {
       }
       const resultLocation = tryResolveLocation(sarifLoc, databaseItem);
       if (!resultLocation) {
-        this.logger.log('Sarif location was not resolvable ' + sarifLoc);
+        void this.logger.log('Sarif location was not resolvable ' + sarifLoc);
         continue;
       }
       const parsedMessage = parseSarifPlainTextMessage(message);

extensions/ql-vscode/src/logging.ts

@@ -1,5 +1,5 @@
 import { window as Window, OutputChannel, Progress, Disposable } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -28,9 +28,16 @@ export interface Logger {
   removeAdditionalLogLocation(location: string | undefined): void;
 
   /**
-   * The base location location where all side log files are stored.
+   * The base location where all side log files are stored.
    */
   getBaseLocation(): string | undefined;
+
+  /**
+   * Sets the location where logs are stored.
+   * @param storagePath The path where logs are stored.
+   * @param isCustomLogDirectory Whether the logs are stored in a custom, user-specified directory.
+   */
+  setLogStoragePath(storagePath: string, isCustomLogDirectory: boolean): Promise<void>;
 }
 
 export type ProgressReporter = Progress<{ message: string }>;
@@ -40,18 +47,24 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   public readonly outputChannel: OutputChannel;
   private readonly additionalLocations = new Map<string, AdditionalLogLocation>();
   private additionalLogLocationPath: string | undefined;
+  isCustomLogDirectory: boolean;
 
   constructor(private title: string) {
     super();
     this.outputChannel = Window.createOutputChannel(title);
     this.push(this.outputChannel);
+    this.isCustomLogDirectory = false;
   }
 
-  init(storagePath: string): void {
+  async setLogStoragePath(storagePath: string, isCustomLogDirectory: boolean): Promise<void> {
     this.additionalLogLocationPath = path.join(storagePath, this.title);
 
-    // clear out any old state from previous runs
-    fs.remove(this.additionalLogLocationPath);
+    this.isCustomLogDirectory = isCustomLogDirectory;
+
+    if (!this.isCustomLogDirectory) {
+      // clear out any old state from previous runs
+      await fs.remove(this.additionalLogLocationPath);
+    }
   }
 
   /**
@@ -80,7 +93,7 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
         this.outputChannel.appendLine(separator);
         this.outputChannel.appendLine(msg);
         this.outputChannel.appendLine(separator);
-        additional = new AdditionalLogLocation(logPath);
+        additional = new AdditionalLogLocation(logPath, !this.isCustomLogDirectory);
         this.additionalLocations.set(logPath, additional);
         this.track(additional);
       }
@@ -112,7 +125,7 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
 }
 
 class AdditionalLogLocation extends Disposable {
-  constructor(private location: string) {
+  constructor(private location: string, private shouldDeleteLogs: boolean) {
     super(() => { /**/ });
   }
 
@@ -128,7 +141,9 @@ class AdditionalLogLocation extends Disposable {
   }
 
   async dispose(): Promise<void> {
-    await fs.remove(this.location);
+    if (this.shouldDeleteLogs) {
+      await fs.remove(this.location);
+    }
   }
 }
 

extensions/ql-vscode/src/pure/bqrs-cli-types.ts

@@ -77,10 +77,10 @@ export interface WholeFileLocation {
   endColumn: never;
 }
 
-export type UrlValue = LineColumnLocation | WholeFileLocation | string;
-
 export type ResolvableLocationValue = WholeFileLocation | LineColumnLocation;
 
+export type UrlValue = ResolvableLocationValue  | string;
+
 export type ColumnValue = EntityValue | number | string | boolean;
 
 export type ResultRow = ColumnValue[];

extensions/ql-vscode/src/pure/bqrs-utils.ts

@@ -1,4 +1,9 @@
-import { UrlValue, ResolvableLocationValue, LineColumnLocation, WholeFileLocation } from './bqrs-cli-types';
+import {
+  UrlValue,
+  ResolvableLocationValue,
+  LineColumnLocation,
+  WholeFileLocation
+} from './bqrs-cli-types';
 
 /**
  * The CodeQL filesystem libraries use this pattern in `getURL()` predicates
@@ -78,8 +83,7 @@ export function isLineColumnLoc(loc: UrlValue): loc is LineColumnLocation {
     && 'startLine' in loc
     && 'startColumn' in loc
     && 'endLine' in loc
-    && 'endColumn' in loc
-    && loc.endColumn > 0;
+    && 'endColumn' in loc;
 }
 
 export function isWholeFileLoc(loc: UrlValue): loc is WholeFileLocation {

extensions/ql-vscode/src/pure/disposable-object.ts

@@ -1,4 +1,11 @@
-import { Disposable } from 'vscode';
+
+// Avoid explicitly referencing Disposable type in vscode.
+// This file cannot have dependencies on the vscode API.
+interface Disposable {
+  dispose(): any;
+}
+
+export type DisposeHandler = (disposable: Disposable) => void;
 
 /**
  * Base class to make it easier to implement a `Disposable` that owns other disposable object.
@@ -40,21 +47,39 @@ export abstract class DisposableObject implements Disposable {
    * @param obj The object to stop tracking.
    */
   protected disposeAndStopTracking(obj: Disposable): void {
-    if (obj !== undefined) {
-      this.tracked!.delete(obj);
+    if (obj && this.tracked) {
+      this.tracked.delete(obj);
       obj.dispose();
     }
   }
 
-  public dispose() {
+  /**
+   * Dispose this object and all contained objects
+   *
+   * @param disposeHandler An optional dispose handler that gets
+   *      passed each element to dispose. The dispose handler
+   *      can choose how (and if) to dispose the object. The
+   *      primary usage is for tests that should not dispose
+   *      all items of a disposable.
+   */
+  public dispose(disposeHandler?: DisposeHandler) {
     if (this.tracked !== undefined) {
       for (const trackedObject of this.tracked.values()) {
-        trackedObject.dispose();
+        if (disposeHandler) {
+          disposeHandler(trackedObject);
+        } else {
+          trackedObject.dispose();
+        }
       }
       this.tracked = undefined;
     }
     while (this.disposables.length > 0) {
-      this.disposables.pop()!.dispose();
+      const disposable = this.disposables.pop()!;
+      if (disposeHandler) {
+        disposeHandler(disposable);
+      } else {
+        disposable.dispose();
+      }
     }
   }
 }

extensions/ql-vscode/src/pure/helpers-pure.ts

@@ -2,7 +2,7 @@
  * helpers-pure.ts
  * ------------
  *
- * Helper functions that don't depend on vscode and therefore can be used by the front-end and pure unit tests.
+ * Helper functions that don't depend on vscode or the CLI and therefore can be used by the front-end and pure unit tests.
  */
 
 /**
@@ -21,3 +21,11 @@ class ExhaustivityCheckingError extends Error {
 export function assertNever(value: never): never {
   throw new ExhaustivityCheckingError(value);
 }
+
+/**
+ * Use to perform array filters where the predicate is asynchronous.
+ */
+export const asyncFilter = async function <T>(arr: T[], predicate: (arg0: T) => Promise<boolean>) {
+  const results = await Promise.all(arr.map(predicate));
+  return arr.filter((_, index) => results[index]);
+};

extensions/ql-vscode/src/pure/interface-types.ts

@@ -1,4 +1,6 @@
 import * as sarif from 'sarif';
+import { DownloadLink } from '../remote-queries/download-link';
+import { RemoteQueryResult } from '../remote-queries/shared/remote-query-result';
 import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationValue } from './bqrs-cli-types';
 
 /**
@@ -23,16 +25,6 @@ export type ResultSet = RawTableResultSet | PathTableResultSet;
  */
 export const RAW_RESULTS_LIMIT = 10000;
 
-/**
- * Show this many rows in a raw result table at a time.
- */
-export const RAW_RESULTS_PAGE_SIZE = 100;
-
-/**
- * Show this many rows in an interpreted results table at a time.
- */
-export const INTERPRETED_RESULTS_PAGE_SIZE = 100;
-
 export interface DatabaseInfo {
   name: string;
   databaseUri: string;
@@ -44,6 +36,7 @@ export interface QueryMetadata {
   description?: string;
   id?: string;
   kind?: string;
+  scored?: string;
 }
 
 export interface PreviousExecution {
@@ -98,6 +91,8 @@ export interface SetStateMsg {
   interpretation: undefined | Interpretation;
   database: DatabaseInfo;
   metadata?: QueryMetadata;
+  queryName: string;
+  queryPath: string;
   /**
    * Whether to keep displaying the old results while rendering the new results.
    *
@@ -124,7 +119,10 @@ export interface ShowInterpretedPageMsg {
   metadata?: QueryMetadata;
   pageNumber: number;
   numPages: number;
+  pageSize: number;
   resultSetNames: string[];
+  queryName: string;
+  queryPath: string;
 }
 
 /** Advance to the next or previous path no in the path viewer */
@@ -162,7 +160,8 @@ export type FromResultsViewMsg =
   | ChangeRawResultsSortMsg
   | ChangeInterpretedResultsSortMsg
   | ResultViewLoaded
-  | ChangePage;
+  | ChangePage
+  | OpenFileMsg;
 
 /**
  * Message from the results view to open a database source
@@ -174,6 +173,19 @@ export interface ViewSourceFileMsg {
   databaseUri: string;
 }
 
+/**
+ * Message from the results view to open a file in an editor.
+ */
+export interface OpenFileMsg {
+  t: 'openFile';
+  /* Full path to the file to open. */
+  filePath: string;
+}
+
+export interface OpenVirtualFileMsg {
+  t: 'openVirtualFile';
+  queryText: string;
+}
 
 /**
  * Message from the results view to toggle the display of
@@ -352,9 +364,39 @@ export function getDefaultResultSetName(
 
 export interface ParsedResultSets {
   pageNumber: number;
+  pageSize: number;
   numPages: number;
   numInterpretedPages: number;
   selectedTable?: string; // when undefined, means 'show default table'
   resultSetNames: string[];
   resultSet: ResultSet;
 }
+
+export type FromRemoteQueriesMessage =
+  | RemoteQueryLoadedMessage
+  | RemoteQueryErrorMessage
+  | OpenFileMsg
+  | OpenVirtualFileMsg
+  | RemoteQueryDownloadLinkClickedMessage;
+
+export type ToRemoteQueriesMessage =
+  | SetRemoteQueryResultMessage;
+
+export interface RemoteQueryLoadedMessage {
+  t: 'remoteQueryLoaded';
+}
+
+export interface SetRemoteQueryResultMessage {
+  t: 'setRemoteQueryResult';
+  queryResult: RemoteQueryResult
+}
+
+export interface RemoteQueryErrorMessage {
+  t: 'remoteQueryError';
+  error: string;
+}
+
+export interface RemoteQueryDownloadLinkClickedMessage {
+  t: 'remoteQueryDownloadLinkClicked';
+  downloadLink: DownloadLink;
+}

extensions/ql-vscode/src/pure/messages.ts

@@ -150,6 +150,11 @@ export interface CompilationOptions {
    * Whether to disable toString values in the results.
    */
   noComputeToString: boolean;
+  /**
+   * Whether to ensure that elements that do not have a displayString
+   * get reported anyway. Useful for universal compilation options.
+   */
+  computeDefaultStrings: boolean;
 }
 
 /**
@@ -257,7 +262,7 @@ export interface CompilationTarget {
   /**
    * Compile as a normal query
    */
-  query?: {};
+  query?: Record<string, never>;
   /**
    * Compile as a quick evaluation
    */
@@ -380,8 +385,8 @@ export namespace ResultColumnKind {
    */
   export const BOOLEAN = 3;
   /**
- * A column of type `date`
- */
+   * A column of type `date`
+   */
   export const DATE = 4;
   /**
    * A column of a non-primitive type
@@ -401,6 +406,25 @@ export interface CompileUpgradeParams {
    * A directory to store parts of the compiled upgrade
    */
   upgradeTempDir: string;
+  /**
+   * Enable single file upgrades, set to true to allow
+   * using single file upgrades.
+   */
+  singleFileUpgrades: true;
+}
+
+/**
+ * Parameters for compiling an upgrade.
+ */
+export interface CompileUpgradeSequenceParams {
+  /**
+   * The sequence of upgrades to compile
+   */
+  upgradePaths: string[];
+  /**
+   * A directory to store parts of the compiled upgrade
+   */
+  upgradeTempDir: string;
 }
 
 /**
@@ -450,6 +474,19 @@ export interface CompileUpgradeResult {
    */
   error?: string;
 }
+
+export interface CompileUpgradeSequenceResult {
+  /**
+   * The compiled upgrades as a single file.
+   */
+  compiledUpgrade?: string;
+  /**
+   * Any errors that occurred when checking the scripts.
+   */
+  error?: string;
+}
+
+
 /**
  * A description of a upgrade process
  */
@@ -487,10 +524,13 @@ export interface UpgradeDescription {
   newSha: string;
 }
 
+
+export type CompiledUpgrades = MultiFileCompiledUpgrades | SingleFileCompiledUpgrades
+
 /**
- * A compiled upgrade.
+ * The parts shared by all compiled upgrades
  */
-export interface CompiledUpgrades {
+interface CompiledUpgradesBase {
   /**
   * The initial sha of the dbscheme to upgrade from
   */
@@ -499,14 +539,46 @@ export interface CompiledUpgrades {
    * The path to the new dataset statistics
    */
   newStatsPath: string;
+  /**
+   * The sha of the target dataset.
+   */
+  targetSha: string;
+}
+
+
+/**
+ * A compiled upgrade.
+ * The upgrade is spread among multiple files.
+ */
+interface MultiFileCompiledUpgrades extends CompiledUpgradesBase {
+  /**
+   * The path to the new dataset dbscheme
+   */
+  newDbscheme: string;
   /**
    * The steps in the upgrade path
    */
   scripts: CompiledUpgradeScript[];
   /**
-   * The sha of the target dataset.
+   * Will never exist in an old result
    */
-  targetSha: string;
+  compiledUpgradeFile?: never;
+}
+
+
+/**
+ * A compiled upgrade.
+ * The upgrade is in a single file.
+ */
+export interface SingleFileCompiledUpgrades extends CompiledUpgradesBase {
+  /**
+   * The steps in the upgrade path
+   */
+  descriptions: UpgradeDescription[];
+  /**
+   * A path to a file containing the upgrade
+   */
+  compiledUpgradeFile: string;
 }
 
 /**
@@ -639,6 +711,11 @@ export interface EvaluateQueriesParams {
 
 export type TemplateDefinitions = { [key: string]: TemplateSource }
 
+export interface MlModel {
+  /** A URI pointing to the root directory of the model. */
+  uri: string;
+}
+
 /**
  * A single query that should be run
  */
@@ -651,6 +728,10 @@ export interface QueryToRun {
    * A uri pointing to the qlo to run.
    */
   qlo: string;
+  /**
+   * A uri pointing to the compiled upgrade file.
+   */
+  compiledUpgrade?: string;
   /**
    * The path where we should save this queries results
    */
@@ -668,6 +749,11 @@ export interface QueryToRun {
    * map should be set to the empty set or give an error.
    */
   allowUnknownTemplates: boolean;
+  /**
+   * The list of ML models that should be made available
+   * when evaluating the query.
+   */
+  availableMlModels?: MlModel[];
 }
 
 /**
@@ -750,7 +836,7 @@ export interface ResultSet {
 /**
  * The type returned when the evaluation is complete
  */
-export type EvaluationComplete = {};
+export type EvaluationComplete = Record<string, never>;
 
 /**
  * The result of a single query
@@ -837,7 +923,6 @@ export interface RunUpgradeParams {
   toRun: CompiledUpgrades;
 }
 
-
 /**
  * The result of running an upgrade
  */
@@ -857,6 +942,21 @@ export interface RunUpgradeResult {
   finalSha: string;
 }
 
+export interface RegisterDatabasesParams {
+  databases: Dataset[];
+}
+
+export interface DeregisterDatabasesParams {
+  databases: Dataset[];
+}
+
+export type RegisterDatabasesResult = {
+  registeredDatabases: Dataset[];
+};
+
+export type DeregisterDatabasesResult = {
+  registeredDatabases: Dataset[];
+};
 
 /**
  * Type for any action that could have progress messages.
@@ -913,7 +1013,10 @@ export const checkUpgrade = new rpc.RequestType<WithProgressId<UpgradeParams>, C
  * Compile an upgrade script to upgrade a dataset.
  */
 export const compileUpgrade = new rpc.RequestType<WithProgressId<CompileUpgradeParams>, CompileUpgradeResult, void, void>('compilation/compileUpgrade');
-
+/**
+ * Compile an upgrade script to upgrade a dataset.
+ */
+export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<CompileUpgradeSequenceParams>, CompileUpgradeSequenceResult, void, void>('compilation/compileUpgradeSequence');
 
 /**
  * Clear the cache of a dataset
@@ -934,6 +1037,20 @@ export const runQueries = new rpc.RequestType<WithProgressId<EvaluateQueriesPara
  */
 export const runUpgrade = new rpc.RequestType<WithProgressId<RunUpgradeParams>, RunUpgradeResult, void, void>('evaluation/runUpgrade');
 
+export const registerDatabases = new rpc.RequestType<
+  WithProgressId<RegisterDatabasesParams>,
+  RegisterDatabasesResult,
+  void,
+  void
+>('evaluation/registerDatabases');
+
+export const deregisterDatabases = new rpc.RequestType<
+  WithProgressId<DeregisterDatabasesParams>,
+  DeregisterDatabasesResult,
+  void,
+  void
+>('evaluation/deregisterDatabases');
+
 /**
  * Request returned to the client to notify completion of a query.
  * The full runQueries job is completed when all queries are acknowledged.

extensions/ql-vscode/src/pure/sarif-utils.ts

@@ -72,10 +72,28 @@ export function getPathRelativeToSourceLocationPrefix(
   sourceLocationPrefix: string,
   sarifRelativeUri: string
 ) {
-  const normalizedSourceLocationPrefix = sourceLocationPrefix.replace(/\\/g, '/');
-  return `file:${normalizedSourceLocationPrefix}/${sarifRelativeUri}`;
+  // convert a platform specific path into encoded path uri segments
+  // need to be careful about drive letters and ensure that there
+  // is a starting '/'
+  let prefix = '';
+  if (sourceLocationPrefix[1] === ':') {
+    // assume this is a windows drive separator
+    prefix = sourceLocationPrefix.substring(0, 2);
+    sourceLocationPrefix = sourceLocationPrefix.substring(2);
+  }
+  const normalizedSourceLocationPrefix = prefix + sourceLocationPrefix.replace(/\\/g, '/')
+    .split('/')
+    .map(encodeURIComponent)
+    .join('/');
+  const slashPrefix = normalizedSourceLocationPrefix.startsWith('/') ? '' : '/';
+  return `file:${slashPrefix + normalizedSourceLocationPrefix}/${sarifRelativeUri}`;
 }
 
+/**
+ *
+ * @param loc specifies the database-relative location of the source location
+ * @param sourceLocationPrefix a file path (usually a full path) to the database containing the source location.
+ */
 export function parseSarifLocation(
   loc: Sarif.Location,
   sourceLocationPrefix: string

extensions/ql-vscode/src/qlpack-discovery.ts

@@ -1,58 +0,0 @@
-import { EventEmitter, Event, Uri, WorkspaceFolder, RelativePattern } from 'vscode';
-import { MultiFileSystemWatcher } from './vscode-utils/multi-file-system-watcher';
-import { CodeQLCliServer, QlpacksInfo } from './cli';
-import { Discovery } from './discovery';
-
-export interface QLPack {
-  name: string;
-  uri: Uri;
-}
-
-/**
- * Service to discover all available QL packs in a workspace folder.
- */
-export class QLPackDiscovery extends Discovery<QlpacksInfo> {
-  private readonly _onDidChangeQLPacks = this.push(new EventEmitter<void>());
-  private readonly watcher = this.push(new MultiFileSystemWatcher());
-  private _qlPacks: readonly QLPack[] = [];
-
-  constructor(
-    private readonly workspaceFolder: WorkspaceFolder,
-    private readonly cliServer: CodeQLCliServer
-  ) {
-    super('QL Pack Discovery');
-
-    // Watch for any changes to `qlpack.yml` files in this workspace folder.
-    // TODO: The CLI server should tell us what paths to watch for.
-    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/qlpack.yml'));
-    this.watcher.addWatch(new RelativePattern(this.workspaceFolder, '**/.codeqlmanifest.json'));
-    this.push(this.watcher.onDidChange(this.handleQLPackFileChanged, this));
-  }
-
-  public get onDidChangeQLPacks(): Event<void> { return this._onDidChangeQLPacks.event; }
-
-  public get qlPacks(): readonly QLPack[] { return this._qlPacks; }
-
-  private handleQLPackFileChanged(_uri: Uri): void {
-    this.refresh();
-  }
-
-  protected discover(): Promise<QlpacksInfo> {
-    // Only look for QL packs in this workspace folder.
-    return this.cliServer.resolveQlpacks([this.workspaceFolder.uri.fsPath], []);
-  }
-
-  protected update(results: QlpacksInfo): void {
-    const qlPacks: QLPack[] = [];
-    for (const id in results) {
-      qlPacks.push(...results[id].map(fsPath => {
-        return {
-          name: id,
-          uri: Uri.file(fsPath)
-        };
-      }));
-    }
-    this._qlPacks = qlPacks;
-    this._onDidChangeQLPacks.fire();
-  }
-}

extensions/ql-vscode/src/qltest-discovery.ts

@@ -1,9 +1,9 @@
 import * as path from 'path';
-import { QLPackDiscovery, QLPack } from './qlpack-discovery';
 import { Discovery } from './discovery';
-import { EventEmitter, Event, Uri, RelativePattern, WorkspaceFolder, env, workspace } from 'vscode';
+import { EventEmitter, Event, Uri, RelativePattern, WorkspaceFolder, env } from 'vscode';
 import { MultiFileSystemWatcher } from './vscode-utils/multi-file-system-watcher';
 import { CodeQLCliServer } from './cli';
+import * as fs from 'fs-extra';
 
 /**
  * A node in the tree of tests. This will be either a `QLTestDirectory` or a `QLTestFile`.
@@ -29,9 +29,8 @@ export abstract class QLTestNode {
  * A directory containing one or more QL tests or other test directories.
  */
 export class QLTestDirectory extends QLTestNode {
-  private _children: QLTestNode[] = [];
 
-  constructor(_path: string, _name: string) {
+  constructor(_path: string, _name: string, private _children: QLTestNode[] = []) {
     super(_path, _name);
   }
 
@@ -55,10 +54,23 @@ export class QLTestDirectory extends QLTestNode {
   }
 
   public finish(): void {
+    // remove empty directories
+    this._children.filter(child =>
+      child instanceof QLTestFile || child.children.length > 0
+    );
     this._children.sort((a, b) => a.name.localeCompare(b.name, env.language));
-    for (const child of this._children) {
+    this._children.forEach((child, i) => {
       child.finish();
-    }
+      if (child.children?.length === 1 && child.children[0] instanceof QLTestDirectory) {
+        // collapse children
+        const replacement = new QLTestDirectory(
+          child.children[0].path,
+          child.name + ' / ' + child.children[0].name,
+          Array.from(child.children[0].children)
+        );
+        this._children[i] = replacement;
+      }
+    });
   }
 
   private createChildDirectory(name: string): QLTestDirectory {
@@ -96,14 +108,15 @@ export class QLTestFile extends QLTestNode {
  */
 interface QLTestDiscoveryResults {
   /**
-   * The root test directory for each QL pack that contains tests.
+   * A directory that contains one or more QL Tests, or other QLTestDirectories.
    */
-  testDirectories: QLTestDirectory[];
+  testDirectory: QLTestDirectory | undefined;
+
   /**
-   * The list of file system paths to watch. If any of these paths changes, the discovery results
-   * may be out of date.
+   * The file system path to a directory to watch. If any ql or qlref file changes in
+   * this directory, then this signifies a change in tests.
    */
-  watchPaths: string[];
+  watchPath: string;
 }
 
 /**
@@ -112,31 +125,30 @@ interface QLTestDiscoveryResults {
 export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
   private readonly _onDidChangeTests = this.push(new EventEmitter<void>());
   private readonly watcher: MultiFileSystemWatcher = this.push(new MultiFileSystemWatcher());
-  private _testDirectories: QLTestDirectory[] = [];
+  private _testDirectory: QLTestDirectory | undefined;
 
   constructor(
-    private readonly qlPackDiscovery: QLPackDiscovery,
     private readonly workspaceFolder: WorkspaceFolder,
     private readonly cliServer: CodeQLCliServer
   ) {
     super('QL Test Discovery');
 
-    this.push(this.qlPackDiscovery.onDidChangeQLPacks(this.handleDidChangeQLPacks, this));
     this.push(this.watcher.onDidChange(this.handleDidChange, this));
   }
 
   /**
    * Event to be fired when the set of discovered tests may have changed.
    */
-  public get onDidChangeTests(): Event<void> { return this._onDidChangeTests.event; }
+  public get onDidChangeTests(): Event<void> {
+    return this._onDidChangeTests.event;
+  }
 
   /**
-   * The root test directory for each QL pack that contains tests.
+   * The root directory. There is at least one test in this directory, or
+   * in a subdirectory of this.
    */
-  public get testDirectories(): QLTestDirectory[] { return this._testDirectories; }
-
-  private handleDidChangeQLPacks(): void {
-    this.refresh();
+  public get testDirectory(): QLTestDirectory | undefined {
+    return this._testDirectory;
   }
 
   private handleDidChange(uri: Uri): void {
@@ -144,61 +156,39 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
       this.refresh();
     }
   }
-
   protected async discover(): Promise<QLTestDiscoveryResults> {
-    const testDirectories: QLTestDirectory[] = [];
-    const watchPaths: string[] = [];
-    const qlPacks = this.qlPackDiscovery.qlPacks;
-    for (const qlPack of qlPacks) {
-      //HACK: Assume that only QL packs whose name ends with '-tests' contain tests.
-      if (this.isRelevantQlPack(qlPack)) {
-        watchPaths.push(qlPack.uri.fsPath);
-        const testPackage = await this.discoverTests(qlPack.uri.fsPath, qlPack.name);
-        if (testPackage !== undefined) {
-          testDirectories.push(testPackage);
-        }
-      }
-    }
-
-    return { testDirectories, watchPaths };
+    const testDirectory = await this.discoverTests();
+    return {
+      testDirectory,
+      watchPath: this.workspaceFolder.uri.fsPath
+    };
   }
 
   protected update(results: QLTestDiscoveryResults): void {
-    this._testDirectories = results.testDirectories;
+    this._testDirectory = results.testDirectory;
 
-    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
     this.watcher.clear();
-    results.watchPaths.forEach(watchPath => {
-      this.watcher.addWatch(new RelativePattern(watchPath, '**/*.{ql,qlref}'));
-    });
-    this._onDidChangeTests.fire();
-  }
-
-  /**
-   * Only include qlpacks suffixed with '-tests' that are contained
-   * within the provided workspace folder.
-   */
-  private isRelevantQlPack(qlPack: QLPack): boolean {
-    return qlPack.name.endsWith('-tests')
-      && workspace.getWorkspaceFolder(qlPack.uri)?.index === this.workspaceFolder.index;
+    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
+    this.watcher.addWatch(new RelativePattern(results.watchPath, '**/*.{ql,qlref}'));
+    // need to explicitly watch for changes to directories themselves.
+    this.watcher.addWatch(new RelativePattern(results.watchPath, '**/'));
+    this._onDidChangeTests.fire(undefined);
   }
 
   /**
    * Discover all QL tests in the specified directory and its subdirectories.
-   * @param fullPath The full path of the test directory.
-   * @param name The display name to use for the returned `TestDirectory` object.
    * @returns A `QLTestDirectory` object describing the contents of the directory, or `undefined` if
    *   no tests were found.
    */
-  private async discoverTests(fullPath: string, name: string): Promise<QLTestDirectory | undefined> {
-    const resolvedTests = (await this.cliServer.resolveTests(fullPath))
-      .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
+  private async discoverTests(): Promise<QLTestDirectory> {
+    const fullPath = this.workspaceFolder.uri.fsPath;
+    const name = this.workspaceFolder.name;
+    const rootDirectory = new QLTestDirectory(fullPath, name);
 
-    if (resolvedTests.length === 0) {
-      return undefined;
-    }
-    else {
-      const rootDirectory = new QLTestDirectory(fullPath, name);
+    // Don't try discovery on workspace folders that don't exist on the filesystem
+    if ((await fs.pathExists(fullPath))) {
+      const resolvedTests = (await this.cliServer.resolveTests(fullPath))
+        .filter((testPath) => !QLTestDiscovery.ignoreTestPath(testPath));
       for (const testPath of resolvedTests) {
         const relativePath = path.normalize(path.relative(fullPath, testPath));
         const dirName = path.dirname(relativePath);
@@ -207,9 +197,8 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
       }
 
       rootDirectory.finish();
-
-      return rootDirectory;
     }
+    return rootDirectory;
   }
 
   /**

extensions/ql-vscode/src/query-history.ts

@@ -1,14 +1,21 @@
 import * as path from 'path';
 import * as vscode from 'vscode';
-import { window as Window } from 'vscode';
+import { window as Window, env } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
-import * as helpers from './helpers';
+import {
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+  showAndLogWarningMessage,
+  showBinaryChoiceDialog
+} from './helpers';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
+import { commandRunner } from './commandRunner';
+import { assertNever } from './pure/helpers-pure';
 
 /**
  * query-history.ts
@@ -25,7 +32,7 @@ export type QueryHistoryItemOptions = {
   isQuickQuery?: boolean;
 };
 
-const SHOW_QUERY_TEXT_MSG = `\
+export const SHOW_QUERY_TEXT_MSG = `\
 ////////////////////////////////////////////////////////////////////////////////////
 // This is the text of the entire query file when it was executed for this query  //
 // run. The text or dependent libraries may have changed since then.              //
@@ -52,23 +59,23 @@ const SHOW_QUERY_TEXT_QUICK_EVAL_MSG = `\
  */
 const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
 
-interface QueryHistoryDataProvider extends vscode.TreeDataProvider<CompletedQuery> {
-  updateTreeItemContextValue(element: CompletedQuery): Promise<void>;
+enum SortOrder {
+  NameAsc = 'NameAsc',
+  NameDesc = 'NameDesc',
+  DateAsc = 'DateAsc',
+  DateDesc = 'DateDesc',
+  CountAsc = 'CountAsc',
+  CountDesc = 'CountDesc',
 }
 
 /**
  * Tree data provider for the query history view.
  */
-class HistoryTreeDataProvider implements QueryHistoryDataProvider {
-  /**
-   * XXX: This idiom for how to get a `.fire()`-able event emitter was
-   * cargo culted from another vscode extension. It seems rather
-   * involved and I hope there's something better that can be done
-   * instead.
-   */
-  private _onDidChangeTreeData: vscode.EventEmitter<
-    CompletedQuery | undefined
-  > = new vscode.EventEmitter<CompletedQuery | undefined>();
+export class HistoryTreeDataProvider extends DisposableObject {
+  private _sortOrder = SortOrder.DateAsc;
+
+  private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
+
   readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
     ._onDidChangeTreeData.event;
 
@@ -82,49 +89,58 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
   private current: CompletedQuery | undefined;
 
   constructor(extensionPath: string) {
+    super();
     this.failedIconPath = path.join(
       extensionPath,
       FAILED_QUERY_HISTORY_ITEM_ICON
     );
   }
 
-  async updateTreeItemContextValue(element: CompletedQuery): Promise<void> {
-    // Mark this query history item according to whether it has a
-    // SARIF file so that we can make context menu items conditionally
-    // available.
-    const hasResults = await element.query.hasInterpretedResults();
-    element.treeItem!.contextValue = hasResults
-      ? 'interpretedResultsItem'
-      : 'rawResultsItem';
-    this.refresh();
-  }
-
   async getTreeItem(element: CompletedQuery): Promise<vscode.TreeItem> {
-    if (element.treeItem !== undefined)
-      return element.treeItem;
+    const treeItem = new vscode.TreeItem(element.toString());
 
-    const it = new vscode.TreeItem(element.toString());
-
-    it.command = {
+    treeItem.command = {
       title: 'Query History Item',
       command: 'codeQLQueryHistory.itemClicked',
       arguments: [element],
     };
 
-    element.treeItem = it;
-    this.updateTreeItemContextValue(element);
+    // Mark this query history item according to whether it has a
+    // SARIF file so that we can make context menu items conditionally
+    // available.
+    const hasResults = await element.query.hasInterpretedResults();
+    treeItem.contextValue = hasResults
+      ? 'interpretedResultsItem'
+      : 'rawResultsItem';
 
     if (!element.didRunSuccessfully) {
-      it.iconPath = this.failedIconPath;
+      treeItem.iconPath = this.failedIconPath;
     }
 
-    return it;
+    return treeItem;
   }
 
   getChildren(
     element?: CompletedQuery
   ): vscode.ProviderResult<CompletedQuery[]> {
-    return element ? [] : this.history;
+    return element ? [] : this.history.sort((q1, q2) => {
+      switch (this.sortOrder) {
+        case SortOrder.NameAsc:
+          return q1.toString().localeCompare(q2.toString(), env.language);
+        case SortOrder.NameDesc:
+          return q2.toString().localeCompare(q1.toString(), env.language);
+        case SortOrder.DateAsc:
+          return q1.date.getTime() - q2.date.getTime();
+        case SortOrder.DateDesc:
+          return q2.date.getTime() - q1.date.getTime();
+        case SortOrder.CountAsc:
+          return q1.resultCount - q2.resultCount;
+        case SortOrder.CountDesc:
+          return q2.resultCount - q1.resultCount;
+        default:
+          assertNever(this.sortOrder);
+      }
+    });
   }
 
   getParent(_element: CompletedQuery): vscode.ProviderResult<CompletedQuery> {
@@ -135,7 +151,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
     return this.current;
   }
 
-  push(item: CompletedQuery): void {
+  pushQuery(item: CompletedQuery): void {
     this.current = item;
     this.history.push(item);
     this.refresh();
@@ -163,13 +179,22 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
     return this.history;
   }
 
-  refresh() {
-    this._onDidChangeTreeData.fire(undefined);
+  refresh(completedQuery?: CompletedQuery) {
+    this._onDidChangeTreeData.fire(completedQuery);
   }
 
   find(queryId: number): CompletedQuery | undefined {
     return this.allHistory.find((query) => query.query.queryID === queryId);
   }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire(undefined);
+  }
 }
 
 /**
@@ -178,6 +203,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
  */
 const DOUBLE_CLICK_TIME = 500;
 
+const NO_QUERY_SELECTED = 'No query selected. Select a query history item you have already run and try again.';
 export class QueryHistoryManager extends DisposableObject {
   treeDataProvider: HistoryTreeDataProvider;
   treeView: vscode.TreeView<CompletedQuery>;
@@ -204,6 +230,7 @@ export class QueryHistoryManager extends DisposableObject {
       canSelectMany: true,
     });
     this.push(this.treeView);
+    this.push(treeDataProvider);
 
     // Lazily update the tree view selection due to limitations of TreeView API (see
     // `updateTreeViewSelectionIfVisible` doc for details)
@@ -222,57 +249,87 @@ export class QueryHistoryManager extends DisposableObject {
       })
     );
 
-    logger.log('Registering query history panel commands.');
+    void logger.log('Registering query history panel commands.');
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.openQuery',
         this.handleOpenQuery.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.removeHistoryItem',
         this.handleRemoveHistoryItem.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
+        'codeQLQueryHistory.sortByName',
+        this.handleSortByName.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByDate',
+        this.handleSortByDate.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByCount',
+        this.handleSortByCount.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
         'codeQLQueryHistory.setLabel',
         this.handleSetLabel.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.compareWith',
         this.handleCompareWith.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.showQueryLog',
         this.handleShowQueryLog.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.showQueryText',
         this.handleShowQueryText.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
-        'codeQLQueryHistory.viewSarif',
-        this.handleViewSarif.bind(this)
+      commandRunner(
+        'codeQLQueryHistory.viewCsvResults',
+        this.handleViewCsvResults.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
+        'codeQLQueryHistory.viewCsvAlerts',
+        this.handleViewCsvAlerts.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.viewSarifAlerts',
+        this.handleViewSarifAlerts.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
         'codeQLQueryHistory.viewDil',
         this.handleViewDil.bind(this)
       )
     );
     this.push(
-      helpers.commandRunner(
+      commandRunner(
         'codeQLQueryHistory.itemClicked',
         async (item: CompletedQuery) => {
           return this.handleItemClicked(item, [item]);
@@ -315,6 +372,10 @@ export class QueryHistoryManager extends DisposableObject {
       return;
     }
 
+    if (!finalSingleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
     const textDocument = await vscode.workspace.openTextDocument(
       vscode.Uri.file(finalSingleItem.query.program.queryPath)
     );
@@ -347,11 +408,35 @@ export class QueryHistoryManager extends DisposableObject {
     });
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
-      this.treeView.reveal(current);
+      await this.treeView.reveal(current);
       await this.invokeCallbackOn(current);
     }
   }
 
+  async handleSortByName() {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  }
+
+  async handleSortByDate() {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAsc;
+    }
+  }
+
+  async handleSortByCount() {
+    if (this.treeDataProvider.sortOrder === SortOrder.CountAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.CountDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.CountAsc;
+    }
+  }
+
   async handleSetLabel(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
@@ -367,11 +452,14 @@ export class QueryHistoryManager extends DisposableObject {
     });
     // undefined response means the user cancelled the dialog; don't change anything
     if (response !== undefined) {
-      if (response === '')
-        // Interpret empty string response as 'go back to using default'
-        singleItem.options.label = undefined;
-      else singleItem.options.label = response;
-      this.treeDataProvider.refresh();
+      // Interpret empty string response as 'go back to using default'
+      singleItem.options.label = response === '' ? undefined : response;
+      if (this.treeDataProvider.sortOrder === SortOrder.NameAsc ||
+        this.treeDataProvider.sortOrder === SortOrder.NameDesc) {
+        this.treeDataProvider.refresh();
+      } else {
+        this.treeDataProvider.refresh(singleItem);
+      }
     }
   }
 
@@ -388,10 +476,10 @@ export class QueryHistoryManager extends DisposableObject {
       const to = await this.findOtherQueryToCompare(from, multiSelect);
 
       if (from && to) {
-        this.doCompareCallback(from, to);
+        await this.doCompareCallback(from, to);
       }
     } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
     }
   }
 
@@ -403,6 +491,11 @@ export class QueryHistoryManager extends DisposableObject {
     if (!this.assertSingleQuery(finalMultiSelect)) {
       return;
     }
+
+    if (!finalSingleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
     this.treeDataProvider.setCurrentItem(finalSingleItem);
 
     const now = new Date();
@@ -412,13 +505,13 @@ export class QueryHistoryManager extends DisposableObject {
     if (
       prevItemClick !== undefined &&
       now.valueOf() - prevItemClick.time.valueOf() < DOUBLE_CLICK_TIME &&
-      singleItem == prevItemClick.item
+      finalSingleItem == prevItemClick.item
     ) {
       // show original query file on double click
-      await this.handleOpenQuery(singleItem, [singleItem]);
+      await this.handleOpenQuery(finalSingleItem, [finalSingleItem]);
     } else {
       // show results on single click
-      await this.invokeCallbackOn(singleItem);
+      await this.invokeCallbackOn(finalSingleItem);
     }
   }
 
@@ -433,7 +526,7 @@ export class QueryHistoryManager extends DisposableObject {
     if (singleItem.logFileLocation) {
       await this.tryOpenExternalFile(singleItem.logFileLocation);
     } else {
-      helpers.showAndLogWarningMessage('No log file available');
+      void showAndLogWarningMessage('No log file available');
     }
   }
 
@@ -445,6 +538,10 @@ export class QueryHistoryManager extends DisposableObject {
       return;
     }
 
+    if (!singleItem) {
+      throw new Error(NO_QUERY_SELECTED);
+    }
+
     const queryName = singleItem.queryName.endsWith('.ql')
       ? singleItem.queryName
       : singleItem.queryName + '.ql';
@@ -453,13 +550,13 @@ export class QueryHistoryManager extends DisposableObject {
       queryText: encodeURIComponent(await this.getQueryText(singleItem)),
     });
     const uri = vscode.Uri.parse(
-      `codeql:${singleItem.query.queryID}-${queryName}?${params.toString()}`
+      `codeql:${singleItem.query.queryID}-${queryName}?${params.toString()}`, true
     );
     const doc = await vscode.workspace.openTextDocument(uri);
     await vscode.window.showTextDocument(doc, { preview: false });
   }
 
-  async handleViewSarif(
+  async handleViewSarifAlerts(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
   ) {
@@ -474,12 +571,43 @@ export class QueryHistoryManager extends DisposableObject {
       );
     } else {
       const label = singleItem.getLabel();
-      helpers.showAndLogInformationMessage(
+      void showAndLogInformationMessage(
         `Query ${label} has no interpreted results.`
       );
     }
   }
 
+  async handleViewCsvResults(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+    if (await singleItem.query.hasCsv()) {
+      void this.tryOpenExternalFile(singleItem.query.csvPath);
+      return;
+    }
+    await singleItem.query.exportCsvResults(this.qs, singleItem.query.csvPath, () => {
+      void this.tryOpenExternalFile(
+        singleItem.query.csvPath
+      );
+    });
+  }
+
+  async handleViewCsvAlerts(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    await this.tryOpenExternalFile(
+      await singleItem.query.ensureCsvProduced(this.qs)
+    );
+  }
+
   async handleViewDil(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[],
@@ -511,13 +639,16 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
-  addQuery(info: QueryWithResults): CompletedQuery {
+  buildCompletedQuery(info: QueryWithResults): CompletedQuery {
     const item = new CompletedQuery(info, this.queryHistoryConfigListener);
-    this.treeDataProvider.push(item);
-    this.updateTreeViewSelectionIfVisible();
     return item;
   }
 
+  addCompletedQuery(item: CompletedQuery) {
+    this.treeDataProvider.pushQuery(item);
+    this.updateTreeViewSelectionIfVisible();
+  }
+
   find(queryId: number): CompletedQuery | undefined {
     return this.treeDataProvider.find(queryId);
   }
@@ -537,7 +668,7 @@ export class QueryHistoryManager extends DisposableObject {
         // We must fire the onDidChangeTreeData event to ensure the current element can be selected
         // using `reveal` if the tree view was not visible when the current element was added.
         this.treeDataProvider.refresh();
-        this.treeView.reveal(current);
+        void this.treeView.reveal(current);
       }
     }
   }
@@ -545,9 +676,7 @@ export class QueryHistoryManager extends DisposableObject {
   private async tryOpenExternalFile(fileLocation: string) {
     const uri = vscode.Uri.file(fileLocation);
     try {
-      await vscode.window.showTextDocument(uri, {
-        preview: false
-      });
+      await vscode.window.showTextDocument(uri, { preview: false });
     } catch (e) {
       if (
         e.message.includes(
@@ -555,7 +684,7 @@ export class QueryHistoryManager extends DisposableObject {
         ) ||
         e.message.includes('too large to open')
       ) {
-        const res = await helpers.showBinaryChoiceDialog(
+        const res = await showBinaryChoiceDialog(
           `VS Code does not allow extensions to open files >50MB. This file
 exceeds that limit. Do you want to open it outside of VS Code?
 
@@ -566,13 +695,13 @@ the file in the file explorer and dragging it into the workspace.`
           try {
             await vscode.commands.executeCommand('revealFileInOS', uri);
           } catch (e) {
-            helpers.showAndLogErrorMessage(e.message);
+            void showAndLogErrorMessage(e.message);
           }
         }
       } else {
-        helpers.showAndLogErrorMessage(`Could not open file ${fileLocation}`);
-        logger.log(e.message);
-        logger.log(e.stack);
+        void showAndLogErrorMessage(`Could not open file ${fileLocation}`);
+        void logger.log(e.message);
+        void logger.log(e.stack);
       }
     }
   }
@@ -624,7 +753,7 @@ the file in the file explorer and dragging it into the workspace.`
 
   private assertSingleQuery(multiSelect: CompletedQuery[] = [], message = 'Please select a single query.') {
     if (multiSelect.length > 1) {
-      helpers.showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         message
       );
       return false;
@@ -688,7 +817,7 @@ the file in the file explorer and dragging it into the workspace.`
     };
   }
 
-  async updateTreeItemContextValue(element: CompletedQuery): Promise<void> {
-    this.treeDataProvider.updateTreeItemContextValue(element);
+  async refreshTreeView(completedQuery: CompletedQuery): Promise<void> {
+    this.treeDataProvider.refresh(completedQuery);
   }
 }

extensions/ql-vscode/src/query-results.ts

@@ -1,23 +1,24 @@
-import { env, TreeItem } from 'vscode';
+import { env } from 'vscode';
 
 import { QueryWithResults, tmpDir, QueryInfo } from './run-queries';
-import * as messages from './messages';
+import * as messages from './pure/messages';
 import * as cli from './cli';
 import * as sarif from 'sarif';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState, ResultsPaths } from './interface-types';
+import { RawResultsSortState, SortedResultSetInfo, DatabaseInfo, QueryMetadata, InterpretedResultsSortState, ResultsPaths } from './pure/interface-types';
 import { QueryHistoryConfig } from './config';
 import { QueryHistoryItemOptions } from './query-history';
 
 export class CompletedQuery implements QueryWithResults {
+  readonly date: Date;
   readonly time: string;
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
   readonly database: DatabaseInfo;
   readonly logFileLocation?: string;
   options: QueryHistoryItemOptions;
-  treeItem?: TreeItem;
+  resultCount: number;
   dispose: () => void;
 
   /**
@@ -45,8 +46,14 @@ export class CompletedQuery implements QueryWithResults {
     this.options = evaluation.options;
     this.dispose = evaluation.dispose;
 
-    this.time = new Date().toLocaleString(env.language);
+    this.date = new Date();
+    this.time = this.date.toLocaleString(env.language);
     this.sortedResultsInfo = new Map();
+    this.resultCount = 0;
+  }
+
+  setResultCount(value: number) {
+    this.resultCount = value;
   }
 
   get databaseName(): string {
@@ -55,6 +62,9 @@ export class CompletedQuery implements QueryWithResults {
   get queryName(): string {
     return getQueryName(this.query);
   }
+  get queryFileName(): string {
+    return getQueryFileName(this.query);
+  }
 
   get statusString(): string {
     switch (this.result.resultType) {
@@ -81,12 +91,14 @@ export class CompletedQuery implements QueryWithResults {
   }
 
   interpolate(template: string): string {
-    const { databaseName, queryName, time, statusString } = this;
+    const { databaseName, queryName, time, resultCount, statusString, queryFileName } = this;
     const replacements: { [k: string]: string } = {
       t: time,
       q: queryName,
       d: databaseName,
+      r: resultCount.toString(),
       s: statusString,
+      f: queryFileName,
       '%': '%',
     };
     return template.replace(/%(.)/g, (match, key) => {
@@ -96,8 +108,8 @@ export class CompletedQuery implements QueryWithResults {
   }
 
   getLabel(): string {
-      return this.options?.label
-        || this.config.format;
+    return this.options?.label
+      || this.config.format;
   }
 
   get didRunSuccessfully(): boolean {
@@ -144,17 +156,28 @@ export class CompletedQuery implements QueryWithResults {
  * Uses metadata if it exists, and defaults to the query file name.
  */
 export function getQueryName(query: QueryInfo) {
+  if (query.quickEvalPosition !== undefined) {
+    return 'Quick evaluation of ' + getQueryFileName(query);
+  } else if (query.metadata?.name) {
+    return query.metadata.name;
+  } else {
+    return getQueryFileName(query);
+  }
+}
+
+/**
+ * Gets the file name for an evaluated query.
+ * Defaults to the query file name and may contain position information for quick eval queries.
+ */
+export function getQueryFileName(query: QueryInfo) {
   // Queries run through quick evaluation are not usually the entire query file.
   // Label them differently and include the line numbers.
   if (query.quickEvalPosition !== undefined) {
     const { line, endLine, fileName } = query.quickEvalPosition;
     const lineInfo = line === endLine ? `${line}` : `${line}-${endLine}`;
-    return `Quick evaluation of ${path.basename(fileName)}:${lineInfo}`;
-  } else if (query.metadata?.name) {
-    return query.metadata.name;
-  } else {
-    return path.basename(query.program.queryPath);
+    return `${path.basename(fileName)}:${lineInfo}`;
   }
+  return path.basename(query.program.queryPath);
 }
 
 
@@ -171,17 +194,20 @@ export async function interpretResults(
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }
+  return await server.interpretBqrs(ensureMetadataIsComplete(metadata), resultsPath, interpretedResultsPath, sourceInfo);
+}
+
+export function ensureMetadataIsComplete(metadata: QueryMetadata | undefined) {
   if (metadata === undefined) {
     throw new Error('Can\'t interpret results without query metadata');
   }
-  let { kind, id } = metadata;
-  if (kind === undefined) {
+  if (metadata.kind === undefined) {
     throw new Error('Can\'t interpret results without query metadata including kind');
   }
-  if (id === undefined) {
+  if (metadata.id === undefined) {
     // Interpretation per se doesn't really require an id, but the
     // SARIF format does, so in the absence of one, we use a dummy id.
-    id = 'dummy-id';
+    metadata.id = 'dummy-id';
   }
-  return await server.interpretBqrs({ kind, id }, resultsPath, interpretedResultsPath, sourceInfo);
+  return metadata;
 }

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,16 +1,20 @@
 import * as cp from 'child_process';
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { Disposable } from 'vscode';
-import { CancellationToken, createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
+import { DisposableObject } from './pure/disposable-object';
+import { Disposable, CancellationToken, commands } from 'vscode';
+import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
 import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
-import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './messages';
-import * as messages from './messages';
+import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
+import * as messages from './pure/messages';
+import { ProgressCallback, ProgressTask } from './commandRunner';
+import * as fs from 'fs-extra';
+import * as helpers from './helpers';
 
 type ServerOpts = {
   logger: Logger;
+  contextStoragePath: string;
 }
 
 /** A running query server process and its associated message connection. */
@@ -26,7 +30,7 @@ class ServerProcess implements Disposable {
   }
 
   dispose(): void {
-    this.logger.log('Stopping query server...');
+    void this.logger.log('Stopping query server...');
     this.connection.dispose();
     this.child.stdin!.end();
     this.child.stderr!.destroy();
@@ -34,7 +38,7 @@ class ServerProcess implements Disposable {
 
     // On Windows, we usually have to terminate the process before closing its stdout.
     this.child.stdout!.destroy();
-    this.logger.log('Stopped query server.');
+    void this.logger.log('Stopped query server.');
   }
 }
 
@@ -47,22 +51,36 @@ type WithProgressReporting = (task: (progress: ProgressReporter, token: Cancella
  * to restart it (which disposes the existing process and starts a new one).
  */
 export class QueryServerClient extends DisposableObject {
+
   serverProcess?: ServerProcess;
   evaluationResultCallbacks: { [key: number]: (res: EvaluationResult) => void };
   progressCallbacks: { [key: number]: ((res: ProgressMessage) => void) | undefined };
   nextCallback: number;
   nextProgress: number;
   withProgressReporting: WithProgressReporting;
+
+  private readonly queryServerStartListeners = [] as ProgressTask<void>[];
+
+  // Can't use standard vscode EventEmitter here since they do not cause the calling
+  // function to fail if one of the event handlers fail. This is something that
+  // we need here.
+  readonly onDidStartQueryServer = (e: ProgressTask<void>) => {
+    this.queryServerStartListeners.push(e);
+  }
+
   public activeQueryName: string | undefined;
 
-  constructor(readonly config: QueryServerConfig, readonly cliServer: cli.CodeQLCliServer, readonly opts: ServerOpts, withProgressReporting: WithProgressReporting) {
+  constructor(
+    readonly config: QueryServerConfig,
+    readonly cliServer: cli.CodeQLCliServer,
+    readonly opts: ServerOpts,
+    withProgressReporting: WithProgressReporting
+  ) {
     super();
     // When the query server configuration changes, restart the query server.
     if (config.onDidChangeConfiguration !== undefined) {
-      this.push(config.onDidChangeConfiguration(async () => {
-        this.logger.log('Restarting query server due to configuration changes...');
-        await this.restartQueryServer();
-      }, this));
+      this.push(config.onDidChangeConfiguration(() =>
+        commands.executeCommand('codeQL.restartQueryServer')));
     }
     this.withProgressReporting = withProgressReporting;
     this.nextCallback = 0;
@@ -71,6 +89,26 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
+  async initLogger() {
+    let storagePath = this.opts.contextStoragePath;
+    let isCustomLogDirectory = false;
+    if (this.config.customLogDirectory) {
+      try {
+        if (!(await fs.pathExists(this.config.customLogDirectory))) {
+          await fs.mkdir(this.config.customLogDirectory);
+        }
+        void this.logger.log(`Saving query server logs to user-specified directory: ${this.config.customLogDirectory}.`);
+        storagePath = this.config.customLogDirectory;
+        isCustomLogDirectory = true;
+      } catch (e) {
+        void helpers.showAndLogErrorMessage(`${this.config.customLogDirectory} is not a valid directory. Logs will be stored in a temporary workspace directory instead.`);
+      }
+    }
+
+    await this.logger.setLogStoragePath(storagePath, isCustomLogDirectory);
+
+  }
+
   get logger(): Logger {
     return this.opts.logger;
   }
@@ -80,14 +118,24 @@ export class QueryServerClient extends DisposableObject {
     if (this.serverProcess !== undefined) {
       this.disposeAndStopTracking(this.serverProcess);
     } else {
-      this.logger.log('No server process to be stopped.');
+      void this.logger.log('No server process to be stopped.');
     }
   }
 
   /** Restarts the query server by disposing of the current server process and then starting a new one. */
-  async restartQueryServer(): Promise<void> {
+  async restartQueryServer(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<void> {
     this.stopQueryServer();
     await this.startQueryServer();
+
+    // Ensure we await all responses from event handlers so that
+    // errors can be properly reported to the user.
+    await Promise.all(this.queryServerStartListeners.map(handler => handler(
+      progress,
+      token
+    )));
   }
 
   showLog(): void {
@@ -102,11 +150,35 @@ export class QueryServerClient extends DisposableObject {
 
   /** Starts a new query server process, sending progress messages to the given reporter. */
   private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
+    await this.initLogger();
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
+
+    if (this.config.saveCache) {
+      args.push('--save-cache');
+    }
+
+    if (this.config.cacheSize > 0) {
+      args.push('--max-disk-cache');
+      args.push(this.config.cacheSize.toString());
+    }
+
+    if (await this.cliServer.cliConstraints.supportsDatabaseRegistration()) {
+      args.push('--require-db-registration');
+    }
+
+    if (await this.cliServer.cliConstraints.supportsOldEvalStats()) {
+      args.push('--old-eval-stats');
+    }
+
     if (this.config.debug) {
       args.push('--debug', '--tuple-counting');
     }
+
+    if (cli.shouldDebugQueryServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9010,server=y,suspend=n,quiet=y');
+    }
+
     const child = cli.spawnServer(
       this.config.codeQlPath,
       'CodeQL query server',
@@ -124,9 +196,8 @@ export class QueryServerClient extends DisposableObject {
     const connection = createMessageConnection(child.stdout, child.stdin);
     connection.onRequest(completeQuery, res => {
       if (!(res.runId in this.evaluationResultCallbacks)) {
-        this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
-      }
-      else {
+        void this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
+      } else {
         const baseLocation = this.logger.getBaseLocation();
         if (baseLocation && this.activeQueryName) {
           res.logFileLocation = path.join(baseLocation, this.activeQueryName);
@@ -141,7 +212,7 @@ export class QueryServerClient extends DisposableObject {
         callback(res);
       }
     });
-    this.serverProcess = new ServerProcess(child, connection, this.opts.logger);
+    this.serverProcess = new ServerProcess(child, connection, this.logger);
     // Ensure the server process is disposed together with this client.
     this.track(this.serverProcess);
     connection.listen();

extensions/ql-vscode/src/quick-query.ts

@@ -1,48 +1,59 @@
 import * as fs from 'fs-extra';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
-import { CancellationToken, ExtensionContext, window as Window, workspace, Uri } from 'vscode';
+import {
+  CancellationToken,
+  ExtensionContext,
+  window as Window,
+  workspace,
+  Uri
+} from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import { CodeQLCliServer } from './cli';
 import { DatabaseUI } from './databases-ui';
-import * as helpers from './helpers';
-import { logger } from './logging';
+import {
+  getInitialQueryContents,
+  getPrimaryDbscheme,
+  getQlPackForDbscheme,
+  showBinaryChoiceDialog,
+} from './helpers';
+import {
+  ProgressCallback,
+  UserCancellationException
+} from './commandRunner';
 
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
 const QUICK_QUERY_WORKSPACE_FOLDER_NAME = 'Quick Queries';
+const QLPACK_FILE_HEADER = '# This is an automatically generated file.\n\n';
 
 export function isQuickQueryPath(queryPath: string): boolean {
   return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
 }
 
-/**
- * `getBaseText` heuristically returns an appropriate import statement
- * prelude based on the filename of the dbscheme file given. TODO: add
- * a 'default import' field to the qlpack itself, and use that.
- */
-function getBaseText(dbschemeBase: string) {
-  if (dbschemeBase == 'semmlecode.javascript.dbscheme') return 'import javascript\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.cpp.dbscheme') return 'import cpp\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.dbscheme') return 'import java\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.python.dbscheme') return 'import python\n\nselect ""';
-  if (dbschemeBase == 'semmlecode.csharp.dbscheme') return 'import csharp\n\nselect ""';
-  if (dbschemeBase == 'go.dbscheme') return 'import go\n\nselect ""';
-  return 'select ""';
-}
-
-function getQuickQueriesDir(ctx: ExtensionContext): string {
+async function getQuickQueriesDir(ctx: ExtensionContext): Promise<string> {
   const storagePath = ctx.storagePath;
   if (storagePath === undefined) {
     throw new Error('Workspace storage path is undefined');
   }
   const queriesPath = path.join(storagePath, QUICK_QUERIES_DIR_NAME);
-  fs.ensureDir(queriesPath, { mode: 0o700 });
+  await fs.ensureDir(queriesPath, { mode: 0o700 });
   return queriesPath;
 }
 
+function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
+  workspace.updateWorkspaceFolders(
+    index,
+    len,
+    { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
+  );
+}
 
-
+function findExistingQuickQueryEditor() {
+  return Window.visibleTextEditors.find(editor =>
+    path.basename(editor.document.uri.fsPath) === QUICK_QUERY_QUERY_NAME
+  );
+}
 
 /**
  * Show a buffer the user can enter a simple query into.
@@ -51,30 +62,22 @@ export async function displayQuickQuery(
   ctx: ExtensionContext,
   cliServer: CodeQLCliServer,
   databaseUI: DatabaseUI,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken
 ) {
 
-  function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
-    workspace.updateWorkspaceFolders(
-      index,
-      len,
-      { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
-    );
-  }
-
   try {
-    const workspaceFolders = workspace.workspaceFolders || [];
-    const queriesDir = await getQuickQueriesDir(ctx);
-
     // If there is already a quick query open, don't clobber it, just
     // show it.
-    const existing = workspace.textDocuments.find(doc => path.basename(doc.uri.fsPath) === QUICK_QUERY_QUERY_NAME);
-    if (existing !== undefined) {
-      Window.showTextDocument(existing);
+    const existing = findExistingQuickQueryEditor();
+    if (existing) {
+      await Window.showTextDocument(existing.document);
       return;
     }
 
+    const workspaceFolders = workspace.workspaceFolders || [];
+    const queriesDir = await getQuickQueriesDir(ctx);
+
     // We need to have a multi-root workspace to make quick query work
     // at all. Changing the workspace from single-root to multi-root
     // causes a restart of the whole extension host environment, so we
@@ -85,7 +88,7 @@ export async function displayQuickQuery(
     // being undefined) just let the user know that they're in for a
     // restart.
     if (workspace.workspaceFile === undefined) {
-      const makeMultiRoot = await helpers.showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
+      const makeMultiRoot = await showBinaryChoiceDialog('Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?');
       if (makeMultiRoot) {
         updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
       }
@@ -93,10 +96,11 @@ export async function displayQuickQuery(
     }
 
     const index = workspaceFolders.findIndex(folder => folder.name === QUICK_QUERY_WORKSPACE_FOLDER_NAME);
-    if (index === -1)
+    if (index === -1) {
       updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
-    else
+    } else {
       updateQuickQueryDir(queriesDir, index, 1);
+    }
 
     // We're going to infer which qlpack to use from the current database
     const dbItem = await databaseUI.getDatabaseItem(progress, token);
@@ -105,31 +109,40 @@ export async function displayQuickQuery(
     }
 
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
-    const { qlpack, dbscheme } = await helpers.resolveDatasetFolder(cliServer, datasetFolder);
-    const quickQueryQlpackYaml: any = {
-      name: 'quick-query',
-      version: '1.0.0',
-      libraryPathDependencies: [qlpack]
-    };
-
-    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const dbscheme = await getPrimaryDbscheme(datasetFolder);
+    const qlpack = (await getQlPackForDbscheme(cliServer, dbscheme)).dbschemePack;
     const qlPackFile = path.join(queriesDir, 'qlpack.yml');
-    await fs.writeFile(qlFile, getBaseText(path.basename(dbscheme)), 'utf8');
-    await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8');
-    Window.showTextDocument(await workspace.openTextDocument(qlFile));
-  }
+    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const shouldRewrite = await checkShouldRewrite(qlPackFile, qlpack);
 
-  // TODO: clean up error handling for top-level commands like this
-  catch (e) {
-    if (e instanceof helpers.UserCancellationException) {
-      logger.log(e.message);
+    // Only rewrite the qlpack file if the database has changed
+    if (shouldRewrite) {
+      const quickQueryQlpackYaml: any = {
+        name: 'vscode/quick-query',
+        version: '1.0.0',
+        libraryPathDependencies: [qlpack]
+      };
+      await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.safeDump(quickQueryQlpackYaml), 'utf8');
     }
-    else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      logger.log(e.message);
+
+    if (shouldRewrite || !(await fs.pathExists(qlFile))) {
+      await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
     }
-    else if (e instanceof Error)
-      helpers.showAndLogErrorMessage(e.message);
-    else
+
+    await Window.showTextDocument(await workspace.openTextDocument(qlFile));
+  } catch (e) {
+    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      throw new UserCancellationException(e.message);
+    } else {
       throw e;
+    }
   }
 }
+
+async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
+  if (!(await fs.pathExists(qlPackFile))) {
+    return true;
+  }
+  const qlPackContents: any = yaml.safeLoad(await fs.readFile(qlPackFile, 'utf8'));
+  return qlPackContents.libraryPathDependencies?.[0] !== newDependency;
+}

extensions/ql-vscode/src/quickEvalCodeLensProvider.ts

@@ -0,0 +1,46 @@
+import {
+  CodeLensProvider,
+  TextDocument,
+  CodeLens,
+  Command,
+  Range
+} from 'vscode';
+import { isQuickEvalCodelensEnabled } from './config';
+
+class QuickEvalCodeLensProvider implements CodeLensProvider {
+  async provideCodeLenses(document: TextDocument): Promise<CodeLens[]> {
+
+    const codeLenses: CodeLens[] = [];
+
+    if (isQuickEvalCodelensEnabled()) {
+      for (let index = 0; index < document.lineCount; index++) {
+        const textLine = document.lineAt(index);
+
+        // Match a predicate signature, including predicate name, parameter list, and opening brace.
+        // This currently does not match predicates that span multiple lines.
+        const regex = new RegExp(/(\w+)\s*\([^()]*\)\s*\{/);
+
+        const matches = textLine.text.match(regex);
+
+        // Make sure that a code lens is not generated for any predicate that is commented out.
+        if (matches && !(/^\s*\/\//).test(textLine.text)) {
+          const range: Range = new Range(
+            textLine.range.start.line, matches.index!,
+            textLine.range.end.line, matches.index! + 1
+          );
+
+          const command: Command = {
+            command: 'codeQL.codeLensQuickEval',
+            title: `Quick Evaluation: ${matches[1]}`,
+            arguments: [document.uri, range]
+          };
+          const codeLens = new CodeLens(range, command);
+          codeLenses.push(codeLens);
+        }
+      }
+    }
+    return codeLenses;
+  }
+}
+
+export default QuickEvalCodeLensProvider;

extensions/ql-vscode/src/remote-queries/download-link.ts

@@ -0,0 +1,20 @@
+/**
+ * Represents a link to an artifact to be downloaded. 
+ */
+export interface DownloadLink {
+  /**
+   *  A unique id of the artifact being downloaded. 
+   */
+  id: string;
+
+  /**
+   * The URL path to use against the GitHub API to download the
+   * linked artifact. 
+   */
+  urlPath: string;
+
+  /**
+   * An optional path to follow inside the downloaded archive containing the artifact.
+   */
+  innerFilePath?: string;
+}

extensions/ql-vscode/src/remote-queries/gh-actions-api-client.ts

@@ -0,0 +1,246 @@
+import * as unzipper from 'unzipper';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { showAndLogWarningMessage } from '../helpers';
+import { Credentials } from '../authentication';
+import { logger } from '../logging';
+import { tmpDir } from '../run-queries';
+import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
+import { DownloadLink } from './download-link';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueryResultIndex, RemoteQueryResultIndexItem } from './remote-query-result-index';
+
+interface ApiResultIndexItem {
+  nwo: string;
+  id: string;
+  results_count: number;
+  bqrs_file_size: number;
+  sarif_file_size?: number;
+}
+
+export async function getRemoteQueryIndex(
+  credentials: Credentials,
+  remoteQuery: RemoteQuery
+): Promise<RemoteQueryResultIndex | undefined> {
+  const controllerRepo = remoteQuery.controllerRepository;
+  const owner = controllerRepo.owner;
+  const repoName = controllerRepo.name;
+  const workflowRunId = remoteQuery.actionsWorkflowRunId;
+
+  const workflowUri = `https://github.com/${owner}/${repoName}/actions/runs/${workflowRunId}`;
+  const artifactsUrlPath = `/repos/${owner}/${repoName}/actions/artifacts`;
+
+  const artifactList = await listWorkflowRunArtifacts(credentials, owner, repoName, workflowRunId);
+  const resultIndexArtifactId = getArtifactIDfromName('result-index', workflowUri, artifactList);
+  const resultIndexItems = await getResultIndexItems(credentials, owner, repoName, resultIndexArtifactId);
+
+  const allResultsArtifactId = getArtifactIDfromName('all-results', workflowUri, artifactList);
+
+  const items = resultIndexItems.map(item => {
+    const artifactId = getArtifactIDfromName(item.id, workflowUri, artifactList);
+
+    return {
+      id: item.id.toString(),
+      artifactId: artifactId,
+      nwo: item.nwo,
+      resultCount: item.results_count,
+      bqrsFileSize: item.bqrs_file_size,
+      sarifFileSize: item.sarif_file_size,
+    } as RemoteQueryResultIndexItem;
+  });
+
+  return {
+    allResultsArtifactId,
+    artifactsUrlPath,
+    items,
+  };
+}
+
+export async function downloadArtifactFromLink(
+  credentials: Credentials,
+  downloadLink: DownloadLink
+): Promise<string> {
+  const octokit = await credentials.getOctokit();
+
+  // Download the zipped artifact.
+  const response = await octokit.request(`GET ${downloadLink.urlPath}/zip`, {});
+
+  const zipFilePath = path.join(tmpDir.name, `${downloadLink.id}.zip`);
+  await saveFile(`${zipFilePath}`, response.data as ArrayBuffer);
+
+  // Extract the zipped artifact.
+  const extractedPath = path.join(tmpDir.name, downloadLink.id);
+  await unzipFile(zipFilePath, extractedPath);
+
+  return downloadLink.innerFilePath
+    ? path.join(extractedPath, downloadLink.innerFilePath)
+    : extractedPath;
+}
+
+/**
+ * Downloads the result index artifact and extracts the result index items.
+ * @param credentials Credentials for authenticating to the GitHub API.
+ * @param owner
+ * @param repo
+ * @param workflowRunId The ID of the workflow run to get the result index for.
+ * @returns An object containing the result index.
+ */
+async function getResultIndexItems(
+  credentials: Credentials,
+  owner: string,
+  repo: string,
+  artifactId: number
+): Promise<ApiResultIndexItem[]> {
+  const artifactPath = await downloadArtifact(credentials, owner, repo, artifactId);
+  const indexFilePath = path.join(artifactPath, 'index.json');
+  if (!(await fs.pathExists(indexFilePath))) {
+    void showAndLogWarningMessage('Could not find an `index.json` file in the result artifact.');
+    return [];
+  }
+  const resultIndex = await fs.readFile(path.join(artifactPath, 'index.json'), 'utf8');
+
+  try {
+    return JSON.parse(resultIndex);
+  } catch (error) {
+    throw new Error(`Invalid result index file: ${error}`);
+  }
+}
+
+/**
+ * Gets the status of a workflow run.
+ * @param credentials Credentials for authenticating to the GitHub API.
+ * @param owner 
+ * @param repo 
+ * @param workflowRunId The ID of the workflow run to get the result index for.
+ * @returns The workflow run status.
+ */
+export async function getWorkflowStatus(
+  credentials: Credentials,
+  owner: string,
+  repo: string,
+  workflowRunId: number): Promise<RemoteQueryWorkflowResult> {
+  const octokit = await credentials.getOctokit();
+
+  const workflowRun = await octokit.rest.actions.getWorkflowRun({
+    owner,
+    repo,
+    run_id: workflowRunId
+  });
+
+  if (workflowRun.data.status === 'completed') {
+    if (workflowRun.data.conclusion === 'success') {
+      return { status: 'CompletedSuccessfully' };
+    } else {
+      const error = getWorkflowError(workflowRun.data.conclusion);
+      return { status: 'CompletedUnsuccessfully', error };
+    }
+  }
+
+  return { status: 'InProgress' };
+}
+
+/**
+ * Lists the workflow run artifacts for the given workflow run ID.
+ * @param credentials Credentials for authenticating to the GitHub API.
+ * @param owner
+ * @param repo
+ * @param workflowRunId The ID of the workflow run to list artifacts for.
+ * @returns An array of artifact details (including artifact name and ID).
+ */
+async function listWorkflowRunArtifacts(
+  credentials: Credentials,
+  owner: string,
+  repo: string,
+  workflowRunId: number
+) {
+  const octokit = await credentials.getOctokit();
+  const response = await octokit.rest.actions.listWorkflowRunArtifacts({
+    owner,
+    repo,
+    run_id: workflowRunId,
+  });
+
+  return response.data.artifacts;
+}
+
+/**
+ * @param artifactName The artifact name, as a string.
+ * @param artifacts An array of artifact details (from the "list workflow run artifacts" API response).
+ * @returns The artifact ID corresponding to the given artifact name.
+ */
+function getArtifactIDfromName(
+  artifactName: string,
+  workflowUri: string,
+  artifacts: Array<{ id: number, name: string }>
+): number {
+  const artifact = artifacts.find(a => a.name === artifactName);
+
+  if (!artifact) {
+    const errorMessage =
+      `Could not find artifact with name ${artifactName} in workflow ${workflowUri}.
+      Please check whether the workflow run has successfully completed.`;
+    throw Error(errorMessage);
+  }
+
+  return artifact?.id;
+}
+
+/**
+ * Downloads an artifact from a workflow run.
+ * @param credentials Credentials for authenticating to the GitHub API.
+ * @param owner
+ * @param repo
+ * @param artifactId The ID of the artifact to download.
+ * @returns The path to the enclosing directory of the unzipped artifact.
+ */
+async function downloadArtifact(
+  credentials: Credentials,
+  owner: string,
+  repo: string,
+  artifactId: number
+): Promise<string> {
+  const octokit = await credentials.getOctokit();
+  const response = await octokit.rest.actions.downloadArtifact({
+    owner,
+    repo,
+    artifact_id: artifactId,
+    archive_format: 'zip',
+  });
+  const artifactPath = path.join(tmpDir.name, `${artifactId}`);
+  await saveFile(`${artifactPath}.zip`, response.data as ArrayBuffer);
+  await unzipFile(`${artifactPath}.zip`, artifactPath);
+  return artifactPath;
+}
+
+async function saveFile(filePath: string, data: ArrayBuffer): Promise<void> {
+  void logger.log(`Saving file to ${filePath}`);
+  await fs.writeFile(filePath, Buffer.from(data));
+}
+
+async function unzipFile(sourcePath: string, destinationPath: string) {
+  void logger.log(`Unzipping file to ${destinationPath}`);
+  const file = await unzipper.Open.file(sourcePath);
+  await file.extract({ path: destinationPath });
+}
+
+function getWorkflowError(conclusion: string | null): string {
+  if (!conclusion) {
+    return 'Workflow finished without a conclusion';
+  }
+
+  if (conclusion === 'cancelled') {
+    return 'The remote query execution was cancelled.';
+  }
+
+  if (conclusion === 'timed_out') {
+    return 'The remote query execution timed out.';
+  }
+
+  if (conclusion === 'failure') {
+    // TODO: Get the actual error from the workflow or potentially
+    // from an artifact from the action itself.
+    return 'The remote query execution has failed.';
+  }
+
+  return `Unexpected query execution conclusion: ${conclusion}`;
+}

extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts

@@ -0,0 +1,278 @@
+import {
+  WebviewPanel,
+  ExtensionContext,
+  window as Window,
+  ViewColumn,
+  Uri,
+  workspace,
+} from 'vscode';
+import * as path from 'path';
+import * as vscode from 'vscode';
+import * as fs from 'fs-extra';
+
+import { tmpDir } from '../run-queries';
+import {
+  ToRemoteQueriesMessage,
+  FromRemoteQueriesMessage,
+  RemoteQueryDownloadLinkClickedMessage,
+} from '../pure/interface-types';
+import { Logger } from '../logging';
+import { getHtmlForWebview } from '../interface-utils';
+import { assertNever } from '../pure/helpers-pure';
+import { AnalysisResult, RemoteQueryResult } from './remote-query-result';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
+import { AnalysisResult as AnalysisResultViewModel } from './shared/remote-query-result';
+import { downloadArtifactFromLink } from './gh-actions-api-client';
+import { Credentials } from '../authentication';
+import { showAndLogWarningMessage, showInformationMessageWithAction } from '../helpers';
+import { URLSearchParams } from 'url';
+import { SHOW_QUERY_TEXT_MSG } from '../query-history';
+
+export class RemoteQueriesInterfaceManager {
+  private panel: WebviewPanel | undefined;
+  private panelLoaded = false;
+  private panelLoadedCallBacks: (() => void)[] = [];
+
+  constructor(
+    private ctx: ExtensionContext,
+    private logger: Logger,
+  ) {
+    this.panelLoadedCallBacks.push(() => {
+      void logger.log('Remote queries view loaded');
+    });
+  }
+
+  async showResults(query: RemoteQuery, queryResult: RemoteQueryResult) {
+    this.getPanel().reveal(undefined, true);
+
+    await this.waitForPanelLoaded();
+    await this.postMessage({
+      t: 'setRemoteQueryResult',
+      queryResult: this.buildViewModel(query, queryResult)
+    });
+  }
+
+  /**
+   * Builds up a model tailored to the view based on the query and result domain entities.
+   * The data is cleaned up, sorted where necessary, and transformed to a format that
+   * the view model can use. 
+   * @param query Information about the query that was run.
+   * @param queryResult The result of the query.
+   * @returns A fully created view model.
+   */
+  private buildViewModel(query: RemoteQuery, queryResult: RemoteQueryResult): RemoteQueryResultViewModel {
+    const queryFileName = path.basename(query.queryFilePath);
+    const totalResultCount = queryResult.analysisResults.reduce((acc, cur) => acc + cur.resultCount, 0);
+    const executionDuration = this.getDuration(queryResult.executionEndTime, query.executionStartTime);
+    const analysisResults = this.buildAnalysisResults(queryResult.analysisResults);
+    const affectedRepositories = queryResult.analysisResults.filter(r => r.resultCount > 0);
+
+    return {
+      queryTitle: query.queryName,
+      queryFileName: queryFileName,
+      queryFilePath: query.queryFilePath,
+      queryText: query.queryText,
+      totalRepositoryCount: query.repositories.length,
+      affectedRepositoryCount: affectedRepositories.length,
+      totalResultCount: totalResultCount,
+      executionTimestamp: this.formatDate(query.executionStartTime),
+      executionDuration: executionDuration,
+      downloadLink: queryResult.allResultsDownloadLink,
+      results: analysisResults
+    };
+  }
+
+  getPanel(): WebviewPanel {
+    if (this.panel == undefined) {
+      const { ctx } = this;
+      const panel = (this.panel = Window.createWebviewPanel(
+        'remoteQueriesView',
+        'Remote Query Results',
+        { viewColumn: ViewColumn.Active, preserveFocus: true },
+        {
+          enableScripts: true,
+          enableFindWidget: true,
+          retainContextWhenHidden: true,
+          localResourceRoots: [
+            Uri.file(tmpDir.name),
+            Uri.file(path.join(this.ctx.extensionPath, 'out')),
+          ],
+        }
+      ));
+      this.panel.onDidDispose(
+        () => {
+          this.panel = undefined;
+        },
+        null,
+        ctx.subscriptions
+      );
+
+      const scriptPathOnDisk = Uri.file(
+        ctx.asAbsolutePath('out/remoteQueriesView.js')
+      );
+
+      const stylesheetPathOnDisk = Uri.file(
+        ctx.asAbsolutePath('out/remote-queries/view/remoteQueries.css')
+      );
+
+      panel.webview.html = getHtmlForWebview(
+        panel.webview,
+        scriptPathOnDisk,
+        stylesheetPathOnDisk
+      );
+      panel.webview.onDidReceiveMessage(
+        async (e) => this.handleMsgFromView(e),
+        undefined,
+        ctx.subscriptions
+      );
+    }
+    return this.panel;
+  }
+
+  private waitForPanelLoaded(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.panelLoaded) {
+        resolve();
+      } else {
+        this.panelLoadedCallBacks.push(resolve);
+      }
+    });
+  }
+
+  private async openFile(filePath: string) {
+    try {
+      const textDocument = await workspace.openTextDocument(filePath);
+      await Window.showTextDocument(textDocument, ViewColumn.One);
+    } catch (error) {
+      void showAndLogWarningMessage(`Could not open file: ${filePath}`);
+    }
+  }
+
+  private async openVirtualFile(text: string) {
+    try {
+      const params = new URLSearchParams({
+        queryText: encodeURIComponent(SHOW_QUERY_TEXT_MSG + text)
+      });
+      const uri = Uri.parse(
+        `remote-query:query-text.ql?${params.toString()}`,
+        true
+      );
+      const doc = await workspace.openTextDocument(uri);
+      await Window.showTextDocument(doc, { preview: false });
+    } catch (error) {
+      void showAndLogWarningMessage('Could not open query text');
+    }
+  }
+
+  private async handleMsgFromView(
+    msg: FromRemoteQueriesMessage
+  ): Promise<void> {
+    switch (msg.t) {
+      case 'remoteQueryLoaded':
+        this.panelLoaded = true;
+        this.panelLoadedCallBacks.forEach((cb) => cb());
+        this.panelLoadedCallBacks = [];
+        break;
+      case 'remoteQueryError':
+        void this.logger.log(
+          `Remote query error: ${msg.error}`
+        );
+        break;
+      case 'openFile':
+        await this.openFile(msg.filePath);
+        break;
+      case 'openVirtualFile':
+        await this.openVirtualFile(msg.queryText);
+        break;
+      case 'remoteQueryDownloadLinkClicked':
+        await this.handleDownloadLinkClicked(msg);
+        break;
+      default:
+        assertNever(msg);
+    }
+  }
+
+  private async handleDownloadLinkClicked(msg: RemoteQueryDownloadLinkClickedMessage): Promise<void> {
+    const credentials = await Credentials.initialize(this.ctx);
+
+    const filePath = await downloadArtifactFromLink(credentials, msg.downloadLink);
+    const isDir = (await fs.stat(filePath)).isDirectory();
+    const message = `Result file saved at ${filePath}`;
+    if (isDir) {
+      await vscode.window.showInformationMessage(message);
+    }
+    else {
+      const shouldOpenResults = await showInformationMessageWithAction(message, 'Open');
+      if (shouldOpenResults) {
+        const textDocument = await vscode.workspace.openTextDocument(filePath);
+        await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One);
+      }
+    }
+  }
+
+  private postMessage(msg: ToRemoteQueriesMessage): Thenable<boolean> {
+    return this.getPanel().webview.postMessage(msg);
+  }
+
+  private getDuration(startTime: Date, endTime: Date): string {
+    const diffInMs = startTime.getTime() - endTime.getTime();
+    return this.formatDuration(diffInMs);
+  }
+
+  private formatDuration(ms: number): string {
+    const seconds = ms / 1000;
+    const minutes = seconds / 60;
+    const hours = minutes / 60;
+    const days = hours / 24;
+    if (days > 1) {
+      return `${days.toFixed(2)} days`;
+    } else if (hours > 1) {
+      return `${hours.toFixed(2)} hours`;
+    } else if (minutes > 1) {
+      return `${minutes.toFixed(2)} minutes`;
+    } else {
+      return `${seconds.toFixed(2)} seconds`;
+    }
+  }
+
+  private formatDate = (d: Date): string => {
+    const datePart = d.toLocaleDateString(undefined, { day: 'numeric', month: 'short' });
+    const timePart = d.toLocaleTimeString(undefined, { hour: 'numeric', minute: 'numeric', hour12: true });
+    return `${datePart} at ${timePart}`;
+  };
+
+  private formatFileSize(bytes: number): string {
+    const kb = bytes / 1024;
+    const mb = kb / 1024;
+    const gb = mb / 1024;
+
+    if (bytes < 1024) {
+      return `${bytes} bytes`;
+    } else if (kb < 1024) {
+      return `${kb.toFixed(2)} KB`;
+    } else if (mb < 1024) {
+      return `${mb.toFixed(2)} MB`;
+    } else {
+      return `${gb.toFixed(2)} GB`;
+    }
+  }
+
+  /**
+   * Builds up a list of analysis results, in a data structure tailored to the view.
+   * @param analysisResults The results of a specific analysis.
+   * @returns A fully created view model.
+   */
+  private buildAnalysisResults(analysisResults: AnalysisResult[]): AnalysisResultViewModel[] {
+    const filteredAnalysisResults = analysisResults.filter(r => r.resultCount > 0);
+
+    const sortedAnalysisResults = filteredAnalysisResults.sort((a, b) => b.resultCount - a.resultCount);
+
+    return sortedAnalysisResults.map((analysisResult) => ({
+      nwo: analysisResult.nwo,
+      resultCount: analysisResult.resultCount,
+      downloadLink: analysisResult.downloadLink,
+      fileSize: this.formatFileSize(analysisResult.fileSizeInBytes)
+    }));
+  }
+}

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -0,0 +1,102 @@
+import { CancellationToken, commands, ExtensionContext, Uri, window } from 'vscode';
+import { Credentials } from '../authentication';
+import { CodeQLCliServer } from '../cli';
+import { ProgressCallback } from '../commandRunner';
+import { showAndLogErrorMessage, showInformationMessageWithAction } from '../helpers';
+import { Logger } from '../logging';
+import { runRemoteQuery } from './run-remote-query';
+import { RemoteQueriesInterfaceManager } from './remote-queries-interface';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueriesMonitor } from './remote-queries-monitor';
+import { getRemoteQueryIndex } from './gh-actions-api-client';
+import { RemoteQueryResultIndex } from './remote-query-result-index';
+import { RemoteQueryResult } from './remote-query-result';
+import { DownloadLink } from './download-link';
+
+export class RemoteQueriesManager {
+  private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
+
+  constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly logger: Logger,
+    private readonly cliServer: CodeQLCliServer
+  ) {
+    this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
+  }
+
+  public async runRemoteQuery(
+    uri: Uri | undefined,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<void> {
+    const credentials = await Credentials.initialize(this.ctx);
+
+    const querySubmission = await runRemoteQuery(
+      this.cliServer,
+      credentials, uri || window.activeTextEditor?.document.uri,
+      false,
+      progress,
+      token);
+
+    if (querySubmission && querySubmission.query) {
+      void commands.executeCommand('codeQL.monitorRemoteQuery', querySubmission.query);
+    }
+  }
+
+  public async monitorRemoteQuery(
+    query: RemoteQuery,
+    cancellationToken: CancellationToken
+  ): Promise<void> {
+    const credentials = await Credentials.initialize(this.ctx);
+
+    const queryResult = await this.remoteQueriesMonitor.monitorQuery(query, cancellationToken);
+
+    const executionEndTime = new Date();
+
+    if (queryResult.status === 'CompletedSuccessfully') {
+      const resultIndex = await getRemoteQueryIndex(credentials, query);
+      if (!resultIndex) {
+        void showAndLogErrorMessage(`There was an issue retrieving the result for the query ${query.queryName}`);
+        return;
+      }
+
+      const queryResult = this.mapQueryResult(executionEndTime, resultIndex);
+
+      const totalResultCount = queryResult.analysisResults.reduce((acc, cur) => acc + cur.resultCount, 0);
+      const message = `Query "${query.queryName}" run on ${query.repositories.length} repositories and returned ${totalResultCount} results`;
+
+      const shouldOpenView = await showInformationMessageWithAction(message, 'View');
+      if (shouldOpenView) {
+        const rqim = new RemoteQueriesInterfaceManager(this.ctx, this.logger);
+        await rqim.showResults(query, queryResult);
+      }
+    } else if (queryResult.status === 'CompletedUnsuccessfully') {
+      await showAndLogErrorMessage(`Remote query execution failed. Error: ${queryResult.error}`);
+      return;
+    } else if (queryResult.status === 'Cancelled') {
+      await showAndLogErrorMessage('Remote query monitoring was cancelled');
+    }
+  }
+
+  private mapQueryResult(executionEndTime: Date, resultIndex: RemoteQueryResultIndex): RemoteQueryResult {
+    const analysisResults = resultIndex.items.map(item => ({
+      nwo: item.nwo,
+      resultCount: item.resultCount,
+      fileSizeInBytes: item.sarifFileSize ? item.sarifFileSize : item.bqrsFileSize,
+      downloadLink: {
+        id: item.artifactId.toString(),
+        urlPath: `${resultIndex.artifactsUrlPath}/${item.artifactId}`,
+        innerFilePath: item.sarifFileSize ? 'results.sarif' : 'results.bqrs'
+      } as DownloadLink
+    }));
+
+    return {
+      executionEndTime,
+      analysisResults,
+      allResultsDownloadLink: {
+        id: resultIndex.allResultsArtifactId.toString(),
+        urlPath: `${resultIndex.artifactsUrlPath}/${resultIndex.allResultsArtifactId}`
+      }
+    };
+  }
+}

extensions/ql-vscode/src/remote-queries/remote-queries-monitor.ts

@@ -0,0 +1,60 @@
+import * as vscode from 'vscode';
+import { Credentials } from '../authentication';
+import { Logger } from '../logging';
+import { getWorkflowStatus } from './gh-actions-api-client';
+import { RemoteQuery } from './remote-query';
+import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
+
+export class RemoteQueriesMonitor {
+  // With a sleep of 5 seconds, the maximum number of attempts takes
+  // us to just over 2 days worth of monitoring.
+  private static readonly maxAttemptCount = 17280;
+  private static readonly sleepTime = 5000;
+
+  constructor(
+    private readonly extensionContext: vscode.ExtensionContext,
+    private readonly logger: Logger
+  ) {
+  }
+
+  public async monitorQuery(
+    remoteQuery: RemoteQuery,
+    cancellationToken: vscode.CancellationToken
+  ): Promise<RemoteQueryWorkflowResult> {
+    const credentials = await Credentials.initialize(this.extensionContext);
+
+    if (!credentials) {
+      throw Error('Error authenticating with GitHub');
+    }
+
+    let attemptCount = 0;
+
+    while (attemptCount <= RemoteQueriesMonitor.maxAttemptCount) {
+      if (cancellationToken && cancellationToken.isCancellationRequested) {
+        return { status: 'Cancelled' };
+      }
+
+      const workflowStatus = await getWorkflowStatus(
+        credentials,
+        remoteQuery.controllerRepository.owner,
+        remoteQuery.controllerRepository.name,
+        remoteQuery.actionsWorkflowRunId);
+
+      if (workflowStatus.status !== 'InProgress') {
+        return workflowStatus;
+      }
+
+      await this.sleep(RemoteQueriesMonitor.sleepTime);
+      attemptCount++;
+    }
+
+    void this.logger.log('Remote query monitoring timed out after 2 days');
+    return { status: 'Cancelled' };
+  }
+
+  private async sleep(ms: number) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+

extensions/ql-vscode/src/remote-queries/remote-query-result-index.ts

@@ -0,0 +1,14 @@
+export interface RemoteQueryResultIndex {
+  artifactsUrlPath: string;
+  allResultsArtifactId: number;
+  items: RemoteQueryResultIndexItem[];
+}
+
+export interface RemoteQueryResultIndexItem {
+  id: string;
+  artifactId: number;
+  nwo: string;
+  resultCount: number;
+  bqrsFileSize: number;
+  sarifFileSize?: number;
+}

extensions/ql-vscode/src/remote-queries/remote-query-result.ts

@@ -0,0 +1,14 @@
+import { DownloadLink } from './download-link';
+
+export interface RemoteQueryResult {
+  executionEndTime: Date;
+  analysisResults: AnalysisResult[];
+  allResultsDownloadLink: DownloadLink;
+}
+
+export interface AnalysisResult {
+  nwo: string,
+  resultCount: number,
+  downloadLink: DownloadLink,
+  fileSizeInBytes: number
+}

extensions/ql-vscode/src/remote-queries/remote-query-submission-result.ts

@@ -0,0 +1,6 @@
+import { RemoteQuery } from './remote-query';
+
+export interface RemoteQuerySubmissionResult {
+  queryDirPath?: string;
+  query?: RemoteQuery;
+}

extensions/ql-vscode/src/remote-queries/remote-query-workflow-result.ts

@@ -0,0 +1,10 @@
+export type RemoteQueryWorkflowStatus =
+  | 'InProgress'
+  | 'CompletedSuccessfully'
+  | 'CompletedUnsuccessfully'
+  | 'Cancelled';
+
+export interface RemoteQueryWorkflowResult {
+  status: RemoteQueryWorkflowStatus;
+  error?: string;
+}

extensions/ql-vscode/src/remote-queries/remote-query.ts

@@ -0,0 +1,11 @@
+import { Repository } from './repository';
+
+export interface RemoteQuery {
+  queryName: string;
+  queryFilePath: string;
+  queryText: string;
+  controllerRepository: Repository;
+  repositories: Repository[];
+  executionStartTime: Date;
+  actionsWorkflowRunId: number;
+}

extensions/ql-vscode/src/remote-queries/repository.ts

@@ -0,0 +1,4 @@
+export interface Repository {
+  owner: string;
+  name: string;
+}

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -0,0 +1,495 @@
+import { CancellationToken, QuickPickItem, Uri, window } from 'vscode';
+import * as path from 'path';
+import * as yaml from 'js-yaml';
+import * as fs from 'fs-extra';
+import * as tmp from 'tmp-promise';
+import {
+  askForLanguage,
+  findLanguage,
+  getOnDiskWorkspaceFolders,
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+  showInformationMessageWithAction,
+  tryGetQueryMetadata
+} from '../helpers';
+import { Credentials } from '../authentication';
+import * as cli from '../cli';
+import { logger } from '../logging';
+import { getRemoteControllerRepo, getRemoteRepositoryLists, setRemoteControllerRepo } from '../config';
+import { tmpDir } from '../run-queries';
+import { ProgressCallback, UserCancellationException } from '../commandRunner';
+import { OctokitResponse } from '@octokit/types/dist-types';
+import { RemoteQuery } from './remote-query';
+import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
+import { QueryMetadata } from '../pure/interface-types';
+
+interface Config {
+  repositories: string[];
+  ref?: string;
+  language?: string;
+}
+
+export interface QlPack {
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  defaultSuite?: Record<string, unknown>[];
+  defaultSuiteFile?: string;
+}
+interface RepoListQuickPickItem extends QuickPickItem {
+  repoList: string[];
+}
+
+interface QueriesResponse {
+  workflow_run_id: number
+}
+
+/**
+ * This regex matches strings of the form `owner/repo` where:
+ * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character
+ * - `repo` is made up of alphanumeric characters, hyphens, or underscores
+ */
+const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/;
+
+/**
+ * Well-known names for the query pack used by the server.
+ */
+const QUERY_PACK_NAME = 'codeql-remote/query';
+
+/**
+ * Gets the repositories to run the query against.
+ */
+export async function getRepositories(): Promise<string[] | undefined> {
+  const repoLists = getRemoteRepositoryLists();
+  if (repoLists && Object.keys(repoLists).length) {
+    const quickPickItems = Object.entries(repoLists).map<RepoListQuickPickItem>(([key, value]) => (
+      {
+        label: key,       // the name of the repository list
+        repoList: value,  // the actual array of repositories
+      }
+    ));
+    const quickpick = await window.showQuickPick<RepoListQuickPickItem>(
+      quickPickItems,
+      {
+        placeHolder: 'Select a repository list. You can define repository lists in the `codeQL.remoteQueries.repositoryLists` setting.',
+        ignoreFocusOut: true,
+      });
+    if (quickpick?.repoList.length) {
+      void logger.log(`Selected repositories: ${quickpick.repoList.join(', ')}`);
+      return quickpick.repoList;
+    } else {
+      void showAndLogErrorMessage('No repositories selected.');
+      return;
+    }
+  } else {
+    void logger.log('No repository lists defined. Displaying text input box.');
+    const remoteRepo = await window.showInputBox({
+      title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
+      placeHolder: '<owner>/<repo>',
+      prompt: 'Tip: you can save frequently used repositories in the `codeQL.remoteQueries.repositoryLists` setting',
+      ignoreFocusOut: true,
+    });
+    if (!remoteRepo) {
+      void showAndLogErrorMessage('No repositories entered.');
+      return;
+    } else if (!REPO_REGEX.test(remoteRepo)) { // Check if user entered invalid input
+      void showAndLogErrorMessage('Invalid repository format. Must be in the format <owner>/<repo> (e.g. github/codeql)');
+      return;
+    }
+    void logger.log(`Entered repository: ${remoteRepo}`);
+    return [remoteRepo];
+  }
+}
+
+/**
+ * Two possibilities:
+ * 1. There is no qlpack.yml in this directory. Assume this is a lone query and generate a synthetic qlpack for it.
+ * 2. There is a qlpack.yml in this directory. Assume this is a query pack and use the yml to pack the query before uploading it.
+ *
+ * @returns the entire qlpack as a base64 string.
+ */
+async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: string, queryPackDir: string, fallbackLanguage?: string): Promise<{
+  base64Pack: string,
+  language: string
+}> {
+  const originalPackRoot = await findPackRoot(queryFile);
+  const packRelativePath = path.relative(originalPackRoot, queryFile);
+  const targetQueryFileName = path.join(queryPackDir, packRelativePath);
+
+  let language: string | undefined;
+  if (await fs.pathExists(path.join(originalPackRoot, 'qlpack.yml'))) {
+    // don't include ql files. We only want the queryFile to be copied.
+    const toCopy = await cliServer.packPacklist(originalPackRoot, false);
+
+    // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist.
+    [path.join(originalPackRoot, 'qlpack.lock.yml'), path.join(originalPackRoot, 'codeql-pack.lock.yml'), queryFile]
+      .forEach(absolutePath => {
+        if (absolutePath) {
+          toCopy.push(absolutePath);
+        }
+      });
+
+    let copiedCount = 0;
+    await fs.copy(originalPackRoot, queryPackDir, {
+      filter: (file: string) =>
+        // copy file if it is in the packlist, or it is a parent directory of a file in the packlist
+        !!toCopy.find(f => {
+          // Normalized paths ensure that Windows drive letters are capitalized consistently.
+          const normalizedPath = Uri.file(f).fsPath;
+          const matches = normalizedPath === file || normalizedPath.startsWith(file + path.sep);
+          if (matches) {
+            copiedCount++;
+          }
+          return matches;
+        })
+    });
+
+    void logger.log(`Copied ${copiedCount} files to ${queryPackDir}`);
+
+    language = await findLanguage(cliServer, Uri.file(targetQueryFileName));
+
+  } else {
+    // open popup to ask for language if not already hardcoded
+    language = fallbackLanguage || await askForLanguage(cliServer);
+
+    // copy only the query file to the query pack directory
+    // and generate a synthetic query pack
+    void logger.log(`Copying ${queryFile} to ${queryPackDir}`);
+    await fs.copy(queryFile, targetQueryFileName);
+    void logger.log('Generating synthetic query pack');
+    const syntheticQueryPack = {
+      name: QUERY_PACK_NAME,
+      version: '0.0.0',
+      dependencies: {
+        [`codeql/${language}-all`]: '*',
+      }
+    };
+    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(syntheticQueryPack));
+  }
+  if (!language) {
+    throw new UserCancellationException('Could not determine language.');
+  }
+
+  await ensureNameAndSuite(queryPackDir, packRelativePath);
+
+  const bundlePath = await getPackedBundlePath(queryPackDir);
+  void logger.log(`Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`);
+  await cliServer.packInstall(queryPackDir);
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+  await cliServer.packBundle(queryPackDir, workspaceFolders, bundlePath, false);
+  const base64Pack = (await fs.readFile(bundlePath)).toString('base64');
+  return {
+    base64Pack,
+    language
+  };
+}
+
+async function findPackRoot(queryFile: string): Promise<string> {
+  // recursively find the directory containing qlpack.yml
+  let dir = path.dirname(queryFile);
+  while (!(await fs.pathExists(path.join(dir, 'qlpack.yml')))) {
+    dir = path.dirname(dir);
+    if (isFileSystemRoot(dir)) {
+      // there is no qlpack.yml in this direcory or any parent directory.
+      // just use the query file's directory as the pack root.
+      return path.dirname(queryFile);
+    }
+  }
+
+  return dir;
+}
+
+function isFileSystemRoot(dir: string): boolean {
+  const pathObj = path.parse(dir);
+  return pathObj.root === dir && pathObj.base === '';
+}
+
+async function createRemoteQueriesTempDirectory() {
+  const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
+  const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
+  await fs.mkdirp(queryPackDir);
+  return { remoteQueryDir, queryPackDir };
+}
+
+async function getPackedBundlePath(queryPackDir: string) {
+  return tmp.tmpName({
+    dir: path.dirname(queryPackDir),
+    postfix: 'generated.tgz',
+    prefix: 'qlpack',
+  });
+}
+
+export async function runRemoteQuery(
+  cliServer: cli.CodeQLCliServer,
+  credentials: Credentials,
+  uri: Uri | undefined,
+  dryRun: boolean,
+  progress: ProgressCallback,
+  token: CancellationToken
+): Promise<void | RemoteQuerySubmissionResult> {
+  if (!(await cliServer.cliConstraints.supportsRemoteQueries())) {
+    throw new Error(`Remote queries are not supported by this version of CodeQL. Please upgrade to v${cli.CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
+      } or later.`);
+  }
+
+  const { remoteQueryDir, queryPackDir } = await createRemoteQueriesTempDirectory();
+  try {
+    if (!uri?.fsPath.endsWith('.ql')) {
+      throw new UserCancellationException('Not a CodeQL query file.');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 1,
+      message: 'Determining project list'
+    });
+
+    const queryFile = uri.fsPath;
+    const repositoriesFile = queryFile.substring(0, queryFile.length - '.ql'.length) + '.repositories';
+    let ref: string | undefined;
+    // For the case of single file remote queries, use the language from the config in order to avoid the user having to select it.
+    let fallbackLanguage: string | undefined;
+    let repositories: string[] | undefined;
+
+    progress({
+      maxStep: 5,
+      step: 2,
+      message: 'Determining query target language'
+    });
+
+    // If the user has an explicit `.repositories` file, use that.
+    // Otherwise, prompt user to select repositories from the `codeQL.remoteQueries.repositoryLists` setting.
+    if (await fs.pathExists(repositoriesFile)) {
+      void logger.log(`Found '${repositoriesFile}'. Using information from that file to run ${queryFile}.`);
+
+      const config = yaml.safeLoad(await fs.readFile(repositoriesFile, 'utf8')) as Config;
+
+      ref = config.ref || 'main';
+      fallbackLanguage = config.language;
+      repositories = config.repositories;
+    } else {
+      ref = 'main';
+      repositories = await getRepositories();
+    }
+
+    if (!repositories || repositories.length === 0) {
+      throw new UserCancellationException('No repositories to query.');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 3,
+      message: 'Determining controller repo'
+    });
+
+    // Get the controller repo from the config, if it exists.
+    // If it doesn't exist, prompt the user to enter it, and save that value to the config.
+    let controllerRepo: string | undefined;
+    controllerRepo = getRemoteControllerRepo();
+    if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
+      void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
+      controllerRepo = await window.showInputBox({
+        title: 'Controller repository in which to display progress and results of remote queries',
+        placeHolder: '<owner>/<repo>',
+        prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
+        ignoreFocusOut: true,
+      });
+      if (!controllerRepo) {
+        void showAndLogErrorMessage('No controller repository entered.');
+        return;
+      } else if (!REPO_REGEX.test(controllerRepo)) { // Check if user entered invalid input
+        void showAndLogErrorMessage('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
+        return;
+      }
+      void logger.log(`Setting the controller repository as: ${controllerRepo}`);
+      await setRemoteControllerRepo(controllerRepo);
+    }
+
+    void logger.log(`Using controller repository: ${controllerRepo}`);
+    const [owner, repo] = controllerRepo.split('/');
+
+    progress({
+      maxStep: 5,
+      step: 4,
+      message: 'Bundling the query pack'
+    });
+
+    if (token.isCancellationRequested) {
+      throw new UserCancellationException('Cancelled');
+    }
+
+    const { base64Pack, language } = await generateQueryPack(cliServer, queryFile, queryPackDir, fallbackLanguage);
+
+    if (token.isCancellationRequested) {
+      throw new UserCancellationException('Cancelled');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 5,
+      message: 'Sending request'
+    });
+
+    const workflowRunId = await runRemoteQueriesApiRequest(credentials, ref, language, repositories, owner, repo, base64Pack, dryRun);
+    const queryStartTime = new Date();
+    const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
+
+    if (dryRun) {
+      return { queryDirPath: remoteQueryDir.path };
+    } else {
+      if (!workflowRunId) {
+        return;
+      }
+
+      const remoteQuery = await buildRemoteQueryEntity(repositories, queryFile, queryMetadata, owner, repo, queryStartTime, workflowRunId);
+
+      // don't return the path because it has been deleted
+      return { query: remoteQuery };
+    }
+
+  } finally {
+    if (dryRun) {
+      // If we are in a dry run keep the data around for debugging purposes.
+      void logger.log(`[DRY RUN] Not deleting ${queryPackDir}.`);
+    } else {
+      await remoteQueryDir.cleanup();
+    }
+  }
+}
+
+async function runRemoteQueriesApiRequest(
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repositories: string[],
+  owner: string,
+  repo: string,
+  queryPackBase64: string,
+  dryRun = false
+): Promise<void | number> {
+  if (dryRun) {
+    void showAndLogInformationMessage('[DRY RUN] Would have sent request. See extension log for the payload.');
+    void logger.log(JSON.stringify({ ref, language, repositories, owner, repo, queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes' }));
+    return;
+  }
+
+  try {
+    const octokit = await credentials.getOctokit();
+    const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
+      'POST /repos/:owner/:repo/code-scanning/codeql/queries',
+      {
+        owner,
+        repo,
+        data: {
+          ref,
+          language,
+          repositories,
+          query_pack: queryPackBase64,
+        }
+      }
+    );
+    const workflowRunId = response.data.workflow_run_id;
+    void showAndLogInformationMessage(`Successfully scheduled runs. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${workflowRunId}).`);
+    return workflowRunId;
+  } catch (error) {
+    return await attemptRerun(error, credentials, ref, language, repositories, owner, repo, queryPackBase64, dryRun);
+  }
+}
+
+/** Attempts to rerun the query on only the valid repositories */
+export async function attemptRerun(
+  error: any,
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repositories: string[],
+  owner: string,
+  repo: string,
+  queryPackBase64: string,
+  dryRun = false
+) {
+  if (typeof error.message === 'string' && error.message.includes('Some repositories were invalid')) {
+    const invalidRepos = error?.response?.data?.invalid_repos || [];
+    const reposWithoutDbUploads = error?.response?.data?.repos_without_db_uploads || [];
+    void logger.log('Unable to run query on some of the specified repositories');
+    if (invalidRepos.length > 0) {
+      void logger.log(`Invalid repos: ${invalidRepos.join(', ')}`);
+    }
+    if (reposWithoutDbUploads.length > 0) {
+      void logger.log(`Repos without DB uploads: ${reposWithoutDbUploads.join(', ')}`);
+    }
+
+    if (invalidRepos.length + reposWithoutDbUploads.length === repositories.length) {
+      // Every repo is invalid in some way
+      void showAndLogErrorMessage('Unable to run query on any of the specified repositories.');
+      return;
+    }
+
+    const popupMessage = 'Unable to run query on some of the specified repositories. [See logs for more details](command:codeQL.showLogs).';
+    const rerunQuery = await showInformationMessageWithAction(popupMessage, 'Rerun on the valid repositories only');
+    if (rerunQuery) {
+      const validRepositories = repositories.filter(r => !invalidRepos.includes(r) && !reposWithoutDbUploads.includes(r));
+      void logger.log(`Rerunning query on set of valid repositories: ${JSON.stringify(validRepositories)}`);
+      return await runRemoteQueriesApiRequest(credentials, ref, language, validRepositories, owner, repo, queryPackBase64, dryRun);
+    }
+  } else {
+    void showAndLogErrorMessage(error);
+  }
+}
+
+/**
+ * Updates the default suite of the query pack. This is used to ensure
+ * only the specified query is run.
+ *
+ * Also, ensure the query pack name is set to the name expected by the server.
+ *
+ * @param queryPackDir The directory containing the query pack
+ * @param packRelativePath The relative path to the query pack from the root of the query pack
+ */
+async function ensureNameAndSuite(queryPackDir: string, packRelativePath: string): Promise<void> {
+  const packPath = path.join(queryPackDir, 'qlpack.yml');
+  const qlpack = yaml.safeLoad(await fs.readFile(packPath, 'utf8')) as QlPack;
+  delete qlpack.defaultSuiteFile;
+
+  qlpack.name = QUERY_PACK_NAME;
+
+  qlpack.defaultSuite = [{
+    description: 'Query suite for remote query'
+  }, {
+    query: packRelativePath.replace(/\\/g, '/')
+  }];
+  await fs.writeFile(packPath, yaml.safeDump(qlpack));
+}
+
+async function buildRemoteQueryEntity(
+  repositories: string[],
+  queryFilePath: string,
+  queryMetadata: QueryMetadata | undefined,
+  controllerRepoOwner: string,
+  controllerRepoName: string,
+  queryStartTime: Date,
+  workflowRunId: number
+): Promise<RemoteQuery> {
+  // The query name is either the name as specified in the query metadata, or the file name. 
+  const queryName = queryMetadata?.name ?? path.basename(queryFilePath);
+
+  const queryRepos = repositories.map(r => {
+    const [owner, repo] = r.split('/');
+    return { owner: owner, name: repo };
+  });
+
+  const queryText = await fs.readFile(queryFilePath, 'utf8');
+
+  return {
+    queryName,
+    queryFilePath,
+    queryText,
+    controllerRepository: {
+      owner: controllerRepoOwner,
+      name: controllerRepoName,
+    },
+    repositories: queryRepos,
+    executionStartTime: queryStartTime,
+    actionsWorkflowRunId: workflowRunId
+  };
+}

extensions/ql-vscode/src/remote-queries/shared/remote-query-result.ts

@@ -0,0 +1,22 @@
+import { DownloadLink } from '../download-link';
+
+export interface RemoteQueryResult {
+  queryTitle: string;
+  queryFileName: string;
+  queryFilePath: string;
+  queryText: string;
+  totalRepositoryCount: number;
+  affectedRepositoryCount: number;
+  totalResultCount: number;
+  executionTimestamp: string;
+  executionDuration: string;
+  downloadLink: DownloadLink;
+  results: AnalysisResult[]
+}
+
+export interface AnalysisResult {
+  nwo: string,
+  resultCount: number,
+  downloadLink: DownloadLink,
+  fileSize: string,
+}

extensions/ql-vscode/src/remote-queries/view/.eslintrc.js

@@ -0,0 +1,13 @@
+module.exports = {
+  env: {
+    browser: true
+  },
+  extends: [
+    "plugin:react/recommended"
+  ],
+  settings: {
+    react: {
+      version: 'detect'
+    }
+  }
+}

extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx

@@ -0,0 +1,161 @@
+import * as React from 'react';
+import { useEffect, useState } from 'react';
+import * as Rdom from 'react-dom';
+import { SetRemoteQueryResultMessage } from '../../pure/interface-types';
+import { AnalysisResult, RemoteQueryResult } from '../shared/remote-query-result';
+import * as octicons from '../../view/octicons';
+
+import { vscode } from '../../view/vscode-api';
+import { DownloadLink } from '../download-link';
+
+const numOfReposInContractedMode = 10;
+
+const emptyQueryResult: RemoteQueryResult = {
+  queryTitle: '',
+  queryFileName: '',
+  queryFilePath: '',
+  queryText: '',
+  totalRepositoryCount: 0,
+  affectedRepositoryCount: 0,
+  totalResultCount: 0,
+  executionTimestamp: '',
+  executionDuration: '',
+  downloadLink: {
+    id: '',
+    urlPath: '',
+  },
+  results: []
+};
+
+const download = (link: DownloadLink) => {
+  vscode.postMessage({
+    t: 'remoteQueryDownloadLinkClicked',
+    downloadLink: link
+  });
+};
+
+const AnalysisResultItem = (props: AnalysisResult) => (
+  <span>
+    <span className="vscode-codeql__analysis-item">{octicons.repo}</span>
+    <span className="vscode-codeql__analysis-item">{props.nwo}</span>
+    <span className="vscode-codeql__analysis-item vscode-codeql__badge-container">
+      <span className="vscode-codeql__badge">{props.resultCount}</span>
+    </span>
+    <span className="vscode-codeql__analysis-item">
+      <a
+        className="vscode-codeql__download-link"
+        onClick={() => download(props.downloadLink)}>
+        {octicons.download}{props.fileSize}
+      </a>
+    </span>
+  </span>
+);
+
+const SummaryWithResults = (queryResult: RemoteQueryResult) => (
+  <div className="vscode-codeql__query-summary-container">
+    <h2 className="vscode-codeql__query-summary-title">Repositories with results ({queryResult.affectedRepositoryCount}):</h2>
+    <a className="vscode-codeql__summary-download-link vscode-codeql__download-link"
+      onClick={() => download(queryResult.downloadLink)}>
+      {octicons.download}Download all
+    </a>
+  </div>
+);
+
+const SummaryNoResults = () => (
+  <div className="vscode-codeql__query-summary-container">
+    <h2 className="vscode-codeql__query-summary-title">No results found</h2>
+  </div>
+);
+
+export function RemoteQueries(): JSX.Element {
+  const [queryResult, setQueryResult] = useState<RemoteQueryResult>(emptyQueryResult);
+
+  useEffect(() => {
+    window.addEventListener('message', (evt: MessageEvent) => {
+      if (evt.origin === window.origin) {
+        const msg: SetRemoteQueryResultMessage = evt.data;
+        if (msg.t === 'setRemoteQueryResult') {
+          setQueryResult(msg.queryResult);
+        }
+      } else {
+        // sanitize origin
+        const origin = evt.origin.replace(/\n|\r/g, '');
+        console.error(`Invalid event origin ${origin}`);
+      }
+    });
+  });
+
+  if (!queryResult) {
+    return <div>Waiting for results to load.</div>;
+  }
+
+  const [repoListExpanded, setRepoListExpanded] = useState(false);
+  const numOfReposToShow = repoListExpanded ? queryResult.results.length : numOfReposInContractedMode;
+
+  const openQueryFile = () => {
+    vscode.postMessage({
+      t: 'openFile',
+      filePath: queryResult.queryFilePath
+    });
+  };
+
+  const openQueryTextVirtualFile = () => {
+    vscode.postMessage({
+      t: 'openVirtualFile',
+      queryText: queryResult.queryText
+    });
+  };
+
+  try {
+    return <div className="vscode-codeql__remote-queries-view">
+      <h1 className="vscode-codeql__query-title">{queryResult.queryTitle}</h1>
+
+      <p className="vscode-codeql__paragraph">
+        {queryResult.totalResultCount} results in {queryResult.totalRepositoryCount} repositories
+        ({queryResult.executionDuration}), {queryResult.executionTimestamp}
+      </p>
+      <p className="vscode-codeql__paragraph">
+        <span className="vscode-codeql__query-file">{octicons.file}
+          <a className="vscode-codeql__query-file-link" href="#" onClick={openQueryFile}>
+            {queryResult.queryFileName}
+          </a>
+        </span>
+        <span>{octicons.codeSquare}
+          <a className="vscode-codeql__query-file-link" href="#" onClick={openQueryTextVirtualFile}>
+            query
+          </a>
+        </span>
+      </p>
+
+      {
+        queryResult.affectedRepositoryCount === 0
+          ? <SummaryNoResults />
+          : <SummaryWithResults {...queryResult} />
+      }
+
+      <ul className="vscode-codeql__results-list">
+        {queryResult.results.slice(0, numOfReposToShow).map((result, i) =>
+          <li key={result.nwo} className="vscode-codeql__results-list-item">
+            <AnalysisResultItem {...result} />
+          </li>
+        )}
+      </ul>
+      {
+        queryResult.results.length > numOfReposInContractedMode &&
+        <button className="vscode-codeql__expand-button" onClick={() => setRepoListExpanded(!repoListExpanded)}>
+          {repoListExpanded ? (<span>View less</span>) : (<span>View all</span>)}
+        </button>
+      }
+    </div>;
+  } catch (err) {
+    console.error(err);
+    return <div>There was an error displaying the view.</div>;
+  }
+}
+
+Rdom.render(
+  <RemoteQueries />,
+  document.getElementById('root'),
+  // Post a message to the extension when fully loaded.
+  () => vscode.postMessage({ t: 'remoteQueryLoaded' })
+);

extensions/ql-vscode/src/remote-queries/view/remoteQueries.css

@@ -0,0 +1,110 @@
+.vscode-codeql__remote-queries-view {
+  font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial,
+    sans-serif, Apple Color Emoji, Segoe UI Emoji;
+}
+
+.vscode-codeql__paragraph {
+  margin: 0.5em 0 0.5em 0;
+}
+
+.vscode-codeql__download-link {
+  display: inline-block;
+  font-size: x-small;
+  text-decoration: none;
+  cursor: pointer;
+}
+
+.vscode-codeql__download-link svg {
+  fill: var(--vscode-textLink-foreground);
+}
+
+.vscode-codeql__query-title {
+  font-size: large;
+  margin-bottom: 0.5em;
+  font-weight: 500;
+}
+
+.octicon {
+  fill: var(--vscode-editor-foreground);
+  height: 1.2em;
+  width: 1.2em;
+  vertical-align: middle;
+  display: inline-block;
+}
+
+.octicon-light {
+  opacity: 0.6;
+}
+
+.vscode-codeql__query-file {
+  padding-right: 1em;
+}
+
+.vscode-codeql__query-file-link {
+  text-decoration: none;
+  padding-left: 0.3em;
+  color: var(--vscode-editor-foreground);
+}
+
+.vscode-codeql__query-file-link:hover {
+  color: var(--vscode-editor-foreground);
+}
+
+.vscode-codeql__query-summary-container {
+  padding-top: 1.5em;
+}
+
+.vscode-codeql__query-summary-title {
+  font-size: medium;
+  font-weight: 500;
+  padding: 0 0.5em 0 0;
+  margin: 0;
+  display: inline-block;
+  vertical-align: middle;
+}
+
+.vscode-codeql__summary-download-link {
+  vertical-align: middle;
+}
+
+.vscode-codeql__results-list {
+  list-style-type: none;
+  margin: 0;
+  padding: 0.5em 0 0 0;
+}
+
+.vscode-codeql__results-list-item {
+  margin-top: 0.5em;
+}
+
+.vscode-codeql__analysis-item {
+  padding-right: 0.1em;
+}
+
+.vscode-codeql__badge-container {
+  justify-content: center;
+  align-items: center;
+  min-height: 100vh;
+  padding-left: 0.2em;
+}
+
+.vscode-codeql__badge {
+  display: inline-block;
+  min-width: 1.5em;
+  padding: 0.3em;
+  border-radius: 35%;
+  font-size: x-small;
+  text-align: center;
+  background: var(--vscode-badge-background);
+  color: var(--vscode-badge-foreground);
+  border-color: var(--vscode-badge-background);
+}
+
+.vscode-codeql__expand-button {
+  background: none;
+  color: var(--vscode-textLink-foreground);
+  border: none;
+  cursor: pointer;
+  padding-top: 1em;
+  font-size: x-small;
+}

extensions/ql-vscode/src/remote-queries/view/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "module": "esnext",
+    "moduleResolution": "node",
+    "target": "es6",
+    "outDir": "out",
+    "lib": ["es6", "dom"],
+    "jsx": "react",
+    "sourceMap": true,
+    "rootDir": "..",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "experimentalDecorators": true
+  },
+  "exclude": ["node_modules"]
+}

extensions/ql-vscode/src/run-queries.ts

@@ -1,28 +1,34 @@
 import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as path from 'path';
-import * as tmp from 'tmp';
+import * as tmp from 'tmp-promise';
 import {
   CancellationToken,
   ConfigurationTarget,
+  Range,
   TextDocument,
   TextEditor,
   Uri,
-  window
+  window,
+  workspace
 } from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 
 import * as cli from './cli';
 import * as config from './config';
-import { DatabaseItem, getUpgradesDirectories } from './databases';
-import * as helpers from './helpers';
-import { DatabaseInfo, QueryMetadata, ResultsPaths } from './interface-types';
+import { DatabaseItem } from './databases';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tryGetQueryMetadata } from './helpers';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
+import { DatabaseInfo, QueryMetadata, ResultsPaths } from './pure/interface-types';
 import { logger } from './logging';
-import * as messages from './messages';
+import * as messages from './pure/messages';
 import { QueryHistoryItemOptions } from './query-history';
 import * as qsClient from './queryserver-client';
 import { isQuickQueryPath } from './quick-query';
-import { upgradeDatabase } from './upgrades';
+import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
+import { ensureMetadataIsComplete } from './query-results';
+import { SELECT_QUERY_NAME } from './contextual/locationFinder';
+import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
 
 /**
  * run-queries.ts
@@ -52,6 +58,7 @@ export class QueryInfo {
 
   readonly compiledQueryPath: string;
   readonly dilPath: string;
+  readonly csvPath: string;
   readonly resultsPaths: ResultsPaths;
   readonly dataset: Uri; // guarantee the existence of a well-defined dataset dir at this point
   readonly queryID: number;
@@ -67,6 +74,7 @@ export class QueryInfo {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
     this.dilPath = path.join(tmpDir.name, `results${this.queryID}.dil`);
+    this.csvPath = path.join(tmpDir.name, `results${this.queryID}.csv`);
     this.resultsPaths = {
       resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`),
       interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`)
@@ -79,18 +87,24 @@ export class QueryInfo {
 
   async run(
     qs: qsClient.QueryServerClient,
-    progress: helpers.ProgressCallback,
+    upgradeQlo: string | undefined,
+    availableMlModels: cli.MlModelInfo[],
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.EvaluationResult> {
     let result: messages.EvaluationResult | null = null;
 
     const callbackId = qs.registerCallback(res => { result = res; });
 
+    const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
+
     const queryToRun: messages.QueryToRun = {
       resultsPath: this.resultsPaths.resultsPath,
       qlo: Uri.file(this.compiledQueryPath).toString(),
+      compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
       allowUnknownTemplates: true,
       templateValues: this.templates,
+      availableMlModels: availableMlModelUris,
       id: callbackId,
       timeoutSecs: qs.config.timeoutSecs,
     };
@@ -121,7 +135,7 @@ export class QueryInfo {
 
   async compile(
     qs: qsClient.QueryServerClient,
-    progress: helpers.ProgressCallback,
+    progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<messages.CompilationMessage[]> {
     let compiled: messages.CheckQueryResult | undefined;
@@ -138,6 +152,7 @@ export class QueryInfo {
           localChecking: false,
           noComputeGetUrl: false,
           noComputeToString: false,
+          computeDefaultStrings: true
         },
         extraOptions: {
           timeoutSecs: qs.config.timeoutSecs
@@ -149,7 +164,7 @@ export class QueryInfo {
 
       compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
     } finally {
-      qs.logger.log(' - - - COMPILATION DONE - - - ');
+      void qs.logger.log(' - - - COMPILATION DONE - - - ');
     }
     return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
   }
@@ -160,9 +175,17 @@ export class QueryInfo {
   async canHaveInterpretedResults(): Promise<boolean> {
     const hasMetadataFile = await this.dbItem.hasMetadataFile();
     if (!hasMetadataFile) {
-      logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
+      void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
     }
-    return hasMetadataFile;
+
+    const hasKind = !!this.metadata?.kind;
+    if (!hasKind) {
+      void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
+    }
+
+    const isTable = hasKind && this.metadata?.kind === 'table';
+
+    return hasMetadataFile && hasKind && !isTable;
   }
 
   /**
@@ -179,6 +202,13 @@ export class QueryInfo {
     return fs.pathExists(this.dilPath);
   }
 
+  /**
+   * Holds if this query already has CSV results produced
+   */
+  async hasCsv(): Promise<boolean> {
+    return fs.pathExists(this.csvPath);
+  }
+
   async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
     if (await this.hasDil()) {
       return this.dilPath;
@@ -194,8 +224,50 @@ export class QueryInfo {
     return this.dilPath;
   }
 
+  async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string, onFinish: () => void): Promise<void> {
+    let stopDecoding = false;
+    const out = fs.createWriteStream(csvPath);
+    out.on('finish', onFinish);
+    out.on('error', () => {
+      if (!stopDecoding) {
+        stopDecoding = true;
+        void showAndLogErrorMessage(`Failed to write CSV results to ${csvPath}`);
+      }
+    });
+    let nextOffset: number | undefined = 0;
+    while (nextOffset !== undefined && !stopDecoding) {
+      const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, SELECT_QUERY_NAME, {
+        pageSize: 100,
+        offset: nextOffset,
+      });
+      for (const tuple of chunk.tuples)
+        out.write(tuple.join(',') + '\n');
+      nextOffset = chunk.next;
+    }
+    out.end();
+  }
+
+  async ensureCsvProduced(qs: qsClient.QueryServerClient): Promise<string> {
+    if (await this.hasCsv()) {
+      return this.csvPath;
+    }
+
+    let sourceInfo;
+    if (this.dbItem.sourceArchive !== undefined) {
+      sourceInfo = {
+        sourceArchive: this.dbItem.sourceArchive.fsPath,
+        sourceLocationPrefix: await this.dbItem.getSourceLocationPrefix(
+          qs.cliServer
+        ),
+      };
+    }
+
+    await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
+    return this.csvPath;
+  }
 }
 
+
 export interface QueryWithResults {
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
@@ -208,7 +280,7 @@ export interface QueryWithResults {
 export async function clearCacheInDatabase(
   qs: qsClient.QueryServerClient,
   dbItem: DatabaseItem,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<messages.ClearCacheResult> {
   if (dbItem.contents === undefined) {
@@ -261,9 +333,10 @@ async function convertToQlPath(filePath: string): Promise<string> {
 
 
 /** Gets the selected position within the given editor. */
-async function getSelectedPosition(editor: TextEditor): Promise<messages.Position> {
-  const pos = editor.selection.start;
-  const posEnd = editor.selection.end;
+async function getSelectedPosition(editor: TextEditor, range?: Range): Promise<messages.Position> {
+  const selectedRange = range || editor.selection;
+  const pos = selectedRange.start;
+  const posEnd = selectedRange.end;
   // Convert from 0-based to 1-based line and column numbers.
   return {
     fileName: await convertToQlPath(editor.document.fileName),
@@ -284,13 +357,13 @@ async function checkDbschemeCompatibility(
   cliServer: cli.CodeQLCliServer,
   qs: qsClient.QueryServerClient,
   query: QueryInfo,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<void> {
-  const searchPath = helpers.getOnDiskWorkspaceFolders();
+  const searchPath = getOnDiskWorkspaceFolders();
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
-    const { scripts, finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
+    const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
     const hash = async function(filename: string): Promise<string> {
       return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
     };
@@ -309,18 +382,15 @@ async function checkDbschemeCompatibility(
     const upgradableTo = await hash(finalDbscheme);
 
     if (upgradableTo != dbschemeOfLib) {
-      logger.log(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but database has scheme ${query.program.dbschemePath}, and no upgrade path found`);
-      throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace. Please try using a newer version of the query libraries.`);
+      reportNoUpgradePath(query);
     }
 
     if (upgradableTo == dbschemeOfLib &&
       dbschemeOfDb != dbschemeOfLib) {
       // Try to upgrade the database
-      await upgradeDatabase(
+      await upgradeDatabaseExplicit(
         qs,
         query.dbItem,
-        Uri.file(finalDbscheme),
-        getUpgradesDirectories(scripts),
         progress,
         token
       );
@@ -328,6 +398,42 @@ async function checkDbschemeCompatibility(
   }
 }
 
+function reportNoUpgradePath(query: QueryInfo) {
+  throw new Error(`Query ${query.program.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`);
+}
+
+/**
+ * Compile a non-destructive upgrade.
+ */
+async function compileNonDestructiveUpgrade(
+  qs: qsClient.QueryServerClient,
+  upgradeTemp: tmp.DirectoryResult,
+  query: QueryInfo,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<string> {
+  const searchPath = getOnDiskWorkspaceFolders();
+
+  if (!query.dbItem?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
+  }
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme);
+
+  if (!matchesTarget) {
+    reportNoUpgradePath(query);
+  }
+  const result = await compileDatabaseUpgradeSequence(qs, query.dbItem, scripts, upgradeTemp, progress, token);
+  if (result.compiledUpgrade === undefined) {
+    const error = result.error || '[no error message available]';
+    throw new Error(error);
+  }
+  // We can upgrade to the actual target
+  query.program.dbschemePath = query.queryDbscheme;
+  // We are new enough that we will always support single file upgrades.
+  return result.compiledUpgrade;
+
+}
+
 /**
  * Prompts the user to save `document` if it has unsaved changes.
  *
@@ -344,7 +450,7 @@ async function promptUserToSaveChanges(document: TextDocument): Promise<boolean>
     else {
       const yesItem = { title: 'Yes', isCloseAffordance: false };
       const alwaysItem = { title: 'Always Save', isCloseAffordance: false };
-      const noItem = { title: 'No (run anyway)', isCloseAffordance: false };
+      const noItem = { title: 'No (run version on disk)', isCloseAffordance: false };
       const cancelItem = { title: 'Cancel', isCloseAffordance: true };
       const message = 'Query file has unsaved changes. Save now?';
       const chosenItem = await window.showInformationMessage(
@@ -363,7 +469,7 @@ async function promptUserToSaveChanges(document: TextDocument): Promise<boolean>
       }
 
       if (chosenItem === cancelItem) {
-        throw new helpers.UserCancellationException('Query run cancelled.', true);
+        throw new UserCancellationException('Query run cancelled.', true);
       }
     }
   }
@@ -386,7 +492,7 @@ type SelectedQuery = {
  * @param selectedResourceUri The selected resource when the command was run.
  * @param quickEval Whether the command being run is `Quick Evaluation`.
 */
-export async function determineSelectedQuery(selectedResourceUri: Uri | undefined, quickEval: boolean): Promise<SelectedQuery> {
+export async function determineSelectedQuery(selectedResourceUri: Uri | undefined, quickEval: boolean, range?: Range): Promise<SelectedQuery> {
   const editor = window.activeTextEditor;
 
   // Choose which QL file to use.
@@ -425,7 +531,7 @@ export async function determineSelectedQuery(selectedResourceUri: Uri | undefine
   // then prompt the user to save it first.
   if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
     if (await promptUserToSaveChanges(editor.document)) {
-      editor.document.save();
+      await editor.document.save();
     }
   }
 
@@ -440,7 +546,7 @@ export async function determineSelectedQuery(selectedResourceUri: Uri | undefine
       // Report an error if we end up in this (hopefully unlikely) situation.
       throw new Error('The selected resource for quick evaluation should match the active editor.');
     }
-    quickEvalPosition = await getSelectedPosition(editor);
+    quickEvalPosition = await getSelectedPosition(editor, range);
     quickEvalText = editor.document.getText(editor.selection);
   }
 
@@ -453,16 +559,17 @@ export async function compileAndRunQueryAgainstDatabase(
   db: DatabaseItem,
   quickEval: boolean,
   selectedQueryUri: Uri | undefined,
-  progress: helpers.ProgressCallback,
+  progress: ProgressCallback,
   token: CancellationToken,
   templates?: messages.TemplateDefinitions,
+  range?: Range
 ): Promise<QueryWithResults> {
   if (!db.contents || !db.contents.dbSchemeUri) {
     throw new Error(`Database ${db.databaseUri} does not have a CodeQL database scheme.`);
   }
 
   // Determine which query to run, based on the selection and the active editor.
-  const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, quickEval);
+  const { queryPath, quickEvalPosition, quickEvalText } = await determineSelectedQuery(selectedQueryUri, quickEval, range);
 
   const historyItemOptions: QueryHistoryItemOptions = {};
   historyItemOptions.isQuickQuery === isQuickQueryPath(queryPath);
@@ -473,10 +580,14 @@ export async function compileAndRunQueryAgainstDatabase(
   }
 
   // Get the workspace folder paths.
-  const diskWorkspaceFolders = helpers.getOnDiskWorkspaceFolders();
+  const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
   // Figure out the library path for the query.
   const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, queryPath);
 
+  if (!packConfig.dbscheme) {
+    throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
+  }
+
   // Check whether the query has an entirely different schema from the
   // database. (Queries that merely need the database to be upgraded
   // won't trigger this check)
@@ -484,8 +595,8 @@ export async function compileAndRunQueryAgainstDatabase(
   const querySchemaName = path.basename(packConfig.dbscheme);
   const dbSchemaName = path.basename(db.contents.dbSchemeUri.fsPath);
   if (querySchemaName != dbSchemaName) {
-    logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
-    throw new Error(`The query ${path.basename(queryPath)} cannot be run against the selected database: their target languages are different. Please select a different database and try again.`);
+    void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
+    throw new Error(`The query ${path.basename(queryPath)} cannot be run against the selected database (${db.name}): their target languages are different. Please select a different database and try again.`);
   }
 
   const qlProgram: messages.QlProgram = {
@@ -501,76 +612,106 @@ export async function compileAndRunQueryAgainstDatabase(
   };
 
   // Read the query metadata if possible, to use in the UI.
-  let metadata: QueryMetadata | undefined;
-  try {
-    metadata = await cliServer.resolveMetadata(qlProgram.queryPath);
-  } catch (e) {
-    // Ignore errors and provide no metadata.
-    logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
+  const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
+
+  let availableMlModels: cli.MlModelInfo[] = [];
+  // The `capabilities.untrustedWorkspaces.restrictedConfigurations` entry in package.json doesn't
+  // work with hidden settings, so we manually check that the workspace is trusted before looking at
+  // whether the `shouldInsecurelyLoadMlModelsFromPacks` setting is enabled.
+  if (workspace.isTrusted &&
+    config.isCanary() &&
+    config.shouldInsecurelyLoadMlModelsFromPacks() &&
+    await cliServer.cliConstraints.supportsResolveMlModels()) {
+    try {
+      availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders)).models;
+      void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
+    } catch (e) {
+      const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
+        `query without any ML models: ${e}.`;
+      void showAndLogErrorMessage(message);
+    }
   }
 
   const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
-  await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
 
-  let errors;
+  const upgradeDir = await tmp.dir({ dir: upgradesTmpDir.name, unsafeCleanup: true });
   try {
-    errors = await query.compile(qs, progress, token);
-  } catch (e) {
-    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+    let upgradeQlo;
+    if (await hasNondestructiveUpgradeCapabilities(qs)) {
+      upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, progress, token);
     } else {
-      throw e;
+      await checkDbschemeCompatibility(cliServer, qs, query, progress, token);
     }
-  }
-
-  if (errors.length == 0) {
-    const result = await query.run(qs, progress, token);
-    if (result.resultType !== messages.QueryResultType.SUCCESS) {
-      const message = result.message || 'Failed to run query';
-      logger.log(message);
-      helpers.showAndLogErrorMessage(message);
-    }
-    return {
-      query,
-      result,
-      database: {
-        name: db.name,
-        databaseUri: db.databaseUri.toString(true)
-      },
-      options: historyItemOptions,
-      logFileLocation: result.logFileLocation,
-      dispose: () => {
-        qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+    let errors;
+    try {
+      errors = await query.compile(qs, progress, token);
+    } catch (e) {
+      if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+        return createSyntheticResult(query, db, historyItemOptions, 'Query cancelled', messages.QueryResultType.CANCELLATION);
+      } else {
+        throw e;
       }
-    };
-  } else {
-    // Error dialogs are limited in size and scrollability,
-    // so we include a general description of the problem,
-    // and direct the user to the output window for the detailed compilation messages.
-    // However we don't show quick eval errors there so we need to display them anyway.
-    qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
-
-    const formattedMessages: string[] = [];
-
-    for (const error of errors) {
-      const message = error.message || '[no error message available]';
-      const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
-      formattedMessages.push(formatted);
-      qs.logger.log(formatted);
     }
-    if (quickEval && formattedMessages.length <= 3) {
-      helpers.showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
+
+    if (errors.length === 0) {
+      const result = await query.run(qs, upgradeQlo, availableMlModels, progress, token);
+      if (result.resultType !== messages.QueryResultType.SUCCESS) {
+        const message = result.message || 'Failed to run query';
+        void logger.log(message);
+        void showAndLogErrorMessage(message);
+      }
+      return {
+        query,
+        result,
+        database: {
+          name: db.name,
+          databaseUri: db.databaseUri.toString(true)
+        },
+        options: historyItemOptions,
+        logFileLocation: result.logFileLocation,
+        dispose: () => {
+          qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+        }
+      };
     } else {
-      helpers.showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') +
-        ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
-        ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
-        ' and choose CodeQL Query Server from the dropdown.');
-    }
+      // Error dialogs are limited in size and scrollability,
+      // so we include a general description of the problem,
+      // and direct the user to the output window for the detailed compilation messages.
+      // However we don't show quick eval errors there so we need to display them anyway.
+      void qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
 
-    return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
+      const formattedMessages: string[] = [];
+
+      for (const error of errors) {
+        const message = error.message || '[no error message available]';
+        const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
+        formattedMessages.push(formatted);
+        void qs.logger.log(formatted);
+      }
+      if (quickEval && formattedMessages.length <= 2) {
+        // If there are more than 2 error messages, they will not be displayed well in a popup
+        // and will be trimmed by the function displaying the error popup. Accordingly, we only
+        // try to show the errors if there are 2 or less, otherwise we direct the user to the log.
+        void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
+      } else {
+        void showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
+      }
+
+      return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
+    }
+  } finally {
+    try {
+      await upgradeDir.cleanup();
+    } catch (e) {
+      void qs.logger.log(`Could not clean up the upgrades dir. Reason: ${e.message || e}`);
+    }
   }
 }
 
+const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
+  ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
+  ' and choose CodeQL Query Server from the dropdown.';
+
 function createSyntheticResult(
   query: QueryInfo,
   db: DatabaseItem,

extensions/ql-vscode/src/sarif-parser.ts

@@ -0,0 +1,48 @@
+import * as Sarif from 'sarif';
+import * as fs from 'fs-extra';
+import { parser } from 'stream-json';
+import { pick } from 'stream-json/filters/Pick';
+import Assembler = require('stream-json/Assembler');
+import { chain } from 'stream-chain';
+
+const DUMMY_TOOL : Sarif.Tool = {driver: {name: ''}};
+
+export async function sarifParser(interpretedResultsPath: string) : Promise<Sarif.Log> {
+  try {
+    // Parse the SARIF file into token streams, filtering out only the results array.
+    const p = parser();
+    const pipeline = chain([
+      fs.createReadStream(interpretedResultsPath),
+      p,
+      pick({filter: 'runs.0.results'})
+    ]);
+
+    // Creates JavaScript objects from the token stream
+    const asm = Assembler.connectTo(pipeline);
+
+    // Returns a constructed Log object with the results or an empty array if no results were found.
+    // If the parser fails for any reason, it will reject the promise.
+    return await new Promise((resolve, reject) => {
+      pipeline.on('error', (error) => {
+        reject(error);
+      });
+      
+      asm.on('done', (asm) => {
+
+        const log : Sarif.Log = {
+          version: '2.1.0', 
+          runs: [
+            { 
+              tool: DUMMY_TOOL, 
+              results: asm.current ?? []
+            }
+          ]
+        };
+        
+        resolve(log);
+      });
+    });
+  } catch (err) {
+    throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
+  }
+}

extensions/ql-vscode/src/status-bar.ts

@@ -0,0 +1,48 @@
+import { ConfigurationChangeEvent, StatusBarAlignment, StatusBarItem, window, workspace } from 'vscode';
+import { CodeQLCliServer } from './cli';
+import { CANARY_FEATURES, CUSTOM_CODEQL_PATH_SETTING, DistributionConfigListener } from './config';
+import { DisposableObject } from './pure/disposable-object';
+
+/**
+ * Creates and manages a status bar item for codeql. THis item contains
+ * the current codeQL cli version as well as a notification if you are
+ * in canary mode
+ *
+ */
+export class CodeQlStatusBarHandler extends DisposableObject {
+
+  private readonly item: StatusBarItem;
+
+  constructor(private cli: CodeQLCliServer, distributionConfigListener: DistributionConfigListener) {
+    super();
+    this.item = window.createStatusBarItem(StatusBarAlignment.Right);
+    this.push(this.item);
+    this.push(workspace.onDidChangeConfiguration(this.handleDidChangeConfiguration, this));
+    this.push(distributionConfigListener.onDidChangeConfiguration(() => this.updateStatusItem()));
+    this.item.command = 'codeQL.copyVersion';
+    void this.updateStatusItem();
+  }
+
+  private handleDidChangeConfiguration(e: ConfigurationChangeEvent) {
+    if (
+      e.affectsConfiguration(CANARY_FEATURES.qualifiedName) ||
+      e.affectsConfiguration(CUSTOM_CODEQL_PATH_SETTING.qualifiedName)
+    ) {
+      // Wait a few seconds before updating the status item.
+      // This avoids a race condition where the cli's version
+      // is not updated before the status bar is refreshed.
+      setTimeout(() => this.updateStatusItem(), 3000);
+    }
+  }
+
+  private async updateStatusItem() {
+    const canary = CANARY_FEATURES.getValue() ? ' (Canary)' : '';
+    // since getting the version may take a few seconds, initialize with some
+    // meaningful text.
+    this.item.text = `CodeQL${canary}`;
+
+    const version = await this.cli.getVersion();
+    this.item.text = `CodeQL CLI v${version}${canary}`;
+    this.item.show();
+  }
+}

extensions/ql-vscode/src/telemetry.ts

@@ -0,0 +1,216 @@
+import { ConfigurationTarget, Extension, ExtensionContext, ConfigurationChangeEvent } from 'vscode';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEMETRY, LOG_TELEMETRY } from './config';
+import * as appInsights from 'applicationinsights';
+import { logger } from './logging';
+import { UserCancellationException } from './commandRunner';
+import { showBinaryChoiceWithUrlDialog } from './helpers';
+
+// Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
+const key = 'REPLACE-APP-INSIGHTS-KEY';
+
+export enum CommandCompletion {
+  Success = 'Success',
+  Failed = 'Failed',
+  Cancelled = 'Cancelled'
+}
+
+// Avoid sending the following data to App insights since we don't need it.
+const tagsToRemove = [
+  'ai.application.ver',
+  'ai.device.id',
+  'ai.cloud.roleInstance',
+  'ai.cloud.role',
+  'ai.device.id',
+  'ai.device.osArchitecture',
+  'ai.device.osPlatform',
+  'ai.device.osVersion',
+  'ai.internal.sdkVersion',
+  'ai.session.id'
+];
+
+const baseDataPropertiesToRemove = [
+  'common.os',
+  'common.platformversion',
+  'common.remotename',
+  'common.uikind',
+  'common.vscodesessionid'
+];
+
+export class TelemetryListener extends ConfigListener {
+
+  static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
+
+  private reporter?: TelemetryReporter;
+
+  constructor(
+    private readonly id: string,
+    private readonly version: string,
+    private readonly key: string,
+    private readonly ctx: ExtensionContext
+  ) {
+    super();
+  }
+
+  /**
+   * This function handles changes to relevant configuration elements. There are 2 configuration
+   * ids that this function cares about:
+   *
+   *     * `codeQL.telemetry.enableTelemetry`: If this one has changed, then we need to re-initialize
+   *        the reporter and the reporter may wind up being removed.
+   *     * `codeQL.canary`: A change here could possibly re-trigger a dialog popup.
+   *
+   * Note that the global telemetry setting also gate-keeps whether or not to send telemetry events
+   * to Application Insights. However, this gatekeeping happens inside of the vscode-extension-telemetry
+   * package. So, this does not need to be handled here.
+   *
+   * @param e the configuration change event
+   */
+  async handleDidChangeConfiguration(e: ConfigurationChangeEvent): Promise<void> {
+    if (
+      e.affectsConfiguration('codeQL.telemetry.enableTelemetry') ||
+      e.affectsConfiguration('telemetry.enableTelemetry')
+    ) {
+      await this.initialize();
+    }
+
+    // Re-request telemetry so that users can see the dialog again.
+    // Re-request if codeQL.canary is being set to `true` and telemetry
+    // is not currently enabled.
+    if (
+      e.affectsConfiguration('codeQL.canary') &&
+      CANARY_FEATURES.getValue() &&
+      !ENABLE_TELEMETRY.getValue()
+    ) {
+      await Promise.all([
+        this.setTelemetryRequested(false),
+        this.requestTelemetryPermission()
+      ]);
+    }
+  }
+
+  async initialize() {
+    await this.requestTelemetryPermission();
+
+    this.disposeReporter();
+
+    if (ENABLE_TELEMETRY.getValue<boolean>()) {
+      this.createReporter();
+    }
+  }
+
+  private createReporter() {
+    this.reporter = new TelemetryReporter(
+      this.id,
+      this.version,
+      this.key,
+      /* anonymize stack traces */ true
+    );
+    this.push(this.reporter);
+
+    const client = (this.reporter as any).appInsightsClient as appInsights.TelemetryClient;
+    if (client) {
+      // add a telemetry processor to delete unwanted properties
+      client.addTelemetryProcessor((envelope: any) => {
+        tagsToRemove.forEach(tag => delete envelope.tags[tag]);
+        const baseDataProperties = (envelope.data as any)?.baseData?.properties;
+        if (baseDataProperties) {
+          baseDataPropertiesToRemove.forEach(prop => delete baseDataProperties[prop]);
+        }
+
+        if (LOG_TELEMETRY.getValue<boolean>()) {
+          void logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
+        }
+        return true;
+      });
+    }
+  }
+
+  dispose() {
+    super.dispose();
+    void this.reporter?.dispose();
+  }
+
+  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+    if (!this.reporter) {
+      return;
+    }
+    const status = !error
+      ? CommandCompletion.Success
+      : error instanceof UserCancellationException
+        ? CommandCompletion.Cancelled
+        : CommandCompletion.Failed;
+
+    const isCanary = (!!CANARY_FEATURES.getValue<boolean>()).toString();
+
+    this.reporter.sendTelemetryEvent(
+      'command-usage',
+      {
+        name,
+        status,
+        isCanary
+      },
+      { executionTime }
+    );
+  }
+
+  /**
+   * Displays a popup asking the user if they want to enable telemetry
+   * for this extension.
+   */
+  async requestTelemetryPermission() {
+    if (!this.wasTelemetryRequested()) {
+      // if global telemetry is disabled, avoid showing the dialog or making any changes
+      let result = undefined;
+      if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
+        // Extension won't start until this completes.
+        result = await showBinaryChoiceWithUrlDialog(
+          'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?',
+          'https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code'
+        );
+      }
+      if (result !== undefined) {
+        await Promise.all([
+          this.setTelemetryRequested(true),
+          ENABLE_TELEMETRY.updateValue<boolean>(result, ConfigurationTarget.Global),
+        ]);
+      }
+    }
+  }
+
+  /**
+   * Exposed for testing
+   */
+  get _reporter() {
+    return this.reporter;
+  }
+
+  private disposeReporter() {
+    if (this.reporter) {
+      void this.reporter.dispose();
+      this.reporter = undefined;
+    }
+  }
+
+  private wasTelemetryRequested(): boolean {
+    return !!this.ctx.globalState.get<boolean>('telemetry-request-viewed');
+  }
+
+  private async setTelemetryRequested(newValue: boolean): Promise<void> {
+    await this.ctx.globalState.update('telemetry-request-viewed', newValue);
+  }
+}
+
+
+/**
+ * The global Telemetry instance
+ */
+export let telemetryListener: TelemetryListener;
+
+export async function initializeTelemetry(extension: Extension<any>, ctx: ExtensionContext): Promise<void> {
+  telemetryListener = new TelemetryListener(extension.id, extension.packageJSON.version, key, ctx);
+  // do not await initialization, since doing so will sometimes cause a modal popup.
+  // this is a particular problem during integration tests, which will hang if a modal popup is displayed.
+  void telemetryListener.initialize();
+  ctx.subscriptions.push(telemetryListener);
+}

extensions/ql-vscode/src/test-adapter.ts

@@ -1,3 +1,4 @@
+import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as vscode from 'vscode';
 import {
@@ -15,11 +16,11 @@ import {
 import { TestAdapterRegistrar } from 'vscode-test-adapter-util';
 import { QLTestFile, QLTestNode, QLTestDirectory, QLTestDiscovery } from './qltest-discovery';
 import { Event, EventEmitter, CancellationTokenSource, CancellationToken } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
-import { QLPackDiscovery } from './qlpack-discovery';
+import { DisposableObject } from './pure/disposable-object';
 import { CodeQLCliServer } from './cli';
-import { getOnDiskWorkspaceFolders } from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
 import { testLogger } from './logging';
+import { DatabaseItem, DatabaseManager } from './databases';
 
 /**
  * Get the full path of the `.expected` file for the specified QL test.
@@ -58,13 +59,13 @@ function getTestOutputFile(testPath: string, extension: string): string {
  * A factory service that creates `QLTestAdapter` objects for workspace folders on demand.
  */
 export class QLTestAdapterFactory extends DisposableObject {
-  constructor(testHub: TestHub, cliServer: CodeQLCliServer) {
+  constructor(testHub: TestHub, cliServer: CodeQLCliServer, databaseManager: DatabaseManager) {
     super();
 
     // this will register a QLTestAdapter for each WorkspaceFolder
     this.push(new TestAdapterRegistrar(
       testHub,
-      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer)
+      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer, databaseManager)
     ));
   }
 }
@@ -82,7 +83,6 @@ function changeExtension(p: string, ext: string): string {
  * Test adapter for QL tests.
  */
 export class QLTestAdapter extends DisposableObject implements TestAdapter {
-  private readonly qlPackDiscovery: QLPackDiscovery;
   private readonly qlTestDiscovery: QLTestDiscovery;
   private readonly _tests = this.push(
     new EventEmitter<TestLoadStartedEvent | TestLoadFinishedEvent>());
@@ -93,13 +93,12 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
 
   constructor(
     public readonly workspaceFolder: vscode.WorkspaceFolder,
-    private readonly cliServer: CodeQLCliServer
+    private readonly cliServer: CodeQLCliServer,
+    private readonly databaseManager: DatabaseManager
   ) {
     super();
 
-    this.qlPackDiscovery = this.push(new QLPackDiscovery(workspaceFolder, cliServer));
-    this.qlTestDiscovery = this.push(new QLTestDiscovery(this.qlPackDiscovery, workspaceFolder, cliServer));
-    this.qlPackDiscovery.refresh();
+    this.qlTestDiscovery = this.push(new QLTestDiscovery(workspaceFolder, cliServer));
     this.qlTestDiscovery.refresh();
 
     this.push(this.qlTestDiscovery.onDidChangeTests(this.discoverTests, this));
@@ -160,20 +159,20 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
   private discoverTests(): void {
     this._tests.fire({ type: 'started' } as TestLoadStartedEvent);
 
-    const testDirectories = this.qlTestDiscovery.testDirectories;
-    const children = testDirectories.map(
-      testDirectory => QLTestAdapter.createTestSuiteInfo(testDirectory, testDirectory.name)
-    );
-    const testSuite: TestSuiteInfo = {
-      type: 'suite',
-      label: 'CodeQL',
-      id: '.',
-      children
-    };
-
+    const testDirectory = this.qlTestDiscovery.testDirectory;
+    let testSuite: TestSuiteInfo | undefined;
+    if (testDirectory?.children.length) {
+      const children = QLTestAdapter.createTestOrSuiteInfos(testDirectory.children);
+      testSuite = {
+        type: 'suite',
+        label: 'CodeQL',
+        id: testDirectory.path,
+        children
+      };
+    }
     this._tests.fire({
       type: 'finished',
-      suite: children.length > 0 ? testSuite : undefined
+      suite: testSuite
     } as TestLoadFinishedEvent);
   }
 
@@ -186,19 +185,86 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
     testLogger.outputChannel.show(true);
 
     this.runningTask = this.track(new CancellationTokenSource());
+    const token = this.runningTask.token;
 
     this._testStates.fire({ type: 'started', tests: tests } as TestRunStartedEvent);
 
+    const currentDatabaseUri = this.databaseManager.currentDatabaseItem?.databaseUri;
+    const databasesUnderTest: DatabaseItem[] = [];
+    for (const database of this.databaseManager.databaseItems) {
+      for (const test of tests) {
+        if (await database.isAffectedByTest(test)) {
+          databasesUnderTest.push(database);
+          break;
+        }
+      }
+    }
+
+    await this.removeDatabasesBeforeTests(databasesUnderTest, token);
     try {
-      await this.runTests(tests, this.runningTask.token);
-    }
-    catch (e) {
-      /**/
+      await this.runTests(tests, token);
+    } catch (e) {
+      // CodeQL testing can throw exception even in normal scenarios. For example, if the test run
+      // produces no output (which is normal), the testing command would throw an exception on
+      // unexpected EOF during json parsing. So nothing needs to be done here - all the relevant
+      // error information (if any) should have already been written to the test logger.
     }
+    await this.reopenDatabasesAfterTests(databasesUnderTest, currentDatabaseUri, token);
+
     this._testStates.fire({ type: 'finished' } as TestRunFinishedEvent);
     this.clearTask();
   }
 
+  private async removeDatabasesBeforeTests(
+    databasesUnderTest: DatabaseItem[], token: vscode.CancellationToken): Promise<void> {
+    for (const database of databasesUnderTest) {
+      try {
+        await this.databaseManager
+          .removeDatabaseItem(_ => { /* no progress reporting */ }, token, database);
+      } catch (e) {
+        // This method is invoked from Test Explorer UI, and testing indicates that Test
+        // Explorer UI swallows any thrown exception without reporting it to the user.
+        // So we need to display the error message ourselves and then rethrow.
+        void showAndLogErrorMessage(`Cannot remove database ${database.name}: ${e}`);
+        throw e;
+      }
+    }
+  }
+
+  private async reopenDatabasesAfterTests(
+    databasesUnderTest: DatabaseItem[],
+    currentDatabaseUri: vscode.Uri | undefined,
+    token: vscode.CancellationToken): Promise<void> {
+    for (const closedDatabase of databasesUnderTest) {
+      const uri = closedDatabase.databaseUri;
+      if (await this.isFileAccessible(uri)) {
+        try {
+          const reopenedDatabase = await this.databaseManager
+            .openDatabase(_ => { /* no progress reporting */ }, token, uri);
+          await this.databaseManager.renameDatabaseItem(reopenedDatabase, closedDatabase.name);
+          if (currentDatabaseUri == uri) {
+            await this.databaseManager.setCurrentDatabaseItem(reopenedDatabase, true);
+          }
+        } catch (e) {
+          // This method is invoked from Test Explorer UI, and testing indicates that Test
+          // Explorer UI swallows any thrown exception without reporting it to the user.
+          // So we need to display the error message ourselves and then rethrow.
+          void showAndLogWarningMessage(`Cannot reopen database ${uri}: ${e}`);
+          throw e;
+        }
+      }
+    }
+  }
+
+  private async isFileAccessible(uri: vscode.Uri): Promise<boolean> {
+    try {
+      await fs.access(uri.fsPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
   private clearTask(): void {
     if (this.runningTask !== undefined) {
       const runningTask = this.runningTask;
@@ -209,7 +275,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
 
   public cancel(): void {
     if (this.runningTask !== undefined) {
-      testLogger.log('Cancelling test run...');
+      void testLogger.log('Cancelling test run...');
       this.runningTask.cancel();
       this.clearTask();
     }
@@ -221,10 +287,27 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
       cancellationToken: cancellationToken,
       logger: testLogger
     })) {
+      const state = event.pass
+        ? 'passed'
+        : event.messages?.length
+          ? 'errored'
+          : 'failed';
+      let message: string | undefined;
+      if (event.failureDescription || event.diff?.length) {
+        message = event.failureStage === 'RESULT'
+          ? ['', `${state}: ${event.test}`, event.failureDescription || event.diff?.join('\n'), ''].join('\n')
+          : ['', `${event.failureStage?.toLowerCase()} error: ${event.test}`, event.failureDescription || `${event.messages[0].severity}: ${event.messages[0].message}`, ''].join('\n');
+        void testLogger.log(message);
+      }
       this._testStates.fire({
         type: 'test',
-        state: event.pass ? 'passed' : 'failed',
-        test: event.test
+        state,
+        test: event.test,
+        message,
+        decorations: event.messages?.map(msg => ({
+          line: msg.position.line,
+          message: msg.message
+        }))
       });
     }
   }