Prepare for the 1.5.7 release (#1016)

Fix large sarif handling

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,7 +10,12 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
-**To Reproduce**
+**Version**
+The CodeQL and VS Code version in which the bug occurs.
+<!-- To copy version information for the CodeQL extension, click "CodeQL CLI vX.X.X" in the status bar at the bottom of the screen. 
+To copy detailed version information for VS Code itself, see https://code.visualstudio.com/docs/supporting/FAQ#_how-do-i-find-the-version. -->
+
+**To reproduce**
 Steps to reproduce the behavior.
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/new-extension-release.md

@@ -1,18 +0,0 @@
----
-name: New extension release
-about: Create an issue with a checklist for the release steps (write access required
-  for the steps)
-title: Release Checklist for version xx.xx.xx
-labels: ''
-assignees: ''
-
----
-
-- [ ] Update this issue title to refer to the version of the release
-- [ ] Trigger a release build on Actions by adding a new tag on branch `main` of the format `vxx.xx.xx`
-- [ ] Monitor the status of the release build in the `Release` workflow in the Actions tab.
-- [ ] Download the VSIX from the draft GitHub release that is created when the release build finishes.
-- [ ] Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
-- [ ] Click the `...` menu in the CodeQL row and click **Update**.
-- [ ] Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-- [ ] Publish the draft GitHub release and confirm the new release is marked as the latest release at https://github.com/github/vscode-codeql/releases.

.github/pull_request_template.md

@@ -9,4 +9,4 @@ Replace this with a description of the changes your pull request makes.
 
 - [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
 - [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] `@github/docs-content-dsp` has been cc'd in all issues for UI or other user-facing changes made by this pull request.
+- [ ] `@github/docs-content-codeql` has been cc'd in all issues for UI or other user-facing changes made by this pull request.

.github/workflows/codeql.yml

@@ -2,24 +2,30 @@ name: "Code Scanning - CodeQL"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
-    - cron: '0 0 * * 0'
+    - cron: '21 17 * * 0'
 
 jobs:
   codeql:
-    strategy:
-      fail-fast: false
     runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      security-events: write
+      pull-requests: read
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@main
         with:
             languages: javascript
             config-file: ./.github/codeql/codeql-config.yml
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@main

.github/workflows/main.yml

@@ -1,6 +1,8 @@
 name: Build Extension
 on:
+  workflow_dispatch:
   pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
   push:
     branches:
       - main
@@ -49,9 +51,26 @@ jobs:
           name: vscode-codeql-extension
           path: artifacts
 
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST"
+
   test:
     name: Test
     runs-on: ${{ matrix.os }}
+    needs: [find-nightly]
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
@@ -84,26 +103,16 @@ jobs:
         run: |
           npm run lint
 
-      - name: Install CodeQL
-        run: |
-          mkdir codeql-home
-          curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
-          unzip -q -o codeql-home/codeql.zip -d codeql-home
-          unzip -q -o codeql-home/codeql.zip codeql/codeql.exe -d codeql-home
-          rm codeql-home/codeql.zip
-        shell: bash
-
       - name: Run unit tests (Linux)
         working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
-          CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
+          npm run test
 
       - name: Run unit tests (Windows)
         if: matrix.os == 'windows-latest'
         working-directory: extensions/ql-vscode
         run: |
-          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
@@ -122,12 +131,14 @@ jobs:
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}
+    needs: [find-nightly]
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.2.6', 'v2.3.3', 'v2.4.2']
+        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.2', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
       TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:
@@ -150,10 +161,26 @@ jobs:
           npm run build
         shell: bash
 
+      - name: Decide on ref of CodeQL repo
+        id: choose-ref
+        shell: bash
+        run: |
+          if [[ "${{ matrix.version }}" == "nightly" ]]
+          then
+            REF="codeql-cli/latest"
+          elif [[ "${{ matrix.version }}" == "v2.2.6" || "${{ matrix.version }}" == "v2.3.3" ]]
+          then
+            REF="codeql-cli/v2.4.5"
+          else
+            REF="codeql-cli/${{ matrix.version }}"
+          fi
+          echo "::set-output name=ref::$REF"
+
       - name: Checkout QL
         uses: actions/checkout@v2
         with:
           repository: github/codeql
+          ref: ${{ steps.choose-ref.outputs.ref }}
           path: codeql
 
       - name: Run CLI tests (Linux)

.github/workflows/release.yml

@@ -6,10 +6,6 @@
 
 name: Release
 on:
-  pull_request:
-    paths:
-      - '**/workflows/release.yml'
-
   workflow_dispatch:
 
   push:
@@ -129,6 +125,7 @@ jobs:
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
           base: main
+          draft: true
 
   vscode-publish:
     name: Publish to VS Code Marketplace

.vscode/extensions.json

@@ -1,9 +1,9 @@
 {
-  // See http://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
+  // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
-    "eamodio.tsl-problem-matcher",
+    "amodio.tsl-problem-matcher",
     "dbaeumer.vscode-eslint",
     "eternalphane.tsfmt-vscode"
   ],

.vscode/launch.json

@@ -21,6 +21,9 @@
         // change to 'true' debug the IDE or Query servers
         "IDE_SERVER_JAVA_DEBUG": "false",
         "QUERY_SERVER_JAVA_DEBUG": "false",
+        "CLI_SERVER_JAVA_DEBUG": "false",
+        // Uncomment to set the JAVA_HOME for the codeql instance to use
+        // "CODEQL_JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk-12.0.1.jdk/Contents/Home"
       }
     },
     {
@@ -93,6 +96,19 @@
         // available in the workspace for the tests.
         // "${workspaceRoot}/../codeql"
       ],
+      "env": {
+        // Optionally, set the version to use for the integration tests.
+        // Use "nightly" to use the latest nightly build.
+        // "CLI_VERSION": "2.7.0",
+
+        // If CLI_VERSION is set to nightly, set this to the url of the nightly build.
+        // "NIGHTLY_URL": "some url to grab the nightly build",
+
+        // Optionally, add a path to the codeql executable to be used during these tests.
+        // If not specified, one will be downloaded automatically.
+        // This option overrides the CLI_VERSION option.
+        // "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
+      },
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [

.vscode/settings.json

@@ -22,7 +22,8 @@
     "common/temp": true,
     "**/.vscode-test": true
   },
-  "typescript.tsdk": "./common/temp/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
+  "typescript.tsdk": "./extensions/ql-vscode/node_modules/typescript/lib", // we want to use the TS server from our node_modules folder to control its version
+  "typescript.enablePromptUseWorkspaceTsdk": true,
   "eslint.validate": [
     "javascript",
     "javascriptreact",

CODEOWNERS

@@ -0,0 +1 @@
+**/* @github/codeql-vscode-reviewers

CONTRIBUTING.md

@@ -25,12 +25,11 @@ Here are a few things you can do that will increase the likelihood of your pull
 * Follow the [style guide][style].
 * Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
 * Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
-* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
 ## Setting up a local build
 
-Make sure you have a fairly recent version of vscode (>1.32) and are using nodejs
-version >=v10.13.0. (Tested on v10.15.1 and v10.16.0).
+Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
 
 ### Installing all packages
 
@@ -57,7 +56,8 @@ We recommend that you keep `npm run watch` running in the backgound and you only
 
 1. on first checkout
 2. whenever any of the non-TypeScript resources have changed
-3. on any change to files included in the webview
+3. on any change to files included in one of the webviews
+   - **Important**: This is easy to forget. You must explicitly run `npm run build` whenever one of the files in the webview is changed. These are the files in the `src/view` and `src/compare/view` folders.
 
 ### Installing the extension
 
@@ -77,9 +77,9 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
-### Running the unit/integration tests
+### Running the unit tests and integration tests that do not require a CLI instance
 
-Ensure the `CODEQL_PATH` environment variable is set to point to the `codeql` cli executable.
+Unit tests and many integration tests do not require a copy of the CodeQL CLI.
 
 Outside of vscode, run:
 
@@ -89,6 +89,16 @@ npm run test && npm run integration
 
 Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
 
+Only the _With CLI_ tests require a CLI instance to run. See below on how to do that.
+
+Running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+### Running the integration tests
+
+The _Launch Integration Tests - With CLI_ tests require a CLI instance in order to run. There are several environment variables you can use to configure this.
+
+From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ uncomment and change the environment variables appropriate for your purpose.
+
 ## Releasing (write access required)
 
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.

README.md

@@ -16,7 +16,6 @@ To see what has changed in the last few versions of the extension, see the [Chan
 * Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
 
-
 ## Project goals and scope
 
 This project will track new feature development in CodeQL and, whenever appropriate, bring that functionality to the Visual Studio Code experience.

extensions/ql-vscode/.eslintrc.js

@@ -22,8 +22,10 @@ module.exports = {
       },
     ],
     "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/explicit-module-boundary-types": "off",
     "@typescript-eslint/no-non-null-assertion": "off",
     "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-floating-promises": [ "error", { ignoreVoid: true } ],
     "prefer-const": ["warn", { destructuring: "all" }],
     indent: "off",
     "@typescript-eslint/indent": "off",

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,112 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.5.7 - 23 November 2021
+
+- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979)
+- Support large SARIF results files (>4GB) without crashing VS Code. [#1004](https://github.com/github/vscode-codeql/pull/1004)
+- Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
+- Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
+- Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)
+- Allow case-insensitive project slugs for GitHub repositories when adding a CodeQL database from LGTM. [#978](https://github.com/github/vscode-codeql/pull/961)
+- Add a _CodeQL: Preview Query Help_ command to generate Markdown previews of `.qhelp` query help files. This command should only be run in trusted workspaces. See https://codeql.github.com/docs/codeql-cli/testing-query-help-files for more information about query help. [#988](https://github.com/github/vscode-codeql/pull/988)
+- Make "Open Referenced File" command accessible from the active editor menu. [#989](https://github.com/github/vscode-codeql/pull/989)
+- Fix a bug where result set names in the result set drop-down were disappearing when viewing a sorted table. [#1007](https://github.com/github/vscode-codeql/pull/1007)
+- Allow query result locations with 0 as the end column value. These are treated as the first column in the line. [#1002](https://github.com/github/vscode-codeql/pull/1002)
+
+## 1.5.6 - 07 October 2021
+
+- Add progress messages to LGTM download option. This makes the two-step process (selecting a project, then selecting a language) more clear. [#960](https://github.com/github/vscode-codeql/pull/960)
+- Remove line about selecting a language from the dropdown when downloading database from LGTM. This makes the download progress visible when the popup is not expanded. [#957](https://github.com/github/vscode-codeql/pull/957)
+- Fix a bug where copying the version information fails when a CodeQL CLI cannot be found. [#958](https://github.com/github/vscode-codeql/pull/958)
+- Avoid a race condition when deleting databases that can cause occasional errors. [#959](https://github.com/github/vscode-codeql/pull/959)
+- Update CodeQL logos. [#965](https://github.com/github/vscode-codeql/pull/965)
+
+## 1.5.5 - 08 September 2021
+
+- Fix bug where a query is sometimes run before the file is saved. [#947](https://github.com/github/vscode-codeql/pull/947)
+- Fix broken contextual queries, including _View AST_. [#949](https://github.com/github/vscode-codeql/pull/949)
+
+## 1.5.4 - 02 September 2021
+
+- Add support for filename pattern in history view. [#930](https://github.com/github/vscode-codeql/pull/930)
+- Add an option _View Results (CSV)_ to view the results of a non-alert query. The existing options for alert queries have been renamed to _View Alerts_ to avoid confusion. [#929](https://github.com/github/vscode-codeql/pull/929)
+- Allow users to specify the number of paths to display for each alert. [#931](https://github.com/github/vscode-codeql/pull/931)
+- Adjust pagination controls in _CodeQL Query Results_ to always be visible [#936](https://github.com/github/vscode-codeql/pull/936)
+- Fix bug where _View AST_ fails due to recent refactoring in the standard library and query packs. [#939](https://github.com/github/vscode-codeql/pull/939)
+
+## 1.5.3 - 18 August 2021
+
+- Add a command _CodeQL: Run Query on Multiple Databases_, which lets users select multiple databases to run a query on. [#898](https://github.com/github/vscode-codeql/pull/898)
+- Autodetect what language a query targets. This refines the _CodeQL: Run Query on Multiple Databases_ command to only show relevant databases. [#915](https://github.com/github/vscode-codeql/pull/915)
+- Adjust test log output to display diffs only when comparing failed test results with expected test results. [#920](https://github.com/github/vscode-codeql/pull/920)
+
+## 1.5.2 - 13 July 2021
+
+- Add the _Add Database Source to Workspace_ command to the right-click context menu in the databases view. This lets users re-add a database's source folder to the workspace and browse the source code. [#891](https://github.com/github/vscode-codeql/pull/891)
+- Fix markdown rendering in the description of the `codeQL.cli.executablePath` setting. [#908](https://github.com/github/vscode-codeql/pull/908)
+- Fix the _Open Query Results_ command in the query history view. [#909](https://github.com/github/vscode-codeql/pull/909)
+
+## 1.5.1 - 23 June 2021
+
+No user facing changes.
+
+## 1.5.0 - 14 June 2021
+
+- Display CodeQL CLI version being downloaded during an upgrade. [#862](https://github.com/github/vscode-codeql/pull/862)
+- Display a helpful message and link to documentation when a query produces no results. [#866](https://github.com/github/vscode-codeql/pull/866)
+- Refresh test databases automatically after a test run. [#868](https://github.com/github/vscode-codeql/pull/868)
+- Allow users to specify a custom directory for storing query server logs (`codeQL.runningQueries.customLogDirectory`). The extension will not delete these logs automatically. [#863](https://github.com/github/vscode-codeql/pull/863)
+- Support the VS Code [Workspace Trust feature](https://code.visualstudio.com/docs/editor/workspace-trust). This extension is now enabled in untrusted workspaces, but it restricts commands that contain arbitrary paths. [#861](https://github.com/github/vscode-codeql/pull/861)
+- Allow the `codeQL.cli.executablePath` configuration setting to be set in workspace-scoped configuration files. This means that each workspace can now specify its own CodeQL CLI compiler, a feature that is unblocked due to implementing Workspace Trust. [#861](https://github.com/github/vscode-codeql/pull/861)
+
+## 1.4.8 - 05 May 2021
+
+- Copy version information to the clipboard when a user clicks the CodeQL section of the status bar. [#845](https://github.com/github/vscode-codeql/pull/845)
+- Ensure changes in directories that contain tests will be properly updated in the test explorer. [#846](https://github.com/github/vscode-codeql/pull/846)
+- Remind users to choose a language when downloading a database from LGTM. [#852](https://github.com/github/vscode-codeql/pull/852)
+
+## 1.4.7 - 23 April 2021
+
+- Fix a bug that prevented the results view from being loaded. [#842](https://github.com/github/vscode-codeql/pull/842)
+
+## 1.4.6 - 21 April 2021
+
+- Avoid showing an error popup when running a query with `@kind table` metadata. [#814](https://github.com/github/vscode-codeql/pull/814)
+- Add an option to jump from a .qlref file to the .ql file it references. [#815](https://github.com/github/vscode-codeql/pull/815)
+- Avoid opening the results panel when a database is deleted. [#831](https://github.com/github/vscode-codeql/pull/831)
+- Forward all query metadata to the CLI when interpreting results. [#838](https://github.com/github/vscode-codeql/pull/838)
+
+## 1.4.5 - 22 March 2021
+
+- Avoid showing an error popup when user runs a query without `@kind` metadata. [#801](https://github.com/github/vscode-codeql/pull/801)
+- Fix running of tests when the `ms-python` extension is installed. [#803](https://github.com/github/vscode-codeql/pull/803)
+
+## 1.4.4 - 19 March 2021
+
+- Introduce evaluator options for saving intermediate results to the disk cache (`codeQL.runningQueries.saveCache`) and for limiting the size of this cache (`codeQL.runningQueries.cacheSize`). [#778](https://github.com/github/vscode-codeql/pull/778)
+- Respect the `codeQL.runningQueries.numberOfThreads` setting when creating SARIF files during result interpretation. [#771](https://github.com/github/vscode-codeql/pull/771)
+- Allow using raw LGTM project slugs for fetching LGTM databases. [#769](https://github.com/github/vscode-codeql/pull/769)
+- Better error messages when BQRS interpretation fails to produce SARIF. [#770](https://github.com/github/vscode-codeql/pull/770)
+- Implement sorting of the query history view by name, date, and results count. [#777](https://github.com/github/vscode-codeql/pull/777)
+- Add a configuration option to pass additional arguments to the CLI when running tests. [#785](https://github.com/github/vscode-codeql/pull/785)
+- Introduce option to view query results as CSV. [#784](https://github.com/github/vscode-codeql/pull/784)
+- Add some snippets for commonly used QL statements. [#782](https://github.com/github/vscode-codeql/pull/782)
+- More descriptive error messages on QL test failures. [#788](https://github.com/github/vscode-codeql/pull/788)
+
+## 1.4.3 - 22 February 2021
+
+- Avoid displaying an error when removing orphaned databases and the storage folder does not exist. [#748](https://github.com/github/vscode-codeql/pull/748)
+- Add better error messages when AST Viewer is unable to create an AST. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Cache AST viewing operations so that subsequent calls to view the AST of a single file will be extremely fast. [#753](https://github.com/github/vscode-codeql/pull/753)
+- Ensure CodeQL version in status bar updates correctly when version changes. [#754](https://github.com/github/vscode-codeql/pull/754)
+- Avoid deleting the quick query file when it is re-opened. [#747](https://github.com/github/vscode-codeql/pull/747)
+
+## 1.4.2 - 2 February 2021
+
+- Add a status bar item for the CodeQL CLI to show the current version. [#741](https://github.com/github/vscode-codeql/pull/741)
+- Fix version constraint for flagging CLI support of non-destructive updates. [#744](https://github.com/github/vscode-codeql/pull/744)
+- Add a _More Information_ button in the telemetry popup that opens the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) in a browser tab. [#742](https://github.com/github/vscode-codeql/pull/742)
+
 ## 1.4.1 - 29 January 2021
 
 - Reword the telemetry modal dialog box. [#738](https://github.com/github/vscode-codeql/pull/738)
@@ -8,7 +115,7 @@
 
 - Fix bug where databases are not reregistered when the query server restarts. [#734](https://github.com/github/vscode-codeql/pull/734)
 - Fix bug where upgrade requests were erroneously being marked as failed. [#734](https://github.com/github/vscode-codeql/pull/734)
-- On a strictly opt-in basis, collect anonymized usage data from the VS Code extension, helping improve CodeQL's usability and performance. See [TELEMETRY.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/TELEMETRY.md) for more information on exactly what data is collected and what it is used for. [#611](https://github.com/github/vscode-codeql/pull/611)
+- On a strictly opt-in basis, collect anonymized usage data from the VS Code extension, helping improve CodeQL's usability and performance. See the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code) for more information on exactly what data is collected and what it is used for. [#611](https://github.com/github/vscode-codeql/pull/611)
 
 ## 1.3.10 - 20 January 2021
 

extensions/ql-vscode/README.md

@@ -1,6 +1,6 @@
 # CodeQL extension for Visual Studio Code
 
-This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://help.semmle.com/codeql) and allows you to easily find problems in codebases. In particular, the extension:
+This project is an extension for Visual Studio Code that adds rich language support for [CodeQL](https://codeql.github.com/docs/) and allows you to easily find problems in codebases. In particular, the extension:
 
 - Enables you to use CodeQL to query databases generated from source code.
 - Shows the flow of data through the results of path queries, which is essential for triaging security results.
@@ -12,7 +12,7 @@ To see what has changed in the last few versions of the extension, see the [Chan
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
-For information about other configurations, see the separate [CodeQL help](https://help.semmle.com/codeql/codeql-for-vscode.html).
+For information about other configurations, see the separate [CodeQL help](https://codeql.github.com/docs/codeql-for-visual-studio-code/).
 
 ### Quick start: Installing and configuring the extension
 
@@ -40,9 +40,9 @@ The CodeQL extension requires a minimum of Visual Studio Code 1.39. Older versio
 
 ### Checking access to the CodeQL CLI
 
-The extension uses the [CodeQL CLI](https://help.semmle.com/codeql/codeql-cli.html) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
+The extension uses the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
 
-If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#configuring-access-to-the-codeql-cli).
+If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#configuring-access-to-the-codeql-cli).
 
 If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Log** in the **Output** view for any error messages.
 
@@ -52,7 +52,7 @@ When you're working with CodeQL, you need access to the standard CodeQL librarie
 Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
-For information about configuring an existing workspace for CodeQL, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/setting-up.html#updating-an-existing-workspace-for-codeql).
+For information about configuring an existing workspace for CodeQL, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#updating-an-existing-workspace-for-codeql).
 
 ## Upgrading CodeQL standard libraries
 
@@ -75,7 +75,7 @@ You can find all the commands contributed by the extension in the Command Palett
 
 ### Importing a database from LGTM
 
-While you can use the [CodeQL CLI to create your own databases](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html), the simplest way to start is by downloading a database from LGTM.com.
+While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from LGTM.com.
 
 1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
 1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
@@ -100,11 +100,11 @@ If there are any problems running a query, a notification is displayed in the bo
 
 ## What next?
 
-For more information about the CodeQL extension, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode.html). Otherwise, you could:
+For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:
 
-- [Create a database for a different codebase](https://help.semmle.com/codeql/codeql-cli/procedures/create-codeql-database.html).
+- [Create a database for a different codebase](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/).
 - [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
-- [Learn more about CodeQL](https://help.semmle.com/QL/learn-ql/).
+- [Learn more about CodeQL](https://codeql.github.com/docs/).
 - [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
 
 ## License
@@ -113,4 +113,4 @@ The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the
 
 ## Data and Telemetry
 
-If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. Please see [telemetry](TELEMETRY.md) for more information.
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information about telemetry, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code).

extensions/ql-vscode/TELEMETRY.md

@@ -1,47 +0,0 @@
-# Telemetry in the CodeQL extension for VS Code
-
-If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days.
-
-## Why do you collect data?
-
-GitHub collects aggregated, anonymous usage data and metrics to help us improve CodeQL for VS Code. IP addresses and installation IDs are collected only to ensure that anonymous data is not duplicated during aggregation.
-
-## What data is collected
-
-If you opt in, GitHub collects the following information related to the usage of the extension. The data collected are:
-
-- The identifiers of any CodeQL-related [VS Code commands](https://code.visualstudio.com/docs/getstarted/tips-and-tricks#_command-palette) that are run
-- For each command: the timestamp, time taken, and whether or not the command completed successfully
-- VS Code and extension version
-- Randomly generated GUID that uniquely identifies a CodeQL extension installation. (Discarded before aggregation.)
-- IP address of the client sending the telemetry data. (Discarded before aggregation.)
-- Whether or not the `codeQL.canary` setting is enabled and set to `true`
-
-## How long will data be retained?
-
-IP address and GUIDs will be retained for a maximum of 30 days. Anonymous, aggregated data that includes command identifiers, run times, and timestamps will be retained for a maximum of 180 days.
-
-## Who will have access to this data?
-
-IP address and GUIDs will only be available to the core developers of CodeQL. Aggregated data will be available to GitHub employees.
-
-## What data is **NOT** collected?
-
-We only collect the minimal amount of data we need to answer the questions about how our users are experiencing this product. To that end, we do not collect the following information:
-
-- No GitHub user ID
-- No CodeQL database names or contents
-- No contents of CodeQL queries
-- No filesystem paths.
-
-## How do I disable telemetry reporting?
-
-When telemetry collection is disabled, no data will be sent to GitHub servers.
-
-You can disable telemetry collection by setting `codeQL.telemetry.enableTelemetry` to `false` in [your settings](https://code.visualstudio.com/docs/getstarted/settings#_settings-editor). Telemetry collection is _disabled_ by default.
-
-Additionally, telemetry collection will be disabled if the global `telemetry.enableTelemetry` setting is set to `false`. For more information on global telemetry collection, see [Microsoft’s documentation](https://code.visualstudio.com/docs/supporting/faq#_how-to-disable-telemetry-reporting).
-
-## More information
-
-See GitHub's [Privacy Statement](https://docs.github.com/en/free-pro-team@latest/github/site-policy/github-privacy-statement) and [Terms of Service](https://docs.github.com/en/free-pro-team@latest/github/site-policy/github-terms-of-service) for more information.

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -13,6 +13,7 @@ const packageFiles = [
   'CHANGELOG.md',
   'README.md',
   'language-configuration.json',
+  'snippets.json',
   'media',
   'node_modules',
   'out'

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json.schemastore.org/tsconfig",
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "declaration": true,
     "strict": true,

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -13,7 +13,10 @@ export const config: webpack.Configuration = {
   },
   devtool: 'inline-source-map',
   resolve: {
-    extensions: ['.js', '.ts', '.tsx', '.json']
+    extensions: ['.js', '.ts', '.tsx', '.json'],
+    fallback: {
+      path: require.resolve('path-browserify')
+    }
   },
   module: {
     rules: [

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -6,21 +6,23 @@ export function compileView(cb: (err?: Error) => void) {
     if (error) {
       cb(error);
     }
-    console.log(stats.toString({
-      errorDetails: true,
-      colors: true,
-      assets: false,
-      builtAt: false,
-      version: false,
-      hash: false,
-      entrypoints: false,
-      timings: false,
-      modules: false,
-      errors: true
-    }));
-    if (stats.hasErrors()) {
-      cb(new Error('Compilation errors detected.'));
-      return;
+    if (stats) {
+      console.log(stats.toString({
+        errorDetails: true,
+        colors: true,
+        assets: false,
+        builtAt: false,
+        version: false,
+        hash: false,
+        entrypoints: false,
+        timings: false,
+        modules: false,
+        errors: true
+      }));
+      if (stats.hasErrors()) {
+        cb(new Error('Compilation errors detected.'));
+        return;
+      }
     }
 
     cb();

extensions/ql-vscode/media/dark/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" fill="#C5C5C5" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" fill="#C5C5C5"/>
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/light/sort-num.svg

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
+<g>
+	<g>
+		<g>
+			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" />
+			<path d="m 259.24622,341.40906 v -32.34375 q 13.35937,6.32812 27.07031,9.66797 13.71094,3.33984 26.89453,3.33984 35.15625,0 53.61328,-23.55469 18.63282,-23.73047 21.26953,-71.89453 -10.19531,15.11719 -25.83984,23.20313 -15.64453,8.08593 -34.62891,8.08593 -39.375,0 -62.40234,-23.73046 -22.85156,-23.90625 -22.85156,-65.21485 0,-40.42969 23.90625,-64.86328 23.90625,-24.433594 63.63281,-24.433594 45.52734,0 69.43359,34.980474 24.08204,34.80468 24.08204,101.25 0,62.05078 -29.53125,99.14062 -29.35547,36.91406 -79.10157,36.91406 -13.35937,0 -27.07031,-2.63672 -13.71094,-2.63671 -28.47656,-7.91015 z m 70.66406,-111.26953 q 23.90625,0 37.79297,-16.34766 14.0625,-16.34766 14.0625,-44.82422 0,-28.30078 -14.0625,-44.64844 -13.88672,-16.52343 -37.79297,-16.52343 -23.90625,0 -37.96875,16.52343 -13.88672,16.34766 -13.88672,44.64844 0,28.47656 13.88672,44.82422 14.0625,16.34766 37.96875,16.34766 z" />
+			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" />
+			<path d="M 35.300905,316.97546 H 93.308718 V 116.76062 L 30.203249,129.41687 V 97.07312 L 92.957155,84.41687 h 35.507815 v 232.55859 h 58.00781 v 29.88282 H 35.300905 Z" />
+		</g>
+	</g>
+</g>
+</svg>

extensions/ql-vscode/media/logo.svg

@@ -1,14 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="27px" height="16px" viewBox="0 0 27 16" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 59 (86127) - https://sketch.com -->
-    <title>Slice</title>
-    <desc>Created with Sketch.</desc>
-    <g id="light" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="QL" transform="translate(1.000000, 1.000000)">
-            <rect id="Rectangle-41" stroke="#2088FF" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" x="0" y="0" width="25" height="14" rx="2"></rect>
-            <line x1="17" y1="5" x2="19" y2="5" id="Stroke-15" stroke="#2088FF" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></line>
-            <line x1="17" y1="9" x2="21" y2="9" id="Stroke-15" stroke="#2088FF" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></line>
-            <path d="M8.85227273,7 C8.85227273,7.51894199 8.76988719,7.97537682 8.60511364,8.36931818 C8.44034009,8.76325955 8.21591051,9.08711994 7.93181818,9.34090909 L8.76420455,10.3863636 L7.61647727,10.3863636 L7.14772727,9.80965909 C6.83143781,9.92897787 6.49147909,9.98863636 6.12784091,9.98863636 C5.61079287,9.98863636 5.14678236,9.8712133 4.73579545,9.63636364 C4.32480855,9.40151398 4.00000119,9.06108178 3.76136364,8.61505682 C3.52272608,8.16903186 3.40340909,7.63068497 3.40340909,7 C3.40340909,6.36552713 3.52272608,5.8257598 3.76136364,5.38068182 C4.00000119,4.93560384 4.32480855,4.59611859 4.73579545,4.36221591 C5.14678236,4.12831322 5.61079287,4.01136364 6.12784091,4.01136364 C6.642995,4.01136364 7.10605855,4.12831322 7.51704545,4.36221591 C7.92803236,4.59611859 8.2533132,4.93560384 8.49289773,5.38068182 C8.73248226,5.8257598 8.85227273,6.36552713 8.85227273,7 Z M5.70170455,7.88636364 L6.74715909,7.88636364 L7.17897727,8.44034091 C7.31344764,8.27935526 7.41808675,8.07859969 7.49289773,7.83806818 C7.56770871,7.59753668 7.60511364,7.31818341 7.60511364,7 C7.60511364,6.38257267 7.47064528,5.91145996 7.20170455,5.58664773 C6.93276381,5.2618355 6.57481284,5.09943182 6.12784091,5.09943182 C5.68086898,5.09943182 5.32291801,5.2618355 5.05397727,5.58664773 C4.78503653,5.91145996 4.65056818,6.38257267 4.65056818,7 C4.65056818,7.61553338 4.78503653,8.08617261 5.05397727,8.41193182 C5.32291801,8.73769102 5.68086898,8.90056818 6.12784091,8.90056818 C6.23958389,8.90056818 6.34564344,8.89015162 6.44602273,8.86931818 L5.70170455,7.88636364 Z M10.1813315,10 L10.1813315,4 L11.4114451,4 L11.4114451,8.98579545 L13.9057633,8.98579545 L13.9057633,10 L10.1813315,10 Z" fill="#2088FF" fill-rule="nonzero"></path>
-        </g>
-    </g>
-</svg>
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M8.19789 8C8.19789 8.51894 8.1155 8.97538 7.95073 9.36932C7.78595 9.76326 7.56152 10.0871 7.27743 10.3409L8.10982 11.3864H6.96209L6.49334 10.8097C6.17705 10.929 5.83709 10.9886 5.47346 10.9886C4.95641 10.9886 4.4924 10.8712 4.08141 10.6364C3.67042 10.4015 3.34562 10.0611 3.10698 9.61506C2.86834 9.16903 2.74902 8.63068 2.74902 8C2.74902 7.36553 2.86834 6.82576 3.10698 6.38068C3.34562 5.9356 3.67042 5.59612 4.08141 5.36222C4.4924 5.12831 4.95641 5.01136 5.47346 5.01136C5.98861 5.01136 6.45167 5.12831 6.86266 5.36222C7.27365 5.59612 7.59893 5.9356 7.83851 6.38068C8.0781 6.82576 8.19789 7.36553 8.19789 8ZM5.04732 8.88636H6.09277L6.52459 9.44034C6.65906 9.27936 6.7637 9.0786 6.83851 8.83807C6.91332 8.59754 6.95073 8.31818 6.95073 8C6.95073 7.38257 6.81626 6.91146 6.54732 6.58665C6.27838 6.26184 5.92043 6.09943 5.47346 6.09943C5.02648 6.09943 4.66853 6.26184 4.39959 6.58665C4.13065 6.91146 3.99618 7.38257 3.99618 8C3.99618 8.61553 4.13065 9.08617 4.39959 9.41193C4.66853 9.73769 5.02648 9.90057 5.47346 9.90057C5.5852 9.90057 5.69126 9.89015 5.79164 9.86932L5.04732 8.88636ZM9.52695 11V5H10.7571V9.9858H13.2514V11H9.52695Z" fill="#24292F"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M13 1.5H3C2.17157 1.5 1.5 2.17157 1.5 3V13C1.5 13.8284 2.17157 14.5 3 14.5H13C13.8284 14.5 14.5 13.8284 14.5 13V3C14.5 2.17157 13.8284 1.5 13 1.5ZM3 0C1.34315 0 0 1.34315 0 3V13C0 14.6569 1.34315 16 3 16H13C14.6569 16 16 14.6569 16 13V3C16 1.34315 14.6569 0 13 0H3Z" fill="#24292F"/>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.4.1",
+  "version": "1.5.7",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,7 +13,7 @@
     "url": "https://github.com/github/vscode-codeql"
   },
   "engines": {
-    "vscode": "^1.43.0"
+    "vscode": "^1.57.0"
   },
   "categories": [
     "Programming Languages"
@@ -21,6 +21,16 @@
   "extensionDependencies": [
     "hbenl.vscode-test-explorer"
   ],
+  "capabilities": {
+    "untrustedWorkspaces": {
+      "supported": "limited",
+      "description": "Workspace trust is required to execute commands that can contain arbitrary paths.",
+      "restrictedConfigurations": [
+        "codeQL.cli.executablePath",
+        "codeQL.runningTests.additionalTestArguments"
+      ]
+    }
+  },
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
@@ -28,12 +38,15 @@
     "onView:codeQLAstViewer",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
+    "onCommand:codeQL.authenticateToGitHub",
     "onCommand:codeQLDatabases.chooseDatabaseFolder",
     "onCommand:codeQLDatabases.chooseDatabaseArchive",
     "onCommand:codeQLDatabases.chooseDatabaseInternet",
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.openReferencedFile",
+    "onCommand:codeQL.previewQueryHelp",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
@@ -106,15 +119,21 @@
         "path": "./out/syntaxes/dbscheme.tmLanguage.json"
       }
     ],
+    "snippets": [
+      {
+        "language": "ql",
+        "path": "./snippets.json"
+      }
+    ],
     "configuration": {
       "type": "object",
       "title": "CodeQL",
       "properties": {
         "codeQL.cli.executablePath": {
-          "scope": "machine",
+          "scope": "window",
           "type": "string",
           "default": "",
-          "description": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. This overrides all other CodeQL CLI settings."
+          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -123,6 +142,21 @@
           "maximum": 1024,
           "description": "Number of threads for running queries."
         },
+        "codeQL.runningQueries.saveCache": {
+          "type": "boolean",
+          "default": false,
+          "scope": "window",
+          "description": "Aggressively save intermediate results to the disk cache. This may speed up subsequent queries if they are similar. Be aware that using this option will greatly increase disk usage and initial evaluation time."
+        },
+        "codeQL.runningQueries.cacheSize": {
+          "type": [
+            "integer",
+            "null"
+          ],
+          "default": null,
+          "minimum": 1024,
+          "description": "Maximum size of the disk cache (in MB). Leave blank to allow the evaluator to automatically adjust the size of the disk cache based on the size of the codebase and the complexity of the queries being executed."
+        },
         "codeQL.runningQueries.timeout": {
           "type": [
             "integer",
@@ -147,6 +181,13 @@
           "default": false,
           "description": "Enable debug logging and tuple counting when running CodeQL queries. This information is useful for debugging query performance."
         },
+        "codeQL.runningQueries.maxPaths": {
+          "type": "integer",
+          "default": 4,
+          "minimum": 1,
+          "maximum": 256,
+          "markdownDescription": "Max number of paths to display for each alert found by a path query (`@kind path-problem`)."
+        },
         "codeQL.runningQueries.autoSave": {
           "type": "boolean",
           "default": false,
@@ -157,15 +198,29 @@
           "default": 20,
           "description": "Max number of simultaneous queries to run using the 'CodeQL: Run Queries' command."
         },
+        "codeQL.runningQueries.customLogDirectory": {
+          "type": [
+            "string",
+            null
+          ],
+          "default": null,
+          "description": "Path to a directory where the CodeQL extension should store query server logs. If empty, the extension stores logs in a temporary workspace folder and deletes the contents after each run."
+        },
         "codeQL.resultsDisplay.pageSize": {
           "type": "integer",
           "default": 200,
-          "description": "Number of queries displayed per page of the results view."
+          "description": "Max number of query results to display per page in the results view."
         },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "[%t] %q on %d - %s",
-          "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
+          "default": "%q on %d - %s, %r result count [%t]",
+          "markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
+        },
+        "codeQL.runningTests.additionalTestArguments": {
+          "scope": "window",
+          "type": "array",
+          "default": [],
+          "markdownDescription": "Additional command line arguments to pass to the CLI when [running tests](https://codeql.github.com/docs/codeql-cli/manual/test-run/). This setting should be an array of strings, each containing an argument to be passed."
         },
         "codeQL.runningTests.numberOfThreads": {
           "scope": "window",
@@ -179,22 +234,56 @@
           "type": "boolean",
           "default": false,
           "scope": "application",
-          "markdownDescription": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub. For more information, see [TELEMETRY.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/TELEMETRY.md)",
-          "description": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub."
+          "markdownDescription": "Specifies whether to send CodeQL usage telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent to GitHub. For more information, see the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code)"
         },
         "codeQL.telemetry.logTelemetry": {
           "type": "boolean",
           "default": false,
           "scope": "application",
           "description": "Specifies whether or not to write telemetry events to the extension log."
+        },
+        "codeQL.remoteQueries.repositoryLists": {
+          "type": [
+            "object",
+            null
+          ],
+          "patternProperties": {
+            ".*": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "default": null,
+          "markdownDescription": "[For internal use only] Lists of GitHub repositories that you want to query remotely. This should be a JSON object where each key is a user-specified name for this repository list, and the value is an array of GitHub repositories (of the form `<owner>/<repo>`)."
+        },
+        "codeQL.remoteQueries.controllerRepo": {
+          "type": "string",
+          "default": "",
+          "pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
+          "patternErrorMessage": "Please enter a valid GitHub repository",
+          "markdownDescription": "[For internal use only] The name of the GitHub repository where you can view the progress and results of the \"Run Remote query\" command. The repository should be of the form `<owner>/<repo>`)."
         }
       }
     },
     "commands": [
+      {
+        "command": "codeQL.authenticateToGitHub",
+        "title": "CodeQL: Authenticate to GitHub"
+      },
       {
         "command": "codeQL.runQuery",
         "title": "CodeQL: Run Query"
       },
+      {
+        "command": "codeQL.runQueryOnMultipleDatabases",
+        "title": "CodeQL: Run Query on Multiple Databases"
+      },
+      {
+        "command": "codeQL.runRemoteQuery",
+        "title": "CodeQL: Run Remote Query"
+      },
       {
         "command": "codeQL.runQueries",
         "title": "CodeQL: Run Queries in Selected Files"
@@ -203,10 +292,26 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.openReferencedFile",
+        "title": "CodeQL: Open Referenced File"
+      },
+      {
+        "command": "codeQL.previewQueryHelp",
+        "title": "CodeQL: Preview Query Help"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
       },
+      {
+        "command": "codeQL.openDocumentation",
+        "title": "CodeQL: Open Documentation"
+      },
+      {
+        "command": "codeQL.copyVersion",
+        "title": "CodeQL: Copy Version Information"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
@@ -279,6 +384,10 @@
         "command": "codeQLDatabases.openDatabaseFolder",
         "title": "Show Database Directory"
       },
+      {
+        "command": "codeQLDatabases.addDatabaseSource",
+        "title": "Add Database Source to Workspace"
+      },
       {
         "command": "codeQL.chooseDatabaseFolder",
         "title": "CodeQL: Choose Database from Folder"
@@ -339,6 +448,30 @@
           "dark": "media/dark/trash.svg"
         }
       },
+      {
+        "command": "codeQLQueryHistory.sortByName",
+        "title": "Sort by Name",
+        "icon": {
+          "light": "media/light/sort-alpha.svg",
+          "dark": "media/dark/sort-alpha.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByDate",
+        "title": "Sort by Query Date",
+        "icon": {
+          "light": "media/light/sort-date.svg",
+          "dark": "media/dark/sort-date.svg"
+        }
+      },
+      {
+        "command": "codeQLQueryHistory.sortByCount",
+        "title": "Sort by Results Count",
+        "icon": {
+          "light": "media/light/sort-num.svg",
+          "dark": "media/dark/sort-num.svg"
+        }
+      },
       {
         "command": "codeQLQueryHistory.showQueryLog",
         "title": "Show Query Log"
@@ -348,8 +481,16 @@
         "title": "Show Query Text"
       },
       {
-        "command": "codeQLQueryHistory.viewSarif",
-        "title": "View SARIF"
+        "command": "codeQLQueryHistory.viewCsvResults",
+        "title": "View Results (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewCsvAlerts",
+        "title": "View Alerts (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarifAlerts",
+        "title": "View Alerts (SARIF)"
       },
       {
         "command": "codeQLQueryHistory.viewDil",
@@ -443,6 +584,21 @@
           "when": "view == codeQLQueryHistory",
           "group": "navigation"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "view == codeQLQueryHistory",
+          "group": "navigation"
+        },
         {
           "command": "codeQLAstViewer.clear",
           "when": "view == codeQLAstViewer",
@@ -453,7 +609,7 @@
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
-          "when": "view == codeQLDatabases"
+          "when": "view == codeQLDatabases && viewItem != currentDatabase"
         },
         {
           "command": "codeQLDatabases.removeDatabase",
@@ -475,6 +631,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLDatabases"
         },
+        {
+          "command": "codeQLDatabases.addDatabaseSource",
+          "group": "9_qlCommands",
+          "when": "view == codeQLDatabases"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "group": "9_qlCommands",
@@ -506,7 +667,17 @@
           "when": "view == codeQLQueryHistory"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem != interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewCsvAlerts",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifAlerts",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
         },
@@ -515,6 +686,11 @@
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
         },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && resourceScheme == .qhelp && isWorkspaceTrusted"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -541,13 +717,35 @@
           "command": "codeQL.runQueries",
           "group": "9_qlCommands",
           "when": "resourceScheme != codeql-zip-archive"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .qlref"
+        },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
       ],
       "commandPalette": [
+        {
+          "command": "codeQL.authenticateToGitHub",
+          "when": "config.codeQL.canary"
+        },
         {
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueryOnMultipleDatabases",
+          "when": "resourceLangId == ql && resourceExtname == .ql"
+        },
+        {
+          "command": "codeQL.runRemoteQuery",
+          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
+        },
         {
           "command": "codeQL.runQueries",
           "when": "false"
@@ -556,6 +754,14 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
+        },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -576,6 +782,10 @@
           "command": "codeQLDatabases.openDatabaseFolder",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.addDatabaseSource",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.sortByName",
           "when": "false"
@@ -633,7 +843,15 @@
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewCsvAlerts",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifAlerts",
           "when": "false"
         },
         {
@@ -648,6 +866,18 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.sortByName",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByDate",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.sortByCount",
+          "when": "false"
+        },
         {
           "command": "codeQLAstViewer.gotoCode",
           "when": "false"
@@ -670,6 +900,14 @@
           "command": "codeQL.runQuery",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueryOnMultipleDatabases",
+          "when": "editorLangId == ql && resourceExtname == .ql"
+        },
+        {
+          "command": "codeQL.runRemoteQuery",
+          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
+        },
         {
           "command": "codeQL.viewAst",
           "when": "resourceScheme == codeql-zip-archive"
@@ -677,6 +915,14 @@
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
+        },
+        {
+          "command": "codeQL.openReferencedFile",
+          "when": "resourceExtname == .qlref"
+        },
+        {
+          "command": "codeQL.previewQueryHelp",
+          "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
       ]
     },
@@ -734,6 +980,7 @@
     "format-staged": "lint-staged"
   },
   "dependencies": {
+    "@octokit/rest": "^18.5.6",
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
     "fs-extra": "^9.0.1",
@@ -741,9 +988,13 @@
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.5",
     "node-fetch": "~2.6.0",
+    "path-browserify": "^1.0.1",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "semver": "~7.3.2",
+    "stream": "^0.0.2",
+    "stream-chain": "~2.2.4",
+    "stream-json": "~1.7.3",
     "tmp": "^0.1.0",
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
@@ -763,12 +1014,12 @@
     "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
-    "@types/gulp": "^4.0.6",
+    "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "0.0.31",
     "@types/gulp-sourcemaps": "0.0.32",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
-    "@types/mocha": "^8.0.4",
+    "@types/mocha": "^9.0.0",
     "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
@@ -778,14 +1029,16 @@
     "@types/semver": "~7.2.0",
     "@types/sinon": "~7.5.2",
     "@types/sinon-chai": "~3.2.3",
+    "@types/stream-chain": "~2.0.1",
+    "@types/stream-json": "~1.7.1",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.43.0",
+    "@types/vscode": "^1.57.0",
     "@types/webpack": "^4.32.1",
     "@types/xml2js": "~0.4.4",
-    "@typescript-eslint/eslint-plugin": "~2.23.0",
-    "@typescript-eslint/parser": "~2.23.0",
+    "@typescript-eslint/eslint-plugin": "^4.26.0",
+    "@typescript-eslint/parser": "^4.26.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^1.8.7",
     "chai": "^4.2.0",
@@ -801,8 +1054,8 @@
     "husky": "~4.2.5",
     "jsonc-parser": "^2.3.0",
     "lint-staged": "~10.2.2",
-    "mocha": "^8.2.1",
-    "mocha-sinon": "~2.1.0",
+    "mocha": "^9.1.3",
+    "mocha-sinon": "~2.1.2",
     "npm-run-all": "^4.1.5",
     "prettier": "~2.0.5",
     "proxyquire": "~2.1.3",
@@ -810,15 +1063,15 @@
     "sinon-chai": "~3.5.0",
     "style-loader": "~0.23.1",
     "through2": "^3.0.1",
-    "ts-loader": "^5.4.5",
+    "ts-loader": "^8.1.0",
     "ts-node": "^8.3.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "~3.8.3",
+    "typescript": "^4.3.2",
     "typescript-formatter": "^7.2.2",
     "vsce": "^1.65.0",
     "vscode-test": "^1.4.0",
-    "webpack": "^4.38.0",
-    "webpack-cli": "^3.3.2"
+    "webpack": "^5.28.0",
+    "webpack-cli": "^4.6.0"
   },
   "husky": {
     "hooks": {
@@ -834,5 +1087,8 @@
       "tsfmt -r",
       "eslint --fix"
     ]
+  },
+  "resolutions": {
+    "glob-parent": "~6.0.0"
   }
 }

extensions/ql-vscode/snippets.json

@@ -0,0 +1,134 @@
+{
+  "Query Metadata": {
+    "prefix": "querymetadata",
+    "body": [
+      "/**",
+      " * @name $1",
+      " * @description $2",
+      " * @kind $3",
+      " * @id $4",
+      " * @tags $5",
+      " */"
+    ],
+    "description": "Metadata for a query"
+  },
+  "Class": {
+    "prefix": "class",
+    "body": ["class $1 extends $2 {", "\t$0", "}"],
+    "description": "A class"
+  },
+  "From/Where/Select": {
+    "prefix": "from",
+    "body": ["from $1", "where $2", "select $3"],
+    "description": "A from/where/select statement"
+  },
+  "Predicate": {
+    "prefix": "predicate",
+    "body": ["predicate $1($2) {", "\t$0", "}"],
+    "description": "A predicate"
+  },
+  "Dataflow Tracking Class": {
+    "prefix": "dataflowtracking",
+    "body": [
+      "class $1 extends DataFlow::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a dataflow tracking class"
+  },
+  "Taint Tracking Class": {
+    "prefix": "tainttracking",
+    "body": [
+      "class $1 extends TaintTracking::Configuration {",
+      "\t$1() { this = \"$1\" }",
+      "\t",
+      "\toverride predicate isSource(DataFlow::Node node) {",
+      "\t\t${2:none()}",
+      "\t}",
+      "\t",
+      "\toverride predicate isSink(DataFlow::Node node) {",
+      "\t\t${3:none()}",
+      "\t}",
+      "}"
+    ],
+    "description": "Boilerplate for a taint tracking class"
+  },
+  "Count": {
+    "prefix": "count",
+    "body": ["count($1 | $2 | $3)"],
+    "description": "A count aggregate"
+  },
+  "Max": {
+    "prefix": "max",
+    "body": ["max($1 | $2 | $3)"],
+    "description": "A max aggregate"
+  },
+  "Min": {
+    "prefix": "min",
+    "body": ["min($1 | $2 | $3)"],
+    "description": "A min aggregate"
+  },
+  "Average": {
+    "prefix": "avg",
+    "body": ["avg($1 | $2 | $3)"],
+    "description": "An average aggregate"
+  },
+  "Sum": {
+    "prefix": "sum",
+    "body": ["sum($1 | $2 | $3)"],
+    "description": "A sum aggregate"
+  },
+  "Concatenation": {
+    "prefix": "concat",
+    "body": ["concat($1 | $2 | $3)"],
+    "description": "A concatenation aggregate"
+  },
+  "Rank": {
+    "prefix": "rank",
+    "body": ["rank[$1]($2 | $3 | $4)"],
+    "description": "A rank aggregate"
+  },
+  "Strict Sum": {
+    "prefix": "strictsum",
+    "body": ["strictsum($1 | $2 | $3)"],
+    "description": "A strict sum aggregate"
+  },
+  "Strict Concatenation": {
+    "prefix": "strictconcat",
+    "body": ["strictconcat($1 | $2 | $3)"],
+    "description": "A strict concatenation aggregate"
+  },
+  "Strict Count": {
+    "prefix": "strictcount",
+    "body": ["strictcount($1 | $2 | $3)"],
+    "description": "A strict count aggregate"
+  },
+  "Unique": {
+    "prefix": "unique",
+    "body": ["unique($1 | $2 | $3)"],
+    "description": "A unique aggregate"
+  },
+  "Exists": {
+    "prefix": "exists",
+    "body": ["exists($1 | $2 | $3)"],
+    "description": "An exists quantifier"
+  },
+  "For All": {
+    "prefix": "forall",
+    "body": ["forall($1 | $2 | $3)"],
+    "description": "A for all quantifier"
+  },
+  "For All and Exists": {
+    "prefix": "forex",
+    "body": ["forex($1 | $2 | $3)"],
+    "description": "A for all and exists quantifier"
+  }
+}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -115,7 +115,7 @@ class InvalidSourceArchiveUriError extends Error {
 export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
   if (!uri.authority) {
     // Uri is malformed, but this is recoverable
-    logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
+    void logger.log(`Warning: ${new InvalidSourceArchiveUriError(uri).message}`);
     return {
       pathWithinSourceArchive: '/',
       sourceArchiveZipPath: uri.path
@@ -141,7 +141,7 @@ function ensureFile(map: DirectoryHierarchyMap, file: string) {
   const dirname = path.dirname(file);
   if (dirname === '.') {
     const error = `Ill-formed path ${file} in zip archive (expected absolute path)`;
-    logger.log(error);
+    void logger.log(error);
     throw new Error(error);
   }
   ensureDir(map, dirname);

extensions/ql-vscode/src/astViewer.ts

@@ -19,7 +19,8 @@ import { UrlValue, BqrsId } from './pure/bqrs-cli-types';
 import { showLocation } from './interface-utils';
 import { isStringLoc, isWholeFileLoc, isLineColumnLoc } from './pure/bqrs-utils';
 import { commandRunner } from './commandRunner';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
 
 export interface AstItem {
   id: BqrsId;
@@ -55,7 +56,7 @@ class AstViewerDataProvider extends DisposableObject implements TreeDataProvider
   }
 
   refresh(): void {
-    this._onDidChangeTreeData.fire();
+    this._onDidChangeTreeData.fire(undefined);
   }
   getChildren(item?: AstItem): ProviderResult<AstItem[]> {
     const children = item ? item.children : this.roots;
@@ -129,8 +130,13 @@ export class AstViewer extends DisposableObject {
     this.treeDataProvider.db = db;
     this.treeDataProvider.refresh();
     this.treeView.message = `AST for ${path.basename(fileName)}`;
-    this.treeView.reveal(roots[0], { focus: false });
     this.currentFile = fileName;
+    // Handle error on reveal. This could happen if
+    // the tree view is disposed during the reveal.
+    this.treeView.reveal(roots[0], { focus: false })?.then(
+      () => { /**/ },
+      err => showAndLogErrorMessage(err)
+    );
   }
 
   private updateTreeSelection(e: TextEditorSelectionChangeEvent) {
@@ -178,7 +184,12 @@ export class AstViewer extends DisposableObject {
 
       const targetItem = findBest(range, this.treeDataProvider.roots);
       if (targetItem) {
-        this.treeView.reveal(targetItem);
+        // Handle error on reveal. This could happen if
+        // the tree view is disposed during the reveal.
+        this.treeView.reveal(targetItem)?.then(
+          () => { /**/ },
+          err => showAndLogErrorMessage(err)
+        );
       }
     }
   }

extensions/ql-vscode/src/authentication.ts

@@ -0,0 +1,62 @@
+import * as vscode from 'vscode';
+import * as Octokit from '@octokit/rest';
+
+const GITHUB_AUTH_PROVIDER_ID = 'github';
+
+// 'repo' scope should be enough for triggering workflows. For a comprehensive list, see:
+// https://docs.github.com/apps/building-oauth-apps/understanding-scopes-for-oauth-apps
+const SCOPES = ['repo'];
+
+/** 
+ * Handles authentication to GitHub, using the VS Code [authentication API](https://code.visualstudio.com/api/references/vscode-api#authentication).
+ */
+export class Credentials {
+  private octokit: Octokit.Octokit | undefined;
+
+  // Explicitly make the constructor private, so that we can't accidentally call the constructor from outside the class
+  // without also initializing the class.
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() { }
+
+  static async initialize(context: vscode.ExtensionContext): Promise<Credentials> {
+    const c = new Credentials();
+    c.registerListeners(context);
+    c.octokit = await c.createOctokit(false);
+    return c;
+  }
+
+  private async createOctokit(createIfNone: boolean): Promise<Octokit.Octokit | undefined> {
+    const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
+
+    if (session) {
+      return new Octokit.Octokit({
+        auth: session.accessToken
+      });
+    } else {
+      return undefined;
+    }
+  }
+
+  registerListeners(context: vscode.ExtensionContext): void {
+    // Sessions are changed when a user logs in or logs out.
+    context.subscriptions.push(vscode.authentication.onDidChangeSessions(async e => {
+      if (e.provider.id === GITHUB_AUTH_PROVIDER_ID) {
+        this.octokit = await this.createOctokit(false);
+      }
+    }));
+  }
+
+  async getOctokit(): Promise<Octokit.Octokit> {
+    if (this.octokit) {
+      return this.octokit;
+    }
+
+    this.octokit = await this.createOctokit(true);
+    // octokit shouldn't be undefined, since we've set "createIfNone: true".
+    // The following block is mainly here to prevent a compiler error.
+    if (!this.octokit) {
+      throw new Error('Did not initialize Octokit.');
+    }
+    return this.octokit;
+  }
+}

extensions/ql-vscode/src/cli-version.ts

@@ -18,7 +18,7 @@ export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): P
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.
-    logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
+    void logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
     return undefined;
   }
 }

extensions/ql-vscode/src/cli.ts

@@ -1,7 +1,5 @@
-/* eslint-disable @typescript-eslint/camelcase */
 import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
-import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
 import { SemVer } from 'semver';
@@ -9,22 +7,28 @@ import { Readable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as tk from 'tree-kill';
 import { promisify } from 'util';
-import { CancellationToken, Disposable } from 'vscode';
+import { CancellationToken, Disposable, Uri } from 'vscode';
 
 import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
-import * as config from './config';
 import { CliConfig } from './config';
 import { DistributionProvider, FindDistributionResultKind } from './distribution';
 import { assertNever } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
 import { CompilationMessage } from './pure/messages';
+import { sarifParser } from './sarif-parser';
+import { dbSchemeToLanguage } from './helpers';
 
 /**
  * The version of the SARIF format that we are using.
  */
 const SARIF_FORMAT = 'sarifv2.1.0';
 
+/**
+ * The string used to specify CSV format.
+ */
+const CSV_FORMAT = 'csv';
+
 /**
  * Flags to pass to all cli commands.
  */
@@ -40,6 +44,16 @@ export interface QuerySetup {
   compilationCache?: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries --format bylanguage`.
+ */
+export interface QueryInfoByLanguage {
+  // Using `unknown` as a placeholder. For now, the value is only ever an empty object.
+  byLanguage: Record<string, Record<string, unknown>>;
+  noDeclaredLanguage: Record<string, unknown>;
+  multipleDeclaredLanguages: Record<string, unknown>;
+}
+
 /**
  * The expected output of `codeql resolve database`.
  */
@@ -68,6 +82,16 @@ export interface UpgradesInfo {
  */
 export type QlpacksInfo = { [name: string]: string[] };
 
+/**
+ * The expected output of `codeql resolve languages`.
+ */
+export type LanguagesInfo = { [name: string]: string[] };
+
+/**
+ * The expected output of `codeql resolve qlref`.
+ */
+export type QlrefInfo = { resolvedPath: string };
+
 // `codeql bqrs interpret` requires both of these to be present or
 // both absent.
 export interface SourceInfo {
@@ -99,6 +123,8 @@ export interface TestCompleted {
   evaluationMs: number;
   expected: string;
   diff: string[] | undefined;
+  failureDescription?: string;
+  failureStage?: string;
 }
 
 /**
@@ -121,15 +147,6 @@ interface BqrsDecodeOptions {
  */
 export class CodeQLCliServer implements Disposable {
 
-  /**
-   * CLI version where --kind=DIL was introduced
-   */
-  private static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
-
-  /**
-   * CLI version where languages are exposed during a `codeql resolve database` command.
-   */
-  private static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
 
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
@@ -143,9 +160,16 @@ export class CodeQLCliServer implements Disposable {
   /** Version of current cli, lazily computed by the `getVersion()` method */
   private _version: SemVer | undefined;
 
+  /** 
+   * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
+   */
+  private _supportedLanguages: string[] | undefined;
+
   /** Path to current codeQL executable, or undefined if not running yet. */
   codeQlPath: string | undefined;
 
+  cliConstraints = new CliVersionConstraint(this);
+
   /**
    * When set to true, ignore some modal popups and assume user has clicked "yes".
    */
@@ -162,12 +186,15 @@ export class CodeQLCliServer implements Disposable {
     if (this.distributionProvider.onDidChangeDistribution) {
       this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
+        this._version = undefined;
+        this._supportedLanguages = undefined;
       });
     }
     if (this.cliConfig.onDidChangeConfiguration) {
       this.cliConfig.onDidChangeConfiguration(() => {
         this.restartCliServer();
         this._version = undefined;
+        this._supportedLanguages = undefined;
       });
     }
   }
@@ -179,15 +206,15 @@ export class CodeQLCliServer implements Disposable {
   killProcessIfRunning(): void {
     if (this.process) {
       // Tell the Java CLI server process to shut down.
-      this.logger.log('Sending shutdown request');
+      void this.logger.log('Sending shutdown request');
       try {
         this.process.stdin.write(JSON.stringify(['shutdown']), 'utf8');
         this.process.stdin.write(this.nullBuffer);
-        this.logger.log('Sent shutdown request');
+        void this.logger.log('Sent shutdown request');
       } catch (e) {
         // We are probably fine here, the process has already closed stdin.
-        this.logger.log(`Shutdown request failed: process stdin may have already closed. The error was ${e}`);
-        this.logger.log('Stopping the process anyway.');
+        void this.logger.log(`Shutdown request failed: process stdin may have already closed. The error was ${e}`);
+        void this.logger.log('Stopping the process anyway.');
       }
       // Close the stdin and stdout streams.
       // This is important on Windows where the child process may not die cleanly.
@@ -238,11 +265,16 @@ export class CodeQLCliServer implements Disposable {
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
     const codeQlPath = await this.getCodeQlPath();
+    const args = [];
+    if (shouldDebugCliServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9012,server=n,suspend=y,quiet=y');
+    }
+
     return await spawnServer(
       codeQlPath,
       'CodeQL CLI Server',
       ['execute', 'cli-server'],
-      [],
+      args,
       this.logger,
       _data => { /**/ }
     );
@@ -267,7 +299,7 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(' ');
-      this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -294,7 +326,7 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString('utf8', 0, fullBuffer.length - 1);
-        this.logger.log('CLI command succeeded.');
+        void this.logger.log('CLI command succeeded.');
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
@@ -307,7 +339,7 @@ export class CodeQLCliServer implements Disposable {
         newError.stack += (err.stack || '');
         throw newError;
       } finally {
-        this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
+        void this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
         // Remove the listeners we set up.
         process.stdout.removeAllListeners('data');
         process.stderr.removeAllListeners('data');
@@ -367,7 +399,7 @@ export class CodeQLCliServer implements Disposable {
       }
       if (logger !== undefined) {
         // The human-readable output goes to stderr.
-        logStream(child.stderr!, logger);
+        void logStream(child.stderr!, logger);
       }
 
       for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
@@ -448,12 +480,15 @@ export class CodeQLCliServer implements Disposable {
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
    * @param commandArgs The arguments to pass to the `codeql` command.
    * @param description Description of the action being run, to be shown in log and error messages.
+   * @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
    * @param progressReporter Used to output progress messages, e.g. to the status bar.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
-  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, progressReporter?: ProgressReporter): Promise<OutputType> {
-    // Add format argument first, in case commandArgs contains positional parameters.
-    const args = ['--format', 'json'].concat(commandArgs);
+  async runJsonCodeQlCliCommand<OutputType>(command: string[], commandArgs: string[], description: string, addFormat = true, progressReporter?: ProgressReporter): Promise<OutputType> {
+    let args: string[] = [];
+    if (addFormat) // Add format argument first, in case commandArgs contains positional parameters.
+      args = args.concat(['--format', 'json']);
+    args = args.concat(commandArgs);
     const result = await this.runCodeQlCliCommand(command, args, description, progressReporter);
     try {
       return JSON.parse(result) as OutputType;
@@ -476,6 +511,20 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, 'Resolving library paths');
   }
 
+  /**
+   * Resolves the language for a query.
+   * @param queryUri The URI of the query
+   */
+  async resolveQueryByLanguage(workspaces: string[], queryUri: Uri): Promise<QueryInfoByLanguage> {
+    const subcommandArgs = [
+      '--format', 'bylanguage',
+      queryUri.fsPath,
+      '--additional-packs',
+      workspaces.join(path.delimiter)
+    ];
+    return JSON.parse(await this.runCodeQlCliCommand(['resolve', 'queries'], subcommandArgs, 'Resolving query by language'));
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.
@@ -492,6 +541,18 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  public async resolveQlref(qlref: string): Promise<QlrefInfo> {
+    const subcommandArgs = [
+      qlref
+    ];
+    return await this.runJsonCodeQlCliCommand<QlrefInfo>(
+      ['resolve', 'qlref'],
+      subcommandArgs,
+      'Resolving qlref',
+      false
+    );
+  }
+
   /**
    * Runs QL tests.
    * @param testPaths Full paths of the tests to run.
@@ -502,12 +563,12 @@ export class CodeQLCliServer implements Disposable {
     testPaths: string[], workspaces: string[], options: TestRunOptions
   ): AsyncGenerator<TestCompleted, void, unknown> {
 
-    const subcommandArgs = [
+    const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
       '--additional-packs', workspaces.join(path.delimiter),
       '--threads',
       this.cliConfig.numberTestThreads.toString(),
       ...testPaths
-    ];
+    ]);
 
     for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
       subcommandArgs, 'Run CodeQL Tests', options.cancellationToken, options.logger)) {
@@ -536,7 +597,7 @@ export class CodeQLCliServer implements Disposable {
     if (queryMemoryMb !== undefined) {
       args.push('--ram', queryMemoryMb.toString());
     }
-    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', progressReporter);
+    return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, 'Resolving RAM settings', true, progressReporter);
   }
   /**
    * Gets the headers (and optionally pagination info) of a bqrs.
@@ -552,6 +613,29 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, 'Reading bqrs header');
   }
 
+  async databaseUnbundle(archivePath: string, target: string, name?: string): Promise<string> {
+    const subcommandArgs = [];
+    if (target) subcommandArgs.push('--target', target);
+    if (name) subcommandArgs.push('--name', name);
+    subcommandArgs.push(archivePath);
+
+    return await this.runCodeQlCliCommand(['database', 'unbundle'], subcommandArgs, `Extracting ${archivePath} to directory ${target}`);
+  }
+  
+  /**
+   * Uses a .qhelp file to generate Query Help documentation in a specified format.
+   * @param pathToQhelp The path to the .qhelp file
+   * @param format The format in which the query help should be generated {@link https://codeql.github.com/docs/codeql-cli/manual/generate-query-help/#cmdoption-codeql-generate-query-help-format}
+   * @param outputDirectory The output directory for the generated file
+   */
+  async generateQueryHelp(pathToQhelp:string, outputDirectory?: string): Promise<string> {
+    const subcommandArgs = ['--format=markdown'];
+    if(outputDirectory) subcommandArgs.push('--output', outputDirectory);
+    subcommandArgs.push(pathToQhelp);
+
+    return await this.runCodeQlCliCommand(['generate', 'query-help'], subcommandArgs, `Generating qhelp in markdown format at ${outputDirectory}`);
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
@@ -575,20 +659,19 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async interpretBqrs(metadata: { kind: string; id: string; scored?: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
-      `-t=kind=${metadata.kind}`,
-      `-t=id=${metadata.id}`,
       '--output', interpretedResultsPath,
-      '--format', SARIF_FORMAT,
+      '--format', format,
+      // Forward all of the query metadata.
+      ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`)
+    ];
+    if (format == SARIF_FORMAT) {
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      '--no-group-results',
-    ];
-    if (config.isCanary() && metadata.scored !== undefined) {
-      args.push(`-t=scored=${metadata.scored}`);
+      args.push('--no-group-results');
     }
     if (sourceInfo !== undefined) {
       args.push(
@@ -596,22 +679,29 @@ export class CodeQLCliServer implements Disposable {
         '--source-location-prefix', sourceInfo.sourceLocationPrefix
       );
     }
+
+    args.push(
+      '--threads',
+      this.cliConfig.numberThreads.toString(),
+    );
+
+    args.push(
+      '--max-paths',
+      this.cliConfig.maxPaths.toString(),
+    );
+
     args.push(resultsPath);
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
-
-    let output: string;
-    try {
-      output = await fs.readFile(interpretedResultsPath, 'utf8');
-    } catch (err) {
-      throw new Error(`Reading output of interpretation failed: ${err.stderr || err}`);
-    }
-    try {
-      return JSON.parse(output) as sarif.Log;
-    } catch (err) {
-      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
-    }
   }
 
+  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
+    return await sarifParser(interpretedResultsPath);
+  }
+
+  async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
+    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+  }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
     const sortDirectionStrings = sortDirections.map(direction => {
@@ -651,15 +741,19 @@ export class CodeQLCliServer implements Disposable {
    * Gets information necessary for upgrading a database.
    * @param dbScheme the path to the dbscheme of the database to be upgraded.
    * @param searchPath A list of directories to search for upgrade scripts.
+   * @param allowDowngradesIfPossible Whether we should try and include downgrades of we can.
    * @param targetDbScheme The dbscheme to try to upgrade to.
    * @returns A list of database upgrade script directories
    */
-  resolveUpgrades(dbScheme: string, searchPath: string[], targetDbScheme?: string): Promise<UpgradesInfo> {
+  async resolveUpgrades(dbScheme: string, searchPath: string[], allowDowngradesIfPossible: boolean, targetDbScheme?: string): Promise<UpgradesInfo> {
     const args = ['--additional-packs', searchPath.join(path.delimiter), '--dbscheme', dbScheme];
     if (targetDbScheme) {
       args.push('--target-dbscheme', targetDbScheme);
+      if (allowDowngradesIfPossible && await this.cliConstraints.supportsDowngrades()) {
+        args.push('--allow-downgrades');
+      }
     }
-    return this.runJsonCodeQlCliCommand<UpgradesInfo>(
+    return await this.runJsonCodeQlCliCommand<UpgradesInfo>(
       ['resolve', 'upgrades'],
       args,
       'Resolving database upgrade scripts',
@@ -686,6 +780,31 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Gets information about the available languages.
+   * @returns A dictionary mapping language name to the directory it comes from
+   */
+  async resolveLanguages(): Promise<LanguagesInfo> {
+    return await this.runJsonCodeQlCliCommand<LanguagesInfo>(['resolve', 'languages'], [], 'Resolving languages');
+  }
+
+  /**
+   * Gets the list of available languages. Refines the result of `resolveLanguages()`, by excluding
+   * extra things like "xml" and "properties".
+   * 
+   * @returns An array of languages that are supported by the current version of the CodeQL CLI.
+   */
+  public async getSupportedLanguages(): Promise<string[]> {
+    if (!this._supportedLanguages) {
+      // Get the intersection of resolveLanguages with the list of hardcoded languages in dbSchemeToLanguage.
+      const resolvedLanguages = Object.keys(await this.resolveLanguages());
+      const hardcodedLanguages = Object.values(dbSchemeToLanguage);
+
+      this._supportedLanguages = resolvedLanguages.filter(lang => hardcodedLanguages.includes(lang));
+    }
+    return this._supportedLanguages;
+  }
+
   /**
    * Gets information about queries in a query suite.
    * @param suite The suite to resolve.
@@ -694,11 +813,15 @@ export class CodeQLCliServer implements Disposable {
    *   the default CLI search path is used.
    * @returns A list of query files found.
    */
-  resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
+  async resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
     const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
     if (searchPath !== undefined) {
       args.push('--search-path', path.join(...searchPath));
     }
+    if (await this.cliConstraints.supportsAllowLibraryPacksInResolveQueries()) {
+      // All of our usage of `codeql resolve queries` needs to handle library packs.
+      args.push('--allow-library-packs');
+    }
     args.push(suite);
     return this.runJsonCodeQlCliCommand<string[]>(
       ['resolve', 'queries'],
@@ -707,8 +830,41 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  async packInstall(dir: string) {
+    return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
+  }
+
+  async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
+    const args = [
+      '-o',
+      outputPath,
+      dir,
+      '--additional-packs',
+      workspaceFolders.join(path.delimiter)
+    ];
+    if (!precompile && await this.cliConstraints.supportsNoPrecompile()) {
+      args.push('--no-precompile');
+    }
+
+    return this.runJsonCodeQlCliCommand(['pack', 'bundle'], args, 'Bundling pack');
+  }
+
+  async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
+    const args = includeQueries ? [dir] : ['--no-include-queries', dir];
+    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
+    // previous versions return a list of packs.
+    const results: { paths: string[] } | string[] = await this.runJsonCodeQlCliCommand(['pack', 'packlist'], args, 'Generating the pack list');
+
+    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
+    if ('paths' in results) {
+      return results.paths;
+    } else {
+      return results;
+    }
+  }
+
   async generateDil(qloFile: string, outFile: string): Promise<void> {
-    const extraArgs = await this.supportsDecompileDil()
+    const extraArgs = await this.cliConstraints.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
       : ['-o', outFile, qloFile];
     await this.runCodeQlCliCommand(
@@ -725,14 +881,6 @@ export class CodeQLCliServer implements Disposable {
     return this._version;
   }
 
-  private async supportsDecompileDil() {
-    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0;
-  }
-
-  public async supportsLanguageName() {
-    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_LANGUAGE) >= 0;
-  }
-
   private async refreshVersion() {
     const distribution = await this.distributionProvider.getDistribution();
     switch (distribution.kind) {
@@ -782,7 +930,7 @@ export function spawnServer(
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Starting ${name}` });
   }
-  logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`);
+  void logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`);
   const child = child_process.spawn(base, args);
   if (!child || !child.pid) {
     throw new Error(`Failed to start ${name} using command ${base} ${argsString}.`);
@@ -798,7 +946,7 @@ export function spawnServer(
   if (progressReporter !== undefined) {
     progressReporter.report({ message: `Started ${name}` });
   }
-  logger.log(`${name} started on PID: ${child.pid}`);
+  void logger.log(`${name} started on PID: ${child.pid}`);
   return child;
 }
 
@@ -827,10 +975,10 @@ export async function runCodeQlCliCommand(
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
     }
-    logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
+    void logger.log(`${description} using CodeQL CLI: ${codeQlPath} ${argsString}...`);
     const result = await promisify(child_process.execFile)(codeQlPath, args);
-    logger.log(result.stderr);
-    logger.log('CLI command succeeded.');
+    void logger.log(result.stderr);
+    void logger.log('CLI command succeeded.');
     return result.stdout;
   } catch (err) {
     throw new Error(`${description} failed: ${err.stderr || err}`);
@@ -866,6 +1014,20 @@ class SplitBuffer {
     this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
   }
 
+  /**
+   * A version of startsWith that isn't overriden by a broken version of ms-python.
+   *
+   * The definition comes from
+   * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
+   * which is CC0/public domain
+   *
+   * See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
+   */
+  private static startsWith(s: string, searchString: string, position: number): boolean {
+    const pos = position > 0 ? position | 0 : 0;
+    return s.substring(pos, pos + searchString.length) === searchString;
+  }
+
   /**
    * Extract the next full line from the buffer, if one is available.
    * @returns The text of the next available full line (without the separator), or `undefined` if no
@@ -874,7 +1036,7 @@ class SplitBuffer {
   public getNextLine(): string | undefined {
     while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
       for (const separator of this.separators) {
-        if (this.buffer.startsWith(separator, this.searchIndex)) {
+        if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
           const line = this.buffer.substr(0, this.searchIndex);
           this.buffer = this.buffer.substr(this.searchIndex + separator.length);
           this.searchIndex = 0;
@@ -931,7 +1093,8 @@ const lineEndings = ['\r\n', '\r', '\n'];
  */
 async function logStream(stream: Readable, logger: Logger): Promise<void> {
   for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
-    logger.log(line);
+    // Await the result of log here in order to ensure the logs are written in the correct order.
+    await logger.log(line);
   }
 }
 
@@ -947,3 +1110,104 @@ export function shouldDebugQueryServer() {
     && process.env.QUERY_SERVER_JAVA_DEBUG !== '0'
     && process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
 }
+
+export function shouldDebugCliServer() {
+  return 'CLI_SERVER_JAVA_DEBUG' in process.env
+    && process.env.CLI_SERVER_JAVA_DEBUG !== '0'
+    && process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}
+
+export class CliVersionConstraint {
+
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  public static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
+
+  /**
+   * CLI version where languages are exposed during a `codeql resolve database` command.
+   */
+  public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
+
+  /**
+   * CLI version where `codeql resolve upgrades` supports
+   * the `--allow-downgrades` flag
+   */
+  public static CLI_VERSION_WITH_DOWNGRADES = new SemVer('2.4.4');
+
+  /**
+   * CLI version where the `codeql resolve qlref` command is available.
+   */
+  public static CLI_VERSION_WITH_RESOLVE_QLREF = new SemVer('2.5.1');
+
+  /**
+   * CLI version where database registration was introduced
+  */
+  public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
+
+  /**
+   * CLI version where the `--allow-library-packs` option to `codeql resolve queries` was
+   * introduced.
+   */
+  public static CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES = new SemVer('2.6.1');
+
+  /**
+   * CLI version where the `database unbundle` subcommand was introduced.
+   */
+  public static CLI_VERSION_WITH_DATABASE_UNBUNDLE = new SemVer('2.6.0');
+
+  /**
+   * CLI version where the `--no-precompile` option for pack creation was introduced.
+   */
+  public static CLI_VERSION_WITH_NO_PRECOMPILE = new SemVer('2.7.1');
+
+  /**
+   * CLI version where remote queries are supported.
+   */
+  public static CLI_VERSION_REMOTE_QUERIES = new SemVer('2.6.3');
+
+  constructor(private readonly cli: CodeQLCliServer) {
+    /**/
+  }
+
+  private async isVersionAtLeast(v: SemVer) {
+    return (await this.cli.getVersion()).compare(v) >= 0;
+  }
+
+  public async supportsDecompileDil() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DECOMPILE_KIND_DIL);
+  }
+
+  public async supportsLanguageName() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
+  }
+
+  public async supportsDowngrades() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
+  }
+
+  public async supportsResolveQlref() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF);
+  }
+
+  public async supportsAllowLibraryPacksInResolveQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES);
+  }
+
+  async supportsDatabaseRegistration() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
+  }
+
+  async supportsDatabaseUnbundle() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
+  }
+
+  async supportsNoPrecompile() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NO_PRECOMPILE);
+  }
+
+  async supportsRemoteQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES);
+  }
+
+}

extensions/ql-vscode/src/commandRunner.ts

@@ -126,16 +126,16 @@ export function commandRunner(
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          logger.log(errorMessage);
+          void logger.log(errorMessage);
         } else {
-          showAndLogWarningMessage(errorMessage);
+          void showAndLogWarningMessage(errorMessage);
         }
       } else {
         // Include the full stack in the error log only.
         const fullMessage = e.stack
           ? `${errorMessage}\n${e.stack}`
           : errorMessage;
-        showAndLogErrorMessage(errorMessage, {
+        void showAndLogErrorMessage(errorMessage, {
           fullMessage
         });
       }
@@ -177,16 +177,16 @@ export function commandRunnerWithProgress<R>(
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          logger.log(errorMessage);
+          void logger.log(errorMessage);
         } else {
-          showAndLogWarningMessage(errorMessage);
+          void showAndLogWarningMessage(errorMessage);
         }
       } else {
         // Include the full stack in the error log only.
         const fullMessage = e.stack
           ? `${errorMessage}\n${e.stack}`
           : errorMessage;
-        showAndLogErrorMessage(errorMessage, {
+        void showAndLogErrorMessage(errorMessage, {
           fullMessage
         });
       }

extensions/ql-vscode/src/compare/compare-interface.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from '../vscode-utils/disposable-object';
+import { DisposableObject } from '../pure/disposable-object';
 import {
   WebviewPanel,
   ExtensionContext,
@@ -173,7 +173,7 @@ export class CompareInterfaceManager extends DisposableObject {
         break;
 
       case 'changeCompare':
-        this.changeTable(msg.newResultSetName);
+        await this.changeTable(msg.newResultSetName);
         break;
 
       case 'viewSourceFile':
@@ -267,11 +267,11 @@ export class CompareInterfaceManager extends DisposableObject {
     return resultsDiff(fromResults, toResults);
   }
 
-  private openQuery(kind: 'from' | 'to') {
+  private async openQuery(kind: 'from' | 'to') {
     const toOpen =
       kind === 'from' ? this.comparePair?.from : this.comparePair?.to;
     if (toOpen) {
-      this.showQueryResultsCallback(toOpen);
+      await this.showQueryResultsCallback(toOpen);
     }
   }
 }

extensions/ql-vscode/src/compare/view/Compare.tsx

@@ -21,7 +21,7 @@ const emptyComparison: SetComparisonsMessage = {
   message: 'Empty comparison'
 };
 
-export function Compare(_: {}): JSX.Element {
+export function Compare(_: Record<string, never>): JSX.Element {
   const [comparison, setComparison] = useState<SetComparisonsMessage>(
     emptyComparison
   );
@@ -38,7 +38,9 @@ export function Compare(_: {}): JSX.Element {
             setComparison(msg);
         }
       } else {
-        console.error(`Invalid event origin ${evt.origin}`);
+        // sanitize origin
+        const origin = evt.origin.replace(/\n|\r/g, '');
+        console.error(`Invalid event origin ${origin}`);
       }
     });
   });
@@ -64,8 +66,8 @@ export function Compare(_: {}): JSX.Element {
         {hasRows ? (
           <CompareTable comparison={comparison}></CompareTable>
         ) : (
-            <div className="vscode-codeql__compare-message">{message}</div>
-          )}
+          <div className="vscode-codeql__compare-message">{message}</div>
+        )}
       </>
     );
   } catch (err) {

extensions/ql-vscode/src/config.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
@@ -41,6 +41,7 @@ const ROOT_SETTING = new Setting('codeQL');
 
 // Global configuration
 const TELEMETRY_SETTING = new Setting('telemetry', ROOT_SETTING);
+const AST_VIEWER_SETTING = new Setting('astViewer', ROOT_SETTING);
 const GLOBAL_TELEMETRY_SETTING = new Setting('telemetry');
 
 export const LOG_TELEMETRY = new Setting('logTelemetry', TELEMETRY_SETTING);
@@ -50,7 +51,7 @@ export const GLOBAL_ENABLE_TELEMETRY = new Setting('enableTelemetry', GLOBAL_TEL
 
 // Distribution configuration
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
-const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
+export const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
 const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
@@ -73,26 +74,34 @@ export interface DistributionConfig {
 
 const RUNNING_QUERIES_SETTING = new Setting('runningQueries', ROOT_SETTING);
 const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES_SETTING);
+const SAVE_CACHE_SETTING = new Setting('saveCache', RUNNING_QUERIES_SETTING);
+const CACHE_SIZE_SETTING = new Setting('cacheSize', RUNNING_QUERIES_SETTING);
 const TIMEOUT_SETTING = new Setting('timeout', RUNNING_QUERIES_SETTING);
 const MEMORY_SETTING = new Setting('memory', RUNNING_QUERIES_SETTING);
 const DEBUG_SETTING = new Setting('debug', RUNNING_QUERIES_SETTING);
+const MAX_PATHS = new Setting('maxPaths', RUNNING_QUERIES_SETTING);
 const RUNNING_TESTS_SETTING = new Setting('runningTests', ROOT_SETTING);
 const RESULTS_DISPLAY_SETTING = new Setting('resultsDisplay', ROOT_SETTING);
 
+export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting('additionalTestArguments', RUNNING_TESTS_SETTING);
 export const NUMBER_OF_TEST_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_TESTS_SETTING);
 export const MAX_QUERIES = new Setting('maxQueries', RUNNING_QUERIES_SETTING);
 export const AUTOSAVE_SETTING = new Setting('autoSave', RUNNING_QUERIES_SETTING);
 export const PAGE_SIZE = new Setting('pageSize', RESULTS_DISPLAY_SETTING);
+const CUSTOM_LOG_DIRECTORY_SETTING = new Setting('customLogDirectory', RUNNING_QUERIES_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
-const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, MEMORY_SETTING, DEBUG_SETTING];
+const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING, DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING];
 
 export interface QueryServerConfig {
   codeQlPath: string;
   debug: boolean;
   numThreads: number;
+  saveCache: boolean;
+  cacheSize: number;
   queryMemoryMb?: number;
   timeoutSecs: number;
+  customLogDirectory?: string;
   onDidChangeConfiguration?: Event<void>;
 }
 
@@ -104,10 +113,13 @@ export interface QueryHistoryConfig {
   onDidChangeConfiguration: Event<void>;
 }
 
-const CLI_SETTINGS = [NUMBER_OF_TEST_THREADS_SETTING];
+const CLI_SETTINGS = [ADDITIONAL_TEST_ARGUMENTS_SETTING, NUMBER_OF_TEST_THREADS_SETTING, NUMBER_OF_THREADS_SETTING, MAX_PATHS];
 
 export interface CliConfig {
+  additionalTestArguments: string[];
   numberTestThreads: number;
+  numberThreads: number;
+  maxPaths: number;
   onDidChangeConfiguration?: Event<void>;
 }
 
@@ -137,7 +149,7 @@ export abstract class ConfigListener extends DisposableObject {
 
   protected abstract handleDidChangeConfiguration(e: ConfigurationChangeEvent): void;
   private updateConfiguration(): void {
-    this._onDidChangeConfiguration.fire();
+    this._onDidChangeConfiguration.fire(undefined);
   }
 
   public get onDidChangeConfiguration(): Event<void> {
@@ -179,7 +191,7 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
       config.push(distributionManager.onDidChangeDistribution(async () => {
         const codeQlPath = await distributionManager.getCodeQlPathWithoutVersionCheck();
         config._codeQlPath = codeQlPath!;
-        config._onDidChangeConfiguration.fire();
+        config._onDidChangeConfiguration.fire(undefined);
       }));
     }
     return config;
@@ -189,10 +201,22 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
     return this._codeQlPath;
   }
 
+  public get customLogDirectory(): string | undefined {
+    return CUSTOM_LOG_DIRECTORY_SETTING.getValue<string>() || undefined;
+  }
+
   public get numThreads(): number {
     return NUMBER_OF_THREADS_SETTING.getValue<number>();
   }
 
+  public get saveCache(): boolean {
+    return SAVE_CACHE_SETTING.getValue<boolean>();
+  }
+
+  public get cacheSize(): number {
+    return CACHE_SIZE_SETTING.getValue<number | null>() || 0;
+  }
+
   /** Gets the configured query timeout, in seconds. This looks up the setting at the time of access. */
   public get timeoutSecs(): number {
     return TIMEOUT_SETTING.getValue<number | null>() || 0;
@@ -204,7 +228,7 @@ export class QueryServerConfigListener extends ConfigListener implements QuerySe
       return undefined;
     }
     if (memory == 0 || typeof (memory) !== 'number') {
-      logger.log(`Ignoring value '${memory}' for setting ${MEMORY_SETTING.qualifiedName}`);
+      void logger.log(`Ignoring value '${memory}' for setting ${MEMORY_SETTING.qualifiedName}`);
       return undefined;
     }
     return memory;
@@ -230,11 +254,22 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
 }
 
 export class CliConfigListener extends ConfigListener implements CliConfig {
+  public get additionalTestArguments(): string[] {
+    return ADDITIONAL_TEST_ARGUMENTS_SETTING.getValue();
+  }
 
   public get numberTestThreads(): number {
     return NUMBER_OF_TEST_THREADS_SETTING.getValue();
   }
 
+  public get numberThreads(): number {
+    return NUMBER_OF_THREADS_SETTING.getValue<number>();
+  }
+
+  public get maxPaths(): number {
+    return MAX_PATHS.getValue<number>();
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
   }
@@ -257,3 +292,44 @@ export const CANARY_FEATURES = new Setting('canary', ROOT_SETTING);
 export function isCanary() {
   return !!CANARY_FEATURES.getValue<boolean>();
 }
+
+/**
+ * Avoids caching in the AST viewer if the user is also a canary user.
+ */
+export const NO_CACHE_AST_VIEWER = new Setting('disableCache', AST_VIEWER_SETTING);
+
+// Settings for remote queries
+const REMOTE_QUERIES_SETTING = new Setting('remoteQueries', ROOT_SETTING);
+
+/**
+ * Lists of GitHub repositories that you want to query remotely via the "Run Remote query" command.
+ * Note: This command is only available for internal users.
+ *
+ * This setting should be a JSON object where each key is a user-specified name (string),
+ * and the value is an array of GitHub repositories (of the form `<owner>/<repo>`).
+ */
+const REMOTE_REPO_LISTS = new Setting('repositoryLists', REMOTE_QUERIES_SETTING);
+
+export function getRemoteRepositoryLists(): Record<string, string[]> | undefined {
+  return REMOTE_REPO_LISTS.getValue<Record<string, string[]>>() || undefined;
+}
+
+export async function setRemoteRepositoryLists(lists: Record<string, string[]> | undefined) {
+  await REMOTE_REPO_LISTS.updateValue(lists, ConfigurationTarget.Global);
+}
+
+/**
+ * The name of the "controller" repository that you want to use with the "Run Remote query" command.
+ * Note: This command is only available for internal users.
+ *
+ * This setting should be a GitHub repository of the form `<owner>/<repo>`.
+ */
+const REMOTE_CONTROLLER_REPO = new Setting('controllerRepo', REMOTE_QUERIES_SETTING);
+
+export function getRemoteControllerRepo(): string | undefined {
+  return REMOTE_CONTROLLER_REPO.getValue<string>() || undefined;
+}
+
+export async function setRemoteControllerRepo(repo: string | undefined) {
+  await REMOTE_CONTROLLER_REPO.updateValue(repo, ConfigurationTarget.Global);
+}

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -12,7 +12,7 @@ import { ProgressCallback } from '../commandRunner';
 import { KeyType } from './keyType';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
 
-const SELECT_QUERY_NAME = '#select';
+export const SELECT_QUERY_NAME = '#select';
 export const TEMPLATE_NAME = 'selectedSourceFile';
 
 export interface FullLocationLink extends vscode.LocationLink {

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -11,8 +11,9 @@ import {
 } from './keyType';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseItem } from '../databases';
+import { QlPacksForLanguage } from '../helpers';
 
-export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string> {
+export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<QlPacksForLanguage> {
   if (db.contents === undefined) {
     throw new Error('Database is invalid and cannot infer QLPack.');
   }
@@ -21,28 +22,85 @@ export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem):
   return await helpers.getQlPackForDbscheme(cli, dbscheme);
 }
 
-
-export async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
+/**
+ * Finds the contextual queries with the specified key in a list of CodeQL packs.
+ *
+ * @param cli The CLI instance to use.
+ * @param qlpacks The list of packs to search.
+ * @param keyType The contextual query key of the query to search for.
+ * @returns The found queries from the first pack in which any matching queries were found.
+ */
+async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[], keyType: KeyType): Promise<string[]> {
   const suiteFile = (await tmp.file({
     postfix: '.qls'
   })).path;
-  const suiteYaml = {
-    qlpack,
-    include: {
-      kind: kindOfKeyType(keyType),
-      'tags contain': tagOfKeyType(keyType)
-    }
-  };
+  const suiteYaml = [];
+  for (const qlpack of qlpacks) {
+    suiteYaml.push({
+      from: qlpack,
+      queries: '.',
+      include: {
+        kind: kindOfKeyType(keyType),
+        'tags contain': tagOfKeyType(keyType)
+      }
+    });
+  }
   await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
 
   const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
-  if (queries.length === 0) {
-    helpers.showAndLogErrorMessage(
-      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
-      Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
-      for this language.`
-    );
-    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
-  }
   return queries;
 }
+
+export async function resolveQueries(cli: CodeQLCliServer, qlpacks: QlPacksForLanguage, keyType: KeyType): Promise<string[]> {
+  const cliCanHandleLibraryPack = await cli.cliConstraints.supportsAllowLibraryPacksInResolveQueries();
+  const packsToSearch: string[] = [];
+  let blameCli: boolean;
+
+  if (cliCanHandleLibraryPack) {
+    // The CLI can handle both library packs and query packs, so search both packs in order.
+    packsToSearch.push(qlpacks.dbschemePack);
+    if (qlpacks.queryPack !== undefined) {
+      packsToSearch.push(qlpacks.queryPack);
+    }
+    // If we don't find the query, it's because it's not there, not because the CLI was unable to
+    // search the pack.
+    blameCli = false;
+  } else {
+    // Older CLIs can't handle `codeql resolve queries` with a suite that references a library pack.
+    if (qlpacks.dbschemePackIsLibraryPack) {
+      if (qlpacks.queryPack !== undefined) {
+        // Just search the query pack, because some older library/query releases still had the
+        // contextual queries in the query pack.
+        packsToSearch.push(qlpacks.queryPack);
+      }
+      // If we don't find it, it's because the CLI was unable to search the library pack that
+      // actually contains the query. Blame any failure on the CLI, not the packs.
+      blameCli = true;
+    } else {
+      // We have an old CLI, but the dbscheme pack is old enough that it's still a unified pack with
+      // both libraries and queries. Just search that pack.
+      packsToSearch.push(qlpacks.dbschemePack);
+      // Any CLI should be able to search the single query pack, so if we don't find it, it's
+      // because the language doesn't support it.
+      blameCli = false;
+    }
+  }
+
+  const queries = await resolveQueriesFromPacks(cli, packsToSearch, keyType);
+  if (queries.length > 0) {
+    return queries;
+  }
+
+  // No queries found. Determine the correct error message for the various scenarios.
+  const errorMessage = blameCli ?
+    `Your current version of the CodeQL CLI, '${(await cli.getVersion()).version}', \
+    is unable to use contextual queries from recent versions of the standard CodeQL libraries. \
+    Please upgrade to the latest version of the CodeQL CLI.`
+    :
+    `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
+    Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
+    for this language.`;
+
+  void helpers.showAndLogErrorMessage(errorMessage);
+  throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} in any of the following packs: ${packsToSearch.join(', ')}.`);
+}

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -1,4 +1,15 @@
-import * as vscode from 'vscode';
+import {
+  CancellationToken,
+  DefinitionProvider,
+  Location,
+  LocationLink,
+  Position,
+  ProgressLocation,
+  ReferenceContext,
+  ReferenceProvider,
+  TextDocument,
+  Uri
+} from 'vscode';
 
 import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
 import { CodeQLCliServer } from '../cli';
@@ -14,6 +25,7 @@ import {
 } from './keyType';
 import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
 
 /**
  * Run templated CodeQL queries to find definitions and references in
@@ -22,20 +34,20 @@ import { qlpackOfDatabase, resolveQueries } from './queryResolver';
  * or from a selected identifier.
  */
 
-export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvider {
-  private cache: CachedOperation<vscode.LocationLink[]>;
+export class TemplateQueryDefinitionProvider implements DefinitionProvider {
+  private cache: CachedOperation<LocationLink[]>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
   ) {
-    this.cache = new CachedOperation<vscode.LocationLink[]>(this.getDefinitions.bind(this));
+    this.cache = new CachedOperation<LocationLink[]>(this.getDefinitions.bind(this));
   }
 
-  async provideDefinition(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken): Promise<vscode.LocationLink[]> {
+  async provideDefinition(document: TextDocument, position: Position, _token: CancellationToken): Promise<LocationLink[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.LocationLink[] = [];
+    const locLinks: LocationLink[] = [];
     for (const link of fileLinks) {
       if (link.originSelectionRange!.contains(position)) {
         locLinks.push(link);
@@ -44,9 +56,9 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
     return locLinks;
   }
 
-  private async getDefinitions(uriString: string): Promise<vscode.LocationLink[]> {
+  private async getDefinitions(uriString: string): Promise<LocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding definitions'
     }, async (progress, token) => {
@@ -64,7 +76,7 @@ export class TemplateQueryDefinitionProvider implements vscode.DefinitionProvide
   }
 }
 
-export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider {
+export class TemplateQueryReferenceProvider implements ReferenceProvider {
   private cache: CachedOperation<FullLocationLink[]>;
 
   constructor(
@@ -76,13 +88,13 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
   }
 
   async provideReferences(
-    document: vscode.TextDocument,
-    position: vscode.Position,
-    _context: vscode.ReferenceContext,
-    _token: vscode.CancellationToken
-  ): Promise<vscode.Location[]> {
+    document: TextDocument,
+    position: Position,
+    _context: ReferenceContext,
+    _token: CancellationToken
+  ): Promise<Location[]> {
     const fileLinks = await this.cache.get(document.uri.toString());
-    const locLinks: vscode.Location[] = [];
+    const locLinks: Location[] = [];
     for (const link of fileLinks) {
       if (link.targetRange!.contains(position)) {
         locLinks.push({ range: link.originSelectionRange!, uri: link.originUri });
@@ -93,7 +105,7 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 
   private async getReferences(uriString: string): Promise<FullLocationLink[]> {
     return withProgress({
-      location: vscode.ProgressLocation.Notification,
+      location: ProgressLocation.Notification,
       cancellable: true,
       title: 'Finding references'
     }, async (progress, token) => {
@@ -112,40 +124,47 @@ export class TemplateQueryReferenceProvider implements vscode.ReferenceProvider
 }
 
 export class TemplatePrintAstProvider {
-  private cache: CachedOperation<QueryWithResults | undefined>;
+  private cache: CachedOperation<QueryWithResults>;
 
   constructor(
     private cli: CodeQLCliServer,
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
-
-    // Note: progress and token are only used if a cached value is not available
-    private progress: ProgressCallback,
-    private token: vscode.CancellationToken
   ) {
-    this.cache = new CachedOperation<QueryWithResults | undefined>(this.getAst.bind(this));
+    this.cache = new CachedOperation<QueryWithResults>(this.getAst.bind(this));
   }
 
-  async provideAst(document?: vscode.TextDocument): Promise<AstBuilder | undefined> {
+  async provideAst(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    document?: TextDocument
+  ): Promise<AstBuilder | undefined> {
     if (!document) {
-      return;
-    }
-    const queryResults = await this.cache.get(document.uri.toString());
-    if (!queryResults) {
-      return;
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
+    const queryResults = this.shouldCache()
+      ? await this.cache.get(document.uri.toString(), progress, token)
+      : await this.getAst(document.uri.toString(), progress, token);
 
     return new AstBuilder(
       queryResults, this.cli,
-      this.dbm.findDatabaseItem(vscode.Uri.parse(queryResults.database.databaseUri!, true))!,
+      this.dbm.findDatabaseItem(Uri.parse(queryResults.database.databaseUri!, true))!,
       document.fileName
     );
   }
 
-  private async getAst(uriString: string): Promise<QueryWithResults> {
-    const uri = vscode.Uri.parse(uriString, true);
+  private shouldCache() {
+    return !(isCanary() && NO_CACHE_AST_VIEWER.getValue<boolean>());
+  }
+
+  private async getAst(
+    uriString: string,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<QueryWithResults> {
+    const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
-      throw new Error('AST Viewing is only available for databases with zipped source archives.');
+      throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
 
     const zippedArchive = decodeSourceArchiveUri(uri);
@@ -156,8 +175,8 @@ export class TemplatePrintAstProvider {
       throw new Error('Can\'t infer database from the provided source.');
     }
 
-    const qlpack = await qlpackOfDatabase(this.cli, db);
-    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintAstQuery);
+    const qlpacks = await qlpackOfDatabase(this.cli, db);
+    const queries = await resolveQueries(this.cli, qlpacks, KeyType.PrintAstQuery);
     if (queries.length > 1) {
       throw new Error('Found multiple Print AST queries. Can\'t continue');
     }
@@ -181,9 +200,9 @@ export class TemplatePrintAstProvider {
       this.qs,
       db,
       false,
-      vscode.Uri.file(query),
-      this.progress,
-      this.token,
+      Uri.file(query),
+      progress,
+      token,
       templates
     );
   }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -1,12 +1,13 @@
 import fetch, { Response } from 'node-fetch';
-import * as unzipper from 'unzipper';
 import { zip } from 'zip-a-folder';
+import * as unzipper from 'unzipper';
 import {
   Uri,
   CancellationToken,
   commands,
   window,
 } from 'vscode';
+import { CodeQLCliServer } from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -32,6 +33,7 @@ export async function promptImportInternetDatabase(
   storagePath: string,
   progress: ProgressCallback,
   token: CancellationToken,
+  cli?: CodeQLCliServer
 ): Promise<DatabaseItem | undefined> {
   const databaseUrl = await window.showInputBox({
     prompt: 'Enter URL of zipfile of database to download',
@@ -47,12 +49,13 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     progress,
-    token
+    token,
+    cli
   );
 
   if (item) {
-    commands.executeCommand('codeQLDatabases.focus');
-    showAndLogInformationMessage('Database downloaded and imported successfully.');
+    await commands.executeCommand('codeQLDatabases.focus');
+    void showAndLogInformationMessage('Database downloaded and imported successfully.');
   }
   return item;
 
@@ -70,29 +73,36 @@ export async function promptImportLgtmDatabase(
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  token: CancellationToken
+  token: CancellationToken,
+  cli?: CodeQLCliServer
 ): Promise<DatabaseItem | undefined> {
+  progress({
+    message: 'Choose project',
+    step: 1,
+    maxStep: 2
+  });
   const lgtmUrl = await window.showInputBox({
     prompt:
-      'Enter the project URL on LGTM (e.g., https://lgtm.com/projects/g/github/codeql)',
+      'Enter the project slug or URL on LGTM (e.g., g/github/codeql or https://lgtm.com/projects/g/github/codeql)',
   });
   if (!lgtmUrl) {
     return;
   }
 
   if (looksLikeLgtmUrl(lgtmUrl)) {
-    const databaseUrl = await convertToDatabaseUrl(lgtmUrl);
+    const databaseUrl = await convertToDatabaseUrl(lgtmUrl, progress);
     if (databaseUrl) {
       const item = await databaseArchiveFetcher(
         databaseUrl,
         databaseManager,
         storagePath,
         progress,
-        token
+        token,
+        cli
       );
       if (item) {
-        commands.executeCommand('codeQLDatabases.focus');
-        showAndLogInformationMessage('Database downloaded and imported successfully.');
+        await commands.executeCommand('codeQLDatabases.focus');
+        void showAndLogInformationMessage('Database downloaded and imported successfully.');
       }
       return item;
     }
@@ -102,6 +112,16 @@ export async function promptImportLgtmDatabase(
   return;
 }
 
+export async function retrieveCanonicalRepoName(lgtmUrl: string) {
+  const givenRepoName = extractProjectSlug(lgtmUrl);
+  const response = await checkForFailingResponse(await fetch(`https://api.github.com/repos/${givenRepoName}`), 'Failed to locate the repository on github');
+  const repo = await response.json();
+  if (!repo || !repo.full_name) {
+    return;
+  }
+  return repo.full_name;
+}
+
 /**
  * Imports a database from a local archive.
  *
@@ -115,6 +135,7 @@ export async function importArchiveDatabase(
   storagePath: string,
   progress: ProgressCallback,
   token: CancellationToken,
+  cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
   try {
     const item = await databaseArchiveFetcher(
@@ -122,11 +143,12 @@ export async function importArchiveDatabase(
       databaseManager,
       storagePath,
       progress,
-      token
+      token,
+      cli
     );
     if (item) {
-      commands.executeCommand('codeQLDatabases.focus');
-      showAndLogInformationMessage('Database unzipped and imported successfully.');
+      await commands.executeCommand('codeQLDatabases.focus');
+      void showAndLogInformationMessage('Database unzipped and imported successfully.');
     }
     return item;
   } catch (e) {
@@ -154,7 +176,8 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  token: CancellationToken
+  token: CancellationToken,
+  cli?: CodeQLCliServer,
 ): Promise<DatabaseItem> {
   progress({
     message: 'Getting database',
@@ -168,9 +191,9 @@ async function databaseArchiveFetcher(
   const unzipPath = await getStorageFolder(storagePath, databaseUrl);
 
   if (isFile(databaseUrl)) {
-    await readAndUnzip(databaseUrl, unzipPath, progress);
+    await readAndUnzip(databaseUrl, unzipPath, cli, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, progress);
+    await fetchAndUnzip(databaseUrl, unzipPath, cli, progress);
   }
 
   progress({
@@ -244,6 +267,7 @@ function validateHttpsUrl(databaseUrl: string) {
 async function readAndUnzip(
   zipUrl: string,
   unzipPath: string,
+  cli?: CodeQLCliServer,
   progress?: ProgressCallback
 ) {
   // TODO: Providing progress as the file is unzipped is currently blocked
@@ -254,16 +278,22 @@ async function readAndUnzip(
     step: 9,
     message: `Unzipping into ${path.basename(unzipPath)}`
   });
-  // Must get the zip central directory since streaming the
-  // zip contents may not have correct local file headers.
-  // Instead, we can only rely on the central directory.
-  const directory = await unzipper.Open.file(zipFile);
-  await directory.extract({ path: unzipPath });
+  if (cli && await cli.cliConstraints.supportsDatabaseUnbundle()) {
+    // Use the `database unbundle` command if the installed cli version supports it
+    await cli.databaseUnbundle(zipFile, unzipPath);
+  } else {
+    // Must get the zip central directory since streaming the
+    // zip contents may not have correct local file headers.
+    // Instead, we can only rely on the central directory.
+    const directory = await unzipper.Open.file(zipFile);
+    await directory.extract({ path: unzipPath });
+  }
 }
 
 async function fetchAndUnzip(
   databaseUrl: string,
   unzipPath: string,
+  cli?: CodeQLCliServer,
   progress?: ProgressCallback
 ) {
   // Although it is possible to download and stream directly to an unzipped directory,
@@ -280,7 +310,7 @@ async function fetchAndUnzip(
     step: 1,
   });
 
-  const response = await checkForFailingResponse(await fetch(databaseUrl));
+  const response = await checkForFailingResponse(await fetch(databaseUrl), 'Error downloading database');
   const archiveFileStream = fs.createWriteStream(archivePath);
 
   const contentLength = response.headers.get('content-length');
@@ -293,13 +323,14 @@ async function fetchAndUnzip(
       .on('error', reject)
   );
 
-  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, progress);
+  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, cli, progress);
+
 
   // remove archivePath eagerly since these archives can be large.
   await fs.remove(archivePath);
 }
 
-async function checkForFailingResponse(response: Response): Promise<Response | never> {
+async function checkForFailingResponse(response: Response, errorMessage: string): Promise<Response | never> {
   if (response.ok) {
     return response;
   }
@@ -313,7 +344,7 @@ async function checkForFailingResponse(response: Response): Promise<Response | n
   } catch (e) {
     msg = text;
   }
-  throw new Error(`Error downloading database.\n\nReason: ${msg}`);
+  throw new Error(`${errorMessage}.\n\nReason: ${msg}`);
 }
 
 function isFile(databaseUrl: string) {
@@ -352,13 +383,14 @@ export async function findDirWithFile(
 
 /**
  * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
- * There are several possibilities for the provider: in addition to GitHub.com(g),
+ * There are several possibilities for the provider: in addition to GitHub.com (g),
  * LGTM currently hosts projects from Bitbucket (b), GitLab (gl) and plain git (git).
  *
- * After the {provider}/{org}/{name} path components, there may be the components
- * related to sub pages.
+ * This function accepts any url that matches the pattern above. It also accepts the
+ * raw project slug, e.g., `g/myorg/myproject`
  *
- * This function accepts any url that matches the patter above
+ * After the `{provider}/{org}/{name}` path components, there may be the components
+ * related to sub pages.
  *
  * @param lgtmUrl The URL to the lgtm project
  *
@@ -370,6 +402,10 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
     return false;
   }
 
+  if (convertRawLgtmSlug(lgtmUrl)) {
+    return true;
+  }
+
   try {
     const uri = Uri.parse(lgtmUrl, true);
     if (uri.scheme !== 'https') {
@@ -387,22 +423,51 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
   }
 }
 
+function convertRawLgtmSlug(maybeSlug: string): string | undefined {
+  if (!maybeSlug) {
+    return;
+  }
+  const segments = maybeSlug.split('/');
+  const providers = ['g', 'gl', 'b', 'git'];
+  if (segments.length === 3 && providers.includes(segments[0])) {
+    return `https://lgtm.com/projects/${maybeSlug}`;
+  }
+  return;
+}
+ 
+function extractProjectSlug(lgtmUrl: string): string | undefined {
+  // Only matches the '/g/' provider (github)
+  const re = new RegExp('https://lgtm.com/projects/g/(.*[^/])');
+  const match = lgtmUrl.match(re);
+  if (!match) {
+    return;
+  }
+  return match[1];
+}
+
 // exported for testing
-export async function convertToDatabaseUrl(lgtmUrl: string) {
+export async function convertToDatabaseUrl(
+  lgtmUrl: string,
+  progress: ProgressCallback) {
   try {
-    const uri = Uri.parse(lgtmUrl, true);
-    const paths = ['api', 'v1.0'].concat(
-      uri.path.split('/').filter((segment) => segment)
-    ).slice(0, 6);
-    const projectUrl = `https://lgtm.com/${paths.join('/')}`;
-    const projectResponse = await fetch(projectUrl);
-    const projectJson = await projectResponse.json();
+    lgtmUrl = convertRawLgtmSlug(lgtmUrl) || lgtmUrl;
+    let projectJson = await downloadLgtmProjectMetadata(lgtmUrl);
 
     if (projectJson.code === 404) {
-      throw new Error();
+      // fallback check for github repositories with same name but different case
+      // will fail for other providers
+      let canonicalName = await retrieveCanonicalRepoName(lgtmUrl);
+      if (!canonicalName) {
+        throw new Error(`Project was not found at ${lgtmUrl}.`);
+      }
+      canonicalName = convertRawLgtmSlug(`g/${canonicalName}`);
+      projectJson = await downloadLgtmProjectMetadata(canonicalName);
+      if (projectJson.code === 404) {
+        throw new Error('Failed to download project from LGTM.');
+      }
     }
 
-    const language = await promptForLanguage(projectJson);
+    const language = await promptForLanguage(projectJson, progress);
     if (!language) {
       return;
     }
@@ -414,14 +479,30 @@ export async function convertToDatabaseUrl(lgtmUrl: string) {
       language,
     ].join('/')}`;
   } catch (e) {
-    logger.log(`Error: ${e.message}`);
+    void logger.log(`Error: ${e.message}`);
     throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
   }
 }
 
+async function downloadLgtmProjectMetadata(lgtmUrl: string): Promise<any> {
+  const uri = Uri.parse(lgtmUrl, true);
+  const paths = ['api', 'v1.0'].concat(
+    uri.path.split('/').filter((segment) => segment)
+  ).slice(0, 6);
+  const projectUrl = `https://lgtm.com/${paths.join('/')}`;
+  const projectResponse = await fetch(projectUrl);
+  return projectResponse.json();
+}
+
 async function promptForLanguage(
-  projectJson: any
+  projectJson: any,
+  progress: ProgressCallback
 ): Promise<string | undefined> {
+  progress({
+    message: 'Choose language',
+    step: 2,
+    maxStep: 2
+  });
   if (!projectJson?.languages?.length) {
     return;
   }

extensions/ql-vscode/src/databases-ui.ts

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import {
   Event,
   EventEmitter,
@@ -108,7 +108,7 @@ class DatabaseTreeDataProvider extends DisposableObject
   }
 
   private handleDidChangeDatabaseItem = (event: DatabaseChangedEvent): void => {
-    // Note that events from the databse manager are instances of DatabaseChangedEvent
+    // Note that events from the database manager are instances of DatabaseChangedEvent
     // and events fired by the UI are instances of DatabaseItem
 
     // When event.item is undefined, then the entire tree is refreshed.
@@ -135,6 +135,7 @@ class DatabaseTreeDataProvider extends DisposableObject
         this.extensionPath,
         SELECTED_DATABASE_ICON
       );
+      item.contextValue = 'currentDatabase';
     } else if (element.error !== undefined) {
       item.iconPath = joinThemableIconPath(
         this.extensionPath,
@@ -179,7 +180,7 @@ class DatabaseTreeDataProvider extends DisposableObject
 
   public set sortOrder(newSortOrder: SortOrder) {
     this._sortOrder = newSortOrder;
-    this._onDidChangeTreeData.fire();
+    this._onDidChangeTreeData.fire(undefined);
   }
 }
 
@@ -234,7 +235,7 @@ export class DatabaseUI extends DisposableObject {
   }
 
   init() {
-    logger.log('Registering database panel commands.');
+    void logger.log('Registering database panel commands.');
     this.push(
       commandRunnerWithProgress(
         'codeQL.setCurrentDatabase',
@@ -348,6 +349,12 @@ export class DatabaseUI extends DisposableObject {
         this.handleOpenFolder
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.addDatabaseSource',
+        this.handleAddSource
+      )
+    );
     this.push(
       commandRunner(
         'codeQLDatabases.removeOrphanedDatabases',
@@ -369,20 +376,20 @@ export class DatabaseUI extends DisposableObject {
     try {
       return await this.chooseAndSetDatabase(true, progress, token);
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
       return undefined;
     }
   };
 
   handleRemoveOrphanedDatabases = async (): Promise<void> => {
-    logger.log('Removing orphaned databases from workspace storage.');
+    void logger.log('Removing orphaned databases from workspace storage.');
     let dbDirs = undefined;
 
     if (
-      !(await fs.pathExists(this.storagePath) ||
-        !(await fs.stat(this.storagePath)).isDirectory())
+      !(await fs.pathExists(this.storagePath)) ||
+      !(await fs.stat(this.storagePath)).isDirectory()
     ) {
-      logger.log('Missing or invalid storage directory. Not trying to remove orphaned databases.');
+      void logger.log('Missing or invalid storage directory. Not trying to remove orphaned databases.');
       return;
     }
 
@@ -403,7 +410,7 @@ export class DatabaseUI extends DisposableObject {
     dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
 
     if (!dbDirs.length) {
-      logger.log('No orphaned databases found.');
+      void logger.log('No orphaned databases found.');
       return;
     }
 
@@ -412,8 +419,8 @@ export class DatabaseUI extends DisposableObject {
     await Promise.all(
       dbDirs.map(async dbDir => {
         try {
-          logger.log(`Deleting orphaned database '${dbDir}'.`);
-          await fs.rmdir(dbDir, { recursive: true } as any);  // typings doesn't recognize the options argument
+          void logger.log(`Deleting orphaned database '${dbDir}'.`);
+          await fs.remove(dbDir);
         } catch (e) {
           failures.push(`${path.basename(dbDir)}`);
         }
@@ -422,10 +429,9 @@ export class DatabaseUI extends DisposableObject {
 
     if (failures.length) {
       const dirname = path.dirname(failures[0]);
-      showAndLogErrorMessage(
-        `Failed to delete unused databases:\n  ${
-        failures.join('\n  ')
-        }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.`
+      void showAndLogErrorMessage(
+        `Failed to delete unused databases (${failures.join(', ')
+        }).\nTo delete unused databases, please remove them manually from the storage folder ${dirname}.`
       );
     }
   };
@@ -438,7 +444,7 @@ export class DatabaseUI extends DisposableObject {
     try {
       return await this.chooseAndSetDatabase(false, progress, token);
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
       return undefined;
     }
   };
@@ -446,14 +452,13 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseInternet = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<
-    DatabaseItem | undefined
-  > => {
+  ): Promise<DatabaseItem | undefined> => {
     return await promptImportInternetDatabase(
       this.databaseManager,
       this.storagePath,
       progress,
-      token
+      token,
+      this.queryServer?.cliServer
     );
   };
 
@@ -465,7 +470,8 @@ export class DatabaseUI extends DisposableObject {
       this.databaseManager,
       this.storagePath,
       progress,
-      token
+      token,
+      this.queryServer?.cliServer
     );
   };
 
@@ -575,7 +581,8 @@ export class DatabaseUI extends DisposableObject {
           this.databaseManager,
           this.storagePath,
           progress,
-          token
+          token,
+          this.queryServer?.cliServer
         );
       } else {
         await this.setCurrentDatabase(progress, token, uri);
@@ -583,8 +590,7 @@ export class DatabaseUI extends DisposableObject {
     } catch (e) {
       // rethrow and let this be handled by default error handling.
       throw new Error(
-        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${
-        e.message
+        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${e.message
         }`
       );
     }
@@ -617,7 +623,7 @@ export class DatabaseUI extends DisposableObject {
     });
 
     if (newName) {
-      this.databaseManager.renameDatabaseItem(databaseItem, newName);
+      await this.databaseManager.renameDatabaseItem(databaseItem, newName);
     }
   };
 
@@ -634,6 +640,24 @@ export class DatabaseUI extends DisposableObject {
     }
   };
 
+  /**
+   * Adds the source folder of a CodeQL database to the workspace.
+   * When a database is first added in the "Databases" view, its source folder is added to the workspace.
+   * If the source folder is removed from the workspace for some reason, we want to be able to re-add it if need be.
+   */
+  private handleAddSource = async (
+    databaseItem: DatabaseItem,
+    multiSelect: DatabaseItem[] | undefined
+  ): Promise<void> => {
+    if (multiSelect?.length) {
+      for (const dbItem of multiSelect) {
+        await this.databaseManager.addDatabaseSourceArchiveFolder(dbItem);
+      }
+    } else {
+      await this.databaseManager.addDatabaseSourceArchiveFolder(databaseItem);
+    }
+  };
+
   /**
    * Return the current database directory. If we don't already have a
    * current database, ask the user for one, and return that, or
@@ -674,7 +698,6 @@ export class DatabaseUI extends DisposableObject {
     token: CancellationToken,
   ): Promise<DatabaseItem | undefined> {
     const uri = await chooseDatabaseDir(byFolder);
-
     if (!uri) {
       return undefined;
     }
@@ -691,7 +714,8 @@ export class DatabaseUI extends DisposableObject {
         this.databaseManager,
         this.storagePath,
         progress,
-        token
+        token,
+        this.queryServer?.cliServer
       );
     }
   }
@@ -703,7 +727,7 @@ export class DatabaseUI extends DisposableObject {
    * 2. If the selected URI is a directory matching db-*, choose the containing directory
    * 3. choose the current directory
    *
-   * @param uri a URI that is a datbase folder or inside it
+   * @param uri a URI that is a database folder or inside it
    *
    * @return the actual database folder found by using the heuristics above.
    */

extensions/ql-vscode/src/databases.ts

@@ -15,7 +15,7 @@ import {
   withProgress
 } from './commandRunner';
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { Logger, logger } from './logging';
 import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
 import { QueryServerClient } from './queryserver-client';
@@ -115,7 +115,7 @@ async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
 
   const dbAbsolutePath = path.join(parentDirectory, dbRelativePaths[0]);
   if (dbRelativePaths.length > 1) {
-    showAndLogWarningMessage(`Found multiple dataset directories in database, using '${dbAbsolutePath}'.`);
+    void showAndLogWarningMessage(`Found multiple dataset directories in database, using '${dbAbsolutePath}'.`);
   }
 
   return vscode.Uri.file(dbAbsolutePath);
@@ -138,7 +138,7 @@ async function findSourceArchive(
     }
   }
   if (!silent) {
-    showAndLogInformationMessage(
+    void showAndLogInformationMessage(
       `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`
     );
   }
@@ -265,6 +265,11 @@ export interface DatabaseItem {
    */
   belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
 
+  /**
+   * Whether the database may be affected by test execution for the given path.
+   */
+  isAffectedByTest(testPath: string): Promise<boolean>;
+
   /**
    * Gets the state of this database, to be persisted in the workspace state.
    */
@@ -470,6 +475,27 @@ export class DatabaseItemImpl implements DatabaseItem {
     return uri.scheme === zipArchiveScheme &&
       decodeSourceArchiveUri(uri).sourceArchiveZipPath === this.sourceArchive.fsPath;
   }
+
+  public async isAffectedByTest(testPath: string): Promise<boolean> {
+    const databasePath = this.databaseUri.fsPath;
+    if (!databasePath.endsWith('.testproj')) {
+      return false;
+    }
+    try {
+      const stats = await fs.stat(testPath);
+      if (stats.isDirectory()) {
+        return !path.relative(testPath, databasePath).startsWith('..');
+      } else {
+        // database for /one/two/three/test.ql is at /one/two/three/three.testproj
+        const testdir = path.dirname(testPath);
+        const testdirbase = path.basename(testdir);
+        return databasePath == path.join(testdir, testdirbase + '.testproj');
+      }
+    } catch {
+      // No information available for test path - assume database is unaffected.
+      return false;
+    }
+  }
 }
 
 /**
@@ -480,7 +506,7 @@ export class DatabaseItemImpl implements DatabaseItem {
 function eventFired<T>(event: vscode.Event<T>, timeoutMs = 1000): Promise<T | undefined> {
   return new Promise((res, _rej) => {
     const timeout = setTimeout(() => {
-      logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
+      void logger.log(`Waiting for event ${event} timed out after ${timeoutMs}ms`);
       res(undefined);
       dispose();
     }, timeoutMs);
@@ -517,7 +543,7 @@ export class DatabaseManager extends DisposableObject {
     qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
 
     // Let this run async.
-    this.loadPersistedState();
+    void this.loadPersistedState();
   }
 
   public async openDatabase(
@@ -561,7 +587,7 @@ export class DatabaseManager extends DisposableObject {
     }));
   }
 
-  private async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
+  public async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
     // The folder may already be in workspace state from a previous
     // session. If not, add it.
     const index = this.getDatabaseWorkspaceFolderIndex(item);
@@ -579,15 +605,15 @@ export class DatabaseManager extends DisposableObject {
       const end = (vscode.workspace.workspaceFolders || []).length;
       const uri = item.getSourceArchiveExplorerUri();
       if (uri === undefined) {
-        logger.log(`Couldn't obtain file explorer uri for ${item.name}`);
+        void logger.log(`Couldn't obtain file explorer uri for ${item.name}`);
       }
       else {
-        logger.log(`Adding workspace folder for ${item.name} source archive at index ${end}`);
+        void logger.log(`Adding workspace folder for ${item.name} source archive at index ${end}`);
         if ((vscode.workspace.workspaceFolders || []).length < 2) {
           // Adding this workspace folder makes the workspace
           // multi-root, which may surprise the user. Let them know
           // we're doing this.
-          vscode.window.showInformationMessage(`Adding workspace folder for source archive of database ${item.name}.`);
+          void vscode.window.showInformationMessage(`Adding workspace folder for source archive of database ${item.name}.`);
         }
         vscode.workspace.updateWorkspaceFolders(end, 0, {
           name: `[${item.name} source archive]`,
@@ -670,7 +696,7 @@ export class DatabaseManager extends DisposableObject {
               await databaseItem.refresh();
               await this.registerDatabase(progress, token, databaseItem);
               if (currentDatabaseUri === database.uri) {
-                this.setCurrentDatabaseItem(databaseItem, true);
+                await this.setCurrentDatabaseItem(databaseItem, true);
               }
             }
             catch (e) {
@@ -680,7 +706,7 @@ export class DatabaseManager extends DisposableObject {
           }
         } catch (e) {
           // database list had an unexpected type - nothing to be done?
-          showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
+          void showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
         }
       });
   }
@@ -705,6 +731,8 @@ export class DatabaseManager extends DisposableObject {
       this._currentDatabaseItem = item;
       this.updatePersistedCurrentDatabaseItem();
 
+      await vscode.commands.executeCommand('setContext', 'codeQL.currentDatabaseItem', item?.name);
+      
       this._onDidChangeCurrentDatabaseItem.fire({
         item,
         kind: DatabaseEventKind.Change
@@ -737,7 +765,7 @@ export class DatabaseManager extends DisposableObject {
     item: DatabaseItem
   ) {
     this._databaseItems.push(item);
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
 
     // Add this database item to the allow-list
     // Database items reconstituted from persisted state
@@ -754,7 +782,7 @@ export class DatabaseManager extends DisposableObject {
 
   public async renameDatabaseItem(item: DatabaseItem, newName: string) {
     item.name = newName;
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
     this._onDidChangeDatabaseItem.fire({
       // pass undefined so that the entire tree is rebuilt in order to re-sort
       item: undefined,
@@ -774,28 +802,28 @@ export class DatabaseManager extends DisposableObject {
     if (index >= 0) {
       this._databaseItems.splice(index, 1);
     }
-    this.updatePersistedDatabaseList();
+    await this.updatePersistedDatabaseList();
 
     // Delete folder from workspace, if it is still there
     const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
       folder => item.belongsToSourceArchiveExplorerUri(folder.uri)
     );
     if (folderIndex >= 0) {
-      logger.log(`Removing workspace folder at index ${folderIndex}`);
+      void logger.log(`Removing workspace folder at index ${folderIndex}`);
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
-    // Delete folder from file system only if it is controlled by the extension
-    if (this.isExtensionControlledLocation(item.databaseUri)) {
-      logger.log('Deleting database from filesystem.');
-      fs.remove(item.databaseUri.fsPath).then(
-        () => logger.log(`Deleted '${item.databaseUri.fsPath}'`),
-        e => logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
-    }
-
     // Remove this database item from the allow-list
     await this.deregisterDatabase(progress, token, item);
 
+    // Delete folder from file system only if it is controlled by the extension
+    if (this.isExtensionControlledLocation(item.databaseUri)) {
+      void logger.log('Deleting database from filesystem.');
+      fs.remove(item.databaseUri.fsPath).then(
+        () => void logger.log(`Deleted '${item.databaseUri.fsPath}'`),
+        e => void logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
+    }
+
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
       item: undefined,
@@ -808,7 +836,7 @@ export class DatabaseManager extends DisposableObject {
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.qs.supportsDatabaseRegistration())) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
       const databases: Dataset[] = [{
         dbDir: dbItem.contents.datasetUri.fsPath,
         workingSet: 'default'
@@ -822,7 +850,7 @@ export class DatabaseManager extends DisposableObject {
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.qs.supportsDatabaseRegistration())) {
+    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
       const databases: Dataset[] = [{
         dbDir: dbItem.contents.datasetUri.fsPath,
         workingSet: 'default'
@@ -832,12 +860,12 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private updatePersistedCurrentDatabaseItem(): void {
-    this.ctx.workspaceState.update(CURRENT_DB, this._currentDatabaseItem ?
+    void this.ctx.workspaceState.update(CURRENT_DB, this._currentDatabaseItem ?
       this._currentDatabaseItem.databaseUri.toString(true) : undefined);
   }
 
-  private updatePersistedDatabaseList(): void {
-    this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
+  private async updatePersistedDatabaseList(): Promise<void> {
+    await this.ctx.workspaceState.update(DB_LIST, this._databaseItems.map(item => item.getPersistedState()));
   }
 
   private isExtensionControlledLocation(uri: vscode.Uri) {
@@ -852,7 +880,7 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async getPrimaryLanguage(dbPath: string) {
-    if (!(await this.cli.supportsLanguageName())) {
+    if (!(await this.cli.cliConstraints.supportsLanguageName())) {
       // return undefined so that we recalculate on restart until the cli is at a version that
       // supports this feature. This recalculation is cheap since we avoid calling into the cli
       // unless we know it can return the langauges property.

extensions/ql-vscode/src/discovery.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { logger } from './logging';
 
 /**
@@ -59,23 +59,23 @@ export abstract class Discovery<T> extends DisposableObject {
         this.discoveryInProgress = false;
         this.update(results);
       }
-    });
+    })
 
-    discoveryPromise.catch(err => {
-      logger.log(`${this.name} failed. Reason: ${err.message}`);
-    });
+      .catch(err => {
+        void logger.log(`${this.name} failed. Reason: ${err.message}`);
+      })
 
-    discoveryPromise.finally(() => {
-      if (this.retry) {
-        // Another refresh request came in while we were still running a previous discovery
-        // operation. Since the discovery results we just computed are now stale, we'll launch
-        // another discovery operation instead of updating.
-        // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-        // initial discovery operation failed.
-        this.retry = false;
-        this.launchDiscovery();
-      }
-    });
+      .finally(() => {
+        if (this.retry) {
+          // Another refresh request came in while we were still running a previous discovery
+          // operation. Since the discovery results we just computed are now stale, we'll launch
+          // another discovery operation instead of updating.
+          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
+          // initial discovery operation failed.
+          this.retry = false;
+          this.launchDiscovery();
+        }
+      });
   }
 
   /**

extensions/ql-vscode/src/distribution.ts

@@ -153,7 +153,7 @@ export class DistributionManager implements DistributionProvider {
     // Check config setting, then extension specific distribution, then PATH.
     if (this.config.customCodeQlPath) {
       if (!await fs.pathExists(this.config.customCodeQlPath)) {
-        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
+        void showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
           'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
           'that a CodeQL executable exists at the specified path or remove the setting.');
         return undefined;
@@ -191,7 +191,7 @@ export class DistributionManager implements DistributionProvider {
           };
         }
       }
-      logger.log('INFO: Could not find CodeQL on path.');
+      void logger.log('INFO: Could not find CodeQL on path.');
     }
 
     return undefined;
@@ -276,7 +276,7 @@ class ExtensionSpecificDistributionManager {
       try {
         await this.removeDistribution();
       } catch (e) {
-        logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
+        void logger.log('WARNING: Tried to remove corrupted CodeQL CLI at ' +
           `${this.getDistributionStoragePath()} but encountered an error: ${e}.`);
       }
     }
@@ -313,7 +313,7 @@ class ExtensionSpecificDistributionManager {
     progressCallback?: ProgressCallback): Promise<void> {
     await this.downloadDistribution(release, progressCallback);
     // Store the installed release within the global extension state.
-    this.storeInstalledRelease(release);
+    await this.storeInstalledRelease(release);
   }
 
   private async downloadDistribution(release: Release,
@@ -321,7 +321,7 @@ class ExtensionSpecificDistributionManager {
     try {
       await this.removeDistribution();
     } catch (e) {
-      logger.log(`Tried to clean up old version of CLI at ${this.getDistributionStoragePath()} ` +
+      void logger.log(`Tried to clean up old version of CLI at ${this.getDistributionStoragePath()} ` +
         `but encountered an error: ${e}.`);
     }
 
@@ -332,7 +332,7 @@ class ExtensionSpecificDistributionManager {
       throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
     }
     if (assets.length > 1) {
-      logger.log('WARNING: chose a release with more than one asset to install, found ' +
+      void logger.log('WARNING: chose a release with more than one asset to install, found ' +
         assets.map(asset => asset.name).join(', '));
     }
 
@@ -345,7 +345,7 @@ class ExtensionSpecificDistributionManager {
 
       const contentLength = assetStream.headers.get('content-length');
       const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined;
-      reportStreamProgress(assetStream.body, 'Downloading CodeQL CLI…', totalNumBytes, progressCallback);
+      reportStreamProgress(assetStream.body, `Downloading CodeQL CLI ${release.name}…`, totalNumBytes, progressCallback);
 
       await new Promise((resolve, reject) =>
         assetStream.body.pipe(archiveFile)
@@ -355,7 +355,7 @@ class ExtensionSpecificDistributionManager {
 
       await this.bumpDistributionFolderIndex();
 
-      logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
+      void logger.log(`Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`);
       await extractZipArchive(archivePath, this.getDistributionStoragePath());
     } finally {
       await fs.remove(tmpDirectory);
@@ -368,7 +368,7 @@ class ExtensionSpecificDistributionManager {
    * This should not be called for a distribution that is currently in use, as remove may fail.
    */
   private async removeDistribution(): Promise<void> {
-    this.storeInstalledRelease(undefined);
+    await this.storeInstalledRelease(undefined);
     if (await fs.pathExists(this.getDistributionStoragePath())) {
       await fs.remove(this.getDistributionStoragePath());
     }
@@ -376,7 +376,7 @@ class ExtensionSpecificDistributionManager {
 
   private async getLatestRelease(): Promise<Release> {
     const requiredAssetName = DistributionManager.getRequiredAssetName();
-    logger.log(`Searching for latest release including ${requiredAssetName}.`);
+    void logger.log(`Searching for latest release including ${requiredAssetName}.`);
     return this.createReleasesApiConsumer().getLatestRelease(
       this.versionRange,
       this.config.includePrerelease,
@@ -384,11 +384,11 @@ class ExtensionSpecificDistributionManager {
         const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
         if (matchingAssets.length === 0) {
           // For example, this could be a release with no platform-specific assets.
-          logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
+          void logger.log(`INFO: Ignoring a release with no assets named ${requiredAssetName}`);
           return false;
         }
         if (matchingAssets.length > 1) {
-          logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
+          void logger.log(`WARNING: Ignoring a release with more than one asset named ${requiredAssetName}`);
           return false;
         }
         return true;
@@ -707,16 +707,14 @@ export async function getExecutableFromDirectory(directory: string, warnWhenNotF
     return alternateExpectedLauncherPath;
   }
   if (warnWhenNotFound) {
-    logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
+    void logger.log(`WARNING: Expected to find a CodeQL CLI executable at ${expectedLauncherPath} but one was not found. ` +
       'Will try PATH.');
   }
   return undefined;
 }
 
 function warnDeprecatedLauncher() {
-
-  showAndLogWarningMessage(
-
+  void showAndLogWarningMessage(
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
     `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`
   );

extensions/ql-vscode/src/extension.ts

@@ -10,24 +10,28 @@ import {
   Uri,
   window as Window,
   env,
-  window
+  window,
+  QuickPickItem
 } from 'vscode';
 import { LanguageClient } from 'vscode-languageclient';
+import * as os from 'os';
 import * as path from 'path';
+import * as tmp from 'tmp-promise';
 import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
 
 import { AstViewer } from './astViewer';
 import * as archiveFilesystemProvider from './archive-filesystem-provider';
-import { CodeQLCliServer } from './cli';
+import { CodeQLCliServer, CliVersionConstraint } from './cli';
 import {
   CliConfigListener,
   DistributionConfigListener,
+  isCanary,
   MAX_QUERIES,
   QueryHistoryConfigListener,
   QueryServerConfigListener
 } from './config';
 import * as languageSupport from './languageSupport';
-import { DatabaseManager } from './databases';
+import { DatabaseItem, DatabaseManager } from './databases';
 import { DatabaseUI } from './databases-ui';
 import {
   TemplateQueryDefinitionProvider,
@@ -67,6 +71,10 @@ import {
   withProgress,
   ProgressUpdate
 } from './commandRunner';
+import { CodeQlStatusBarHandler } from './status-bar';
+
+import { Credentials } from './authentication';
+import { runRemoteQuery } from './run-remote-query';
 
 /**
  * extension.ts
@@ -138,7 +146,7 @@ export interface CodeQLExtensionInterface {
 
 /**
  * Returns the CodeQLExtensionInterface, or an empty object if the interface is not
- * available afer activation is complete. This will happen if there is no cli
+ * available after activation is complete. This will happen if there is no cli
  * installed when the extension starts. Downloading and installing the cli
  * will happen at a later time.
  *
@@ -146,14 +154,14 @@ export interface CodeQLExtensionInterface {
  *
  * @returns CodeQLExtensionInterface
  */
-export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | {}> {
-  logger.log(`Starting ${extensionId} extension`);
+export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | Record<string, never>> {
+  void logger.log(`Starting ${extensionId} extension`);
   if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
   }
 
   const distributionConfigListener = new DistributionConfigListener();
-  initializeLogging(ctx);
+  await initializeLogging(ctx);
   await initializeTelemetry(extension, ctx);
   languageSupport.install();
 
@@ -164,7 +172,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
   const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
   registerErrorStubs([checkForUpdatesCommand], command => (async () => {
-    helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
+    void helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
   }));
 
   interface DistributionUpdateConfig {
@@ -176,7 +184,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
   async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
     const minSecondsSinceLastUpdateCheck = config.isUserInitiated ? 0 : 86400;
     const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
-      helpers.showAndLogInformationMessage : async (message: string) => logger.log(message);
+      helpers.showAndLogInformationMessage : async (message: string) => void logger.log(message);
     const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
 
     // We do want to auto update if there is no distribution at all
@@ -184,7 +192,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
 
     switch (result.kind) {
       case DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult:
-        logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
+        void logger.log('Didn\'t perform CodeQL CLI update check since a check was already performed within the previous ' +
           `${minSecondsSinceLastUpdateCheck} seconds.`);
         break;
       case DistributionUpdateCheckResultKind.AlreadyUpToDate:
@@ -211,7 +219,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
-          helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
+          void helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
         }
         break;
       default:
@@ -243,12 +251,12 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
         codeQlInstalled ? 'check for updates to' : 'install') + ' CodeQL CLI';
 
       if (e instanceof GithubRateLimitedError) {
-        alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
+        void alertFunction(`Rate limited while trying to ${taskDescription}. Please try again after ` +
           `your rate limit window resets at ${e.rateLimitResetDate.toLocaleString(env.language)}.`);
       } else if (e instanceof GithubApiError) {
-        alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
+        void alertFunction(`Encountered GitHub API error while trying to ${taskDescription}. ` + e);
       }
-      alertFunction(`Unable to ${taskDescription}. ` + e);
+      void alertFunction(`Unable to ${taskDescription}. ` + e);
     } finally {
       isInstallingOrUpdatingDistribution = false;
     }
@@ -258,7 +266,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     const result = await distributionManager.getDistribution();
     switch (result.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
-        logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
+        void logger.log(`Found compatible version of CodeQL CLI (version ${result.version.raw})`);
         break;
       case FindDistributionResultKind.IncompatibleDistribution: {
         const fixGuidanceMessage = (() => {
@@ -273,16 +281,20 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
           }
         })();
 
-        helpers.showAndLogWarningMessage(`The current version of the CodeQL CLI (${result.version.raw}) ` +
-          'is incompatible with this extension. ' + fixGuidanceMessage);
+        void helpers.showAndLogWarningMessage(
+          `The current version of the CodeQL CLI (${result.version.raw}) ` +
+          `is incompatible with this extension. ${fixGuidanceMessage}`
+        );
         break;
       }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
-        helpers.showAndLogWarningMessage('Compatibility with the configured CodeQL CLI could not be determined. ' +
-          'You may experience problems using the extension.');
+        void helpers.showAndLogWarningMessage(
+          'Compatibility with the configured CodeQL CLI could not be determined. ' +
+          'You may experience problems using the extension.'
+        );
         break;
       case FindDistributionResultKind.NoDistribution:
-        helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
+        void helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
         break;
       default:
         assertNever(result);
@@ -290,18 +302,26 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<CodeQLExtensionInterface | {}> {
+  async function installOrUpdateThenTryActivate(
+    config: DistributionUpdateConfig
+  ): Promise<CodeQLExtensionInterface | Record<string, never>> {
+
     await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
-    let extensionInterface: CodeQLExtensionInterface | {} = {};
+    let extensionInterface: CodeQLExtensionInterface | Record<string, never> = {};
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      extensionInterface = await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(
+        ctx,
+        distributionManager,
+        distributionConfigListener
+      );
+
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = 'Install CodeQL CLI';
-        const chosenAction = await helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+        const chosenAction = await void helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
           items: [installActionName]
         });
         if (chosenAction === installActionName) {
@@ -339,20 +359,21 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
 
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
-  distributionManager: DistributionManager
+  distributionManager: DistributionManager,
+  distributionConfigListener: DistributionConfigListener
 ): Promise<CodeQLExtensionInterface> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
   errorStubs.forEach((stub) => stub.dispose());
 
-  logger.log('Initializing configuration listener...');
+  void logger.log('Initializing configuration listener...');
   const qlConfigurationListener = await QueryServerConfigListener.createQueryServerConfigListener(
     distributionManager
   );
   ctx.subscriptions.push(qlConfigurationListener);
 
-  logger.log('Initializing CodeQL cli server...');
+  void logger.log('Initializing CodeQL cli server...');
   const cliServer = new CodeQLCliServer(
     distributionManager,
     new CliConfigListener(),
@@ -360,12 +381,16 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cliServer);
 
-  logger.log('Initializing query server client.');
+  const statusBar = new CodeQlStatusBarHandler(cliServer, distributionConfigListener);
+  ctx.subscriptions.push(statusBar);
+
+  void logger.log('Initializing query server client.');
   const qs = new qsClient.QueryServerClient(
     qlConfigurationListener,
     cliServer,
     {
       logger: queryServerLogger,
+      contextStoragePath: getContextStoragePath(ctx),
     },
     (task) =>
       Window.withProgress(
@@ -376,10 +401,10 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(qs);
   await qs.startQueryServer();
 
-  logger.log('Initializing database manager.');
+  void logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
   ctx.subscriptions.push(dbm);
-  logger.log('Initializing database panel.');
+  void logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(
     dbm,
     qs,
@@ -389,7 +414,7 @@ async function activateWithInstalledDistribution(
   databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
-  logger.log('Initializing query history manager.');
+  void logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
   ctx.subscriptions.push(queryHistoryConfigurationListener);
   const showResults = async (item: CompletedQuery) =>
@@ -404,11 +429,11 @@ async function activateWithInstalledDistribution(
       showResultsForComparison(from, to),
   );
   ctx.subscriptions.push(qhm);
-  logger.log('Initializing results panel interface.');
+  void logger.log('Initializing results panel interface.');
   const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
   ctx.subscriptions.push(intm);
 
-  logger.log('Initializing compare panel interface.');
+  void logger.log('Initializing compare panel interface.');
   const cmpm = new CompareInterfaceManager(
     ctx,
     dbm,
@@ -418,7 +443,7 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(cmpm);
 
-  logger.log('Initializing source archive filesystem provider.');
+  void logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
   async function showResultsForComparison(
@@ -428,7 +453,7 @@ async function activateWithInstalledDistribution(
     try {
       await cmpm.showResults(from, to);
     } catch (e) {
-      helpers.showAndLogErrorMessage(e.message);
+      void helpers.showAndLogErrorMessage(e.message);
     }
   }
 
@@ -444,33 +469,80 @@ async function activateWithInstalledDistribution(
     selectedQuery: Uri | undefined,
     progress: ProgressCallback,
     token: CancellationToken,
+    databaseItem: DatabaseItem | undefined,
   ): Promise<void> {
     if (qs !== undefined) {
-      const dbItem = await databaseUI.getDatabaseItem(progress, token);
-      if (dbItem === undefined) {
+      // If no databaseItem is specified, use the database currently selected in the Databases UI
+      databaseItem = databaseItem || await databaseUI.getDatabaseItem(progress, token);
+      if (databaseItem === undefined) {
         throw new Error('Can\'t run query without a selected database');
       }
       const info = await compileAndRunQueryAgainstDatabase(
         cliServer,
         qs,
-        dbItem,
+        databaseItem,
         quickEval,
         selectedQuery,
         progress,
         token
       );
-      const item = qhm.addQuery(info);
+      const item = qhm.buildCompletedQuery(info);
       await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      // The call to showResults potentially creates SARIF file;
-      // Update the tree item context value to allow viewing that
-      // SARIF file from context menu.
-      await qhm.refreshTreeView(item);
+      // Note we must update the query history view after showing results as the
+      // display and sorting might depend on the number of results
+      await qhm.addCompletedQuery(item);
+    }
+  }
+
+  const qhelpTmpDir = tmp.dirSync({ prefix: 'qhelp_', keep: false, unsafeCleanup: true });
+  ctx.subscriptions.push({ dispose: qhelpTmpDir.removeCallback });
+
+  async function previewQueryHelp(
+    selectedQuery: Uri
+  ): Promise<void> {
+    // selectedQuery is unpopulated when executing through the command palette
+    const pathToQhelp =  selectedQuery ? selectedQuery.fsPath : window.activeTextEditor?.document.uri.fsPath;
+    if(pathToQhelp) {
+      // Create temporary directory
+      const relativePathToMd = path.basename(pathToQhelp, '.qhelp') + '.md';
+      const absolutePathToMd = path.join(qhelpTmpDir.name, relativePathToMd);
+      const uri = Uri.file(absolutePathToMd);
+      try {
+        await cliServer.generateQueryHelp(pathToQhelp , absolutePathToMd);
+        await commands.executeCommand('markdown.showPreviewToSide', uri);
+      } catch (err) {
+        const errorMessage =  err.message.includes('Generating qhelp in markdown') ? (
+          `Could not generate markdown from ${pathToQhelp}: Bad formatting in .qhelp file.`
+        ) : `Could not open a preview of the generated file (${absolutePathToMd}).`;
+        void helpers.showAndLogErrorMessage(errorMessage, { fullMessage: `${errorMessage}\n${err}` });
+      }
+    }
+    
+  }
+
+  async function openReferencedFile(
+    selectedQuery: Uri
+  ): Promise<void> {
+    // If no file is selected, the path of the file in the editor is selected
+    const path = selectedQuery?.fsPath || window.activeTextEditor?.document.uri.fsPath;
+    if (qs !== undefined && path) {
+      if (await cliServer.cliConstraints.supportsResolveQlref()) {
+        const resolved = await cliServer.resolveQlref(path);
+        const uri = Uri.file(resolved.resolvedPath);
+        await window.showTextDocument(uri, { preview: false });
+      } else {
+        void helpers.showAndLogErrorMessage(
+          'Jumping from a .qlref file to the .ql file it references is not '
+          + 'supported with the CLI version you are running.\n'
+          + `Please upgrade your CLI to version ${CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF
+          } or later to use this feature.`);
+      }
     }
   }
 
   ctx.subscriptions.push(tmpDirDisposal);
 
-  logger.log('Initializing CodeQL language server.');
+  void logger.log('Initializing CodeQL language server.');
   const client = new LanguageClient(
     'CodeQL Language Server',
     () => spawnIdeServer(qlConfigurationListener),
@@ -488,20 +560,20 @@ async function activateWithInstalledDistribution(
     true
   );
 
-  logger.log('Initializing QLTest interface.');
+  void logger.log('Initializing QLTest interface.');
   const testExplorerExtension = extensions.getExtension<TestHub>(
     testExplorerExtensionId
   );
   if (testExplorerExtension) {
     const testHub = testExplorerExtension.exports;
-    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer);
+    const testAdapterFactory = new QLTestAdapterFactory(testHub, cliServer, dbm);
     ctx.subscriptions.push(testAdapterFactory);
 
     const testUIService = new TestUIService(testHub);
     ctx.subscriptions.push(testUIService);
   }
 
-  logger.log('Registering top-level command palette commands.');
+  void logger.log('Registering top-level command palette commands.');
   ctx.subscriptions.push(
     commandRunnerWithProgress(
       'codeQL.runQuery',
@@ -509,13 +581,80 @@ async function activateWithInstalledDistribution(
         progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
-      ) => await compileAndRunQuery(false, uri, progress, token),
+      ) => await compileAndRunQuery(false, uri, progress, token, undefined),
       {
         title: 'Running query',
         cancellable: true
       }
     )
   );
+  interface DatabaseQuickPickItem extends QuickPickItem {
+    databaseItem: DatabaseItem;
+  }
+  ctx.subscriptions.push(
+    commandRunnerWithProgress(
+      'codeQL.runQueryOnMultipleDatabases',
+      async (
+        progress: ProgressCallback,
+        token: CancellationToken,
+        uri: Uri | undefined
+      ) => {
+        let filteredDBs = dbm.databaseItems;
+        if (filteredDBs.length === 0) {
+          void helpers.showAndLogErrorMessage('No databases found. Please add a suitable database to your workspace.');
+          return;
+        }
+        // If possible, only show databases with the right language (otherwise show all databases).
+        const queryLanguage = await helpers.findLanguage(cliServer, uri);
+        if (queryLanguage) {
+          filteredDBs = dbm.databaseItems.filter(db => db.language === queryLanguage);
+          if (filteredDBs.length === 0) {
+            void helpers.showAndLogErrorMessage(`No databases found for language ${queryLanguage}. Please add a suitable database to your workspace.`);
+            return;
+          }
+        }
+        const quickPickItems = filteredDBs.map<DatabaseQuickPickItem>(dbItem => (
+          {
+            databaseItem: dbItem,
+            label: dbItem.name,
+            description: dbItem.language,
+          }
+        ));
+        /**
+         * Databases that were selected in the quick pick menu.
+         */
+        const quickpick = await window.showQuickPick<DatabaseQuickPickItem>(
+          quickPickItems,
+          { canPickMany: true, ignoreFocusOut: true }
+        );
+        if (quickpick !== undefined) {
+          // Collect all skipped databases and display them at the end (instead of popping up individual errors)
+          const skippedDatabases = [];
+          const errors = [];
+          for (const item of quickpick) {
+            try {
+              await compileAndRunQuery(false, uri, progress, token, item.databaseItem);
+            } catch (error) {
+              skippedDatabases.push(item.label);
+              errors.push(error.message);
+            }
+          }
+          if (skippedDatabases.length > 0) {
+            void logger.log(`Errors:\n${errors.join('\n')}`);
+            void helpers.showAndLogWarningMessage(
+              `The following databases were skipped:\n${skippedDatabases.join('\n')}.\nFor details about the errors, see the logs.`
+            );
+          }
+        } else {
+          void helpers.showAndLogErrorMessage('No databases selected.');
+        }
+      },
+      {
+        title: 'Running query on selected databases',
+        cancellable: true
+      }
+    )
+  );
   ctx.subscriptions.push(
     commandRunnerWithProgress(
       'codeQL.runQueries',
@@ -571,7 +710,7 @@ async function activateWithInstalledDistribution(
         });
 
         await Promise.all(queryUris.map(async uri =>
-          compileAndRunQuery(false, uri, wrappedProgress, token)
+          compileAndRunQuery(false, uri, wrappedProgress, token, undefined)
             .then(() => queriesRemaining--)
         ));
       },
@@ -587,7 +726,7 @@ async function activateWithInstalledDistribution(
         progress: ProgressCallback,
         token: CancellationToken,
         uri: Uri | undefined
-      ) => await compileAndRunQuery(true, uri, progress, token),
+      ) => await compileAndRunQuery(true, uri, progress, token, undefined),
       {
         title: 'Running query',
         cancellable: true
@@ -604,6 +743,42 @@ async function activateWithInstalledDistribution(
       }
     )
   );
+  // The "runRemoteQuery" command is internal-only.
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.runRemoteQuery', async (
+      progress: ProgressCallback,
+      token: CancellationToken,
+      uri: Uri | undefined
+    ) => {
+      if (isCanary()) {
+        progress({
+          maxStep: 5,
+          step: 0,
+          message: 'Getting credentials'
+        });
+        const credentials = await Credentials.initialize(ctx);
+        await runRemoteQuery(cliServer, credentials, uri || window.activeTextEditor?.document.uri, false, progress, token);
+      } else {
+        throw new Error('Remote queries require the CodeQL Canary version to run.');
+      }
+    }, {
+      title: 'Run Remote Query',
+      cancellable: true
+    })
+  );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.openReferencedFile',
+      openReferencedFile
+    )
+  );
+
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.previewQueryHelp',
+      previewQueryHelp
+    )
+  );
 
   ctx.subscriptions.push(
     commandRunnerWithProgress('codeQL.restartQueryServer', async (
@@ -611,7 +786,7 @@ async function activateWithInstalledDistribution(
       token: CancellationToken
     ) => {
       await qs.restartQueryServer(progress, token);
-      helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
+      void helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
       });
     }, {
@@ -659,11 +834,49 @@ async function activateWithInstalledDistribution(
       })
   );
 
-  logger.log('Starting language server.');
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openDocumentation', async () =>
+      env.openExternal(Uri.parse('https://codeql.github.com/docs/'))));
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.copyVersion', async () => {
+      const text = `CodeQL extension version: ${extension?.packageJSON.version} \nCodeQL CLI version: ${await getCliVersion()} \nPlatform: ${os.platform()} ${os.arch()}`;
+      await env.clipboard.writeText(text);
+      void helpers.showAndLogInformationMessage(text);
+    }));
+
+  const getCliVersion = async () => {
+    try {
+      return await cliServer.getVersion();
+    } catch {
+      return '<missing>';
+    }
+  };
+
+  // The "authenticateToGitHub" command is internal-only.
+  ctx.subscriptions.push(
+    commandRunner('codeQL.authenticateToGitHub', async () => {
+      if (isCanary()) {
+        /**
+         * Credentials for authenticating to GitHub.
+         * These are used when making API calls.
+         */
+        const credentials = await Credentials.initialize(ctx);
+        const octokit = await credentials.getOctokit();
+        const userInfo = await octokit.users.getAuthenticated();
+        void helpers.showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
+      }
+    }));
+
+  commands.registerCommand('codeQL.showLogs', () => {
+    logger.show();
+  });
+
+  void logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
   // Jump-to-definition and find-references
-  logger.log('Registering jump-to-definition handlers.');
+  void logger.log('Registering jump-to-definition handlers.');
   languages.registerDefinitionProvider(
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
     new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
@@ -675,13 +888,18 @@ async function activateWithInstalledDistribution(
   );
 
   const astViewer = new AstViewer();
+  const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm);
+
   ctx.subscriptions.push(astViewer);
   ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
     progress: ProgressCallback,
     token: CancellationToken
   ) => {
-    const ast = await new TemplatePrintAstProvider(cliServer, qs, dbm, progress, token)
-      .provideAst(window.activeTextEditor?.document);
+    const ast = await templateProvider.provideAst(
+      progress,
+      token,
+      window.activeTextEditor?.document,
+    );
     if (ast) {
       astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
     }
@@ -690,9 +908,9 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
-  commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
+  await commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
 
-  logger.log('Successfully finished extension initialization.');
+  void logger.log('Successfully finished extension initialization.');
 
   return {
     ctx,
@@ -711,11 +929,10 @@ function getContextStoragePath(ctx: ExtensionContext) {
   return ctx.storagePath || ctx.globalStoragePath;
 }
 
-function initializeLogging(ctx: ExtensionContext): void {
+async function initializeLogging(ctx: ExtensionContext): Promise<void> {
   const storagePath = getContextStoragePath(ctx);
-  logger.init(storagePath);
-  queryServerLogger.init(storagePath);
-  ideServerLogger.init(storagePath);
+  await logger.setLogStoragePath(storagePath, false);
+  await ideServerLogger.setLogStoragePath(storagePath, false);
   ctx.subscriptions.push(logger);
   ctx.subscriptions.push(queryServerLogger);
   ctx.subscriptions.push(ideServerLogger);

extensions/ql-vscode/src/helpers.ts

@@ -4,10 +4,13 @@ import * as yaml from 'js-yaml';
 import * as path from 'path';
 import {
   ExtensionContext,
+  Uri,
   window as Window,
-  workspace
+  workspace,
+  env
 } from 'vscode';
-import { CodeQLCliServer } from './cli';
+import { CodeQLCliServer, QlpacksInfo } from './cli';
+import { UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 
 /**
@@ -27,8 +30,13 @@ export async function showAndLogErrorMessage(message: string, {
   items = [] as string[],
   fullMessage = undefined as (string | undefined)
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showErrorMessage, fullMessage);
+  return internalShowAndLog(dropLinesExceptInitial(message), items, outputLogger, Window.showErrorMessage, fullMessage);
 }
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join('\n');
+}
+
 /**
  * Show a warning message and log it to the console
  *
@@ -70,7 +78,7 @@ async function internalShowAndLog(
   fullMessage?: string
 ): Promise<string | undefined> {
   const label = 'Show Log';
-  outputLogger.log(fullMessage || message);
+  void outputLogger.log(fullMessage || message);
   const result = await fn(message, label, ...items);
   if (result === label) {
     outputLogger.show();
@@ -100,6 +108,41 @@ export async function showBinaryChoiceDialog(message: string, modal = true): Pro
   return chosenItem?.title === yesItem.title;
 }
 
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(message: string, url: string): Promise<boolean | undefined> {
+  const urlItem = { title: 'More Information', isCloseAffordance: false };
+  const yesItem = { title: 'Yes', isCloseAffordance: false };
+  const noItem = { title: 'No', isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await Window.showInformationMessage(message, { modal: true }, urlItem, yesItem, noItem);
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
 /**
  * Show an information message with a customisable action.
  * @param message The message to show.
@@ -212,31 +255,75 @@ function createRateLimitedResult(): RateLimitedResult {
   };
 }
 
-export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
+export interface QlPacksForLanguage {
+  /** The name of the pack containing the dbscheme. */
+  dbschemePack: string;
+  /** `true` if `dbschemePack` is a library pack. */
+  dbschemePackIsLibraryPack: boolean;
+  /**
+   * The name of the corresponding standard query pack.
+   * Only defined if `dbschemePack` is a library pack.
+   */
+  queryPack?: string;
+}
+
+interface QlPackWithPath {
+  packName: string;
+  packDir: string | undefined;
+}
+
+async function findDbschemePack(packs: QlPackWithPath[], dbschemePath: string): Promise<{ name: string; isLibraryPack: boolean; }> {
+  for (const { packDir, packName } of packs) {
+    if (packDir !== undefined) {
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
+      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
+        return {
+          name: packName,
+          isLibraryPack: qlpack.library === true
+        };
+      }
+    }
+  }
+  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+}
+
+function findStandardQueryPack(qlpacks: QlpacksInfo, dbschemePackName: string): string | undefined {
+  const matches = dbschemePackName.match(/^codeql\/(?<language>[a-z]+)-all$/);
+  if (matches) {
+    const queryPackName = `codeql/${matches.groups!.language}-queries`;
+    if (qlpacks[queryPackName] !== undefined) {
+      return queryPackName;
+    }
+  }
+
+  // Either the dbscheme pack didn't look like one where the queries might be in the query pack, or
+  // no query pack was found in the search path. Either is OK.
+  return undefined;
+}
+
+export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<QlPacksForLanguage> {
   const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
-  const packs: { packDir: string | undefined; packName: string }[] =
+  const packs: QlPackWithPath[] =
     Object.entries(qlpacks).map(([packName, dirs]) => {
       if (dirs.length < 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
+        void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
         return { packName, packDir: undefined };
       }
       if (dirs.length > 1) {
-        logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
+        void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`);
       }
       return {
         packName,
         packDir: dirs[0]
       };
     });
-  for (const { packDir, packName } of packs) {
-    if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme: string };
-      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
-        return packName;
-      }
-    }
-  }
-  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+  const dbschemePack = await findDbschemePack(packs, dbschemePath);
+  const queryPack = dbschemePack.isLibraryPack ? findStandardQueryPack(qlpacks, dbschemePack.name) : undefined;
+  return {
+    dbschemePack: dbschemePack.name,
+    dbschemePackIsLibraryPack: dbschemePack.isLibraryPack,
+    queryPack
+  };
 }
 
 export async function getPrimaryDbscheme(datasetFolder: string): Promise<string> {
@@ -250,7 +337,7 @@ export async function getPrimaryDbscheme(datasetFolder: string): Promise<string>
   const dbscheme = dbschemes[0];
 
   if (dbschemes.length > 1) {
-    Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
+    void Window.showErrorMessage(`Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`);
   }
   return dbscheme;
 }
@@ -259,19 +346,19 @@ export async function getPrimaryDbscheme(datasetFolder: string): Promise<string>
  * A cached mapping from strings to value of type U.
  */
 export class CachedOperation<U> {
-  private readonly operation: (t: string) => Promise<U>;
+  private readonly operation: (t: string, ...args: any[]) => Promise<U>;
   private readonly cached: Map<string, U>;
   private readonly lru: string[];
   private readonly inProgressCallbacks: Map<string, [(u: U) => void, (reason?: any) => void][]>;
 
-  constructor(operation: (t: string) => Promise<U>, private cacheSize = 100) {
+  constructor(operation: (t: string, ...args: any[]) => Promise<U>, private cacheSize = 100) {
     this.operation = operation;
     this.lru = [];
     this.inProgressCallbacks = new Map<string, [(u: U) => void, (reason?: any) => void][]>();
     this.cached = new Map<string, U>();
   }
 
-  async get(t: string): Promise<U> {
+  async get(t: string, ...args: any[]): Promise<U> {
     // Try and retrieve from the cache
     const fromCache = this.cached.get(t);
     if (fromCache !== undefined) {
@@ -292,7 +379,7 @@ export class CachedOperation<U> {
     const callbacks: [(u: U) => void, (reason?: any) => void][] = [];
     this.inProgressCallbacks.set(t, callbacks);
     try {
-      const result = await this.operation(t);
+      const result = await this.operation(t, ...args);
       callbacks.forEach(f => f[0](result));
       this.inProgressCallbacks.delete(t);
       if (this.lru.length > this.cacheSize) {
@@ -325,18 +412,25 @@ export class CachedOperation<U> {
  * `cli.CodeQLCliServer.resolveDatabase` and use the first entry in the
  * `languages` property.
  *
- * @see cli.CodeQLCliServer.supportsLanguageName
+ * @see cli.CliVersionConstraint.supportsLanguageName
  * @see cli.CodeQLCliServer.resolveDatabase
  */
-const dbSchemeToLanguage = {
+export const dbSchemeToLanguage = {
   'semmlecode.javascript.dbscheme': 'javascript',
   'semmlecode.cpp.dbscheme': 'cpp',
   'semmlecode.dbscheme': 'java',
   'semmlecode.python.dbscheme': 'python',
   'semmlecode.csharp.dbscheme': 'csharp',
-  'go.dbscheme': 'go'
+  'go.dbscheme': 'go',
+  'ruby.dbscheme': 'ruby'
 };
 
+export const languageToDbScheme = Object.entries(dbSchemeToLanguage).reduce((acc, [k, v]) => {
+  acc[v] = k;
+  return acc;
+}, {} as { [k: string]: string });
+
+
 /**
  * Returns the initial contents for an empty query, based on the language of the selected
  * databse.
@@ -382,3 +476,43 @@ export async function isLikelyDatabaseRoot(maybeRoot: string) {
 export function isLikelyDbLanguageFolder(dbPath: string) {
   return !!path.basename(dbPath).startsWith('db-');
 }
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined
+): Promise<string | undefined> {
+  const uri = queryUri || Window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(getOnDiskWorkspaceFolders(), uri);
+      const language = (Object.keys(queryInfo.byLanguage))[0];
+      void logger.log(`Detected query language: ${language}`);
+      return language;
+    } catch (e) {
+      void logger.log('Could not autodetect query language. Select language manually.');
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(cliServer: CodeQLCliServer, throwOnEmpty = true): Promise<string | undefined> {
+  const language = await Window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    { placeHolder: 'Select target language for your query', ignoreFocusOut: true }
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException('Cancelled.');
+    } else {
+      void showAndLogErrorMessage('Language not found. Language must be specified manually.');
+    }
+  }
+  return language;
+}

extensions/ql-vscode/src/interface-utils.ts

@@ -70,7 +70,7 @@ function resolveFivePartLocation(
     Math.max(0, loc.startLine - 1),
     Math.max(0, loc.startColumn - 1),
     Math.max(0, loc.endLine - 1),
-    Math.max(0, loc.endColumn)
+    Math.max(1, loc.endColumn)
   );
 
   return new Location(databaseItem.resolveSourceFile(loc.uri), range);
@@ -224,14 +224,14 @@ export async function jumpToLocation(
     } catch (e) {
       if (e instanceof Error) {
         if (e.message.match(/File not found/)) {
-          Window.showErrorMessage(
+          void Window.showErrorMessage(
             'Original file of this result is not in the database\'s source archive.'
           );
         } else {
-          logger.log(`Unable to handleMsgFromView: ${e.message}`);
+          void logger.log(`Unable to handleMsgFromView: ${e.message}`);
         }
       } else {
-        logger.log(`Unable to handleMsgFromView: ${e}`);
+        void logger.log(`Unable to handleMsgFromView: ${e}`);
       }
     }
   }

extensions/ql-vscode/src/interface.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as Sarif from 'sarif';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as vscode from 'vscode';
 import {
   Diagnostic,
@@ -119,7 +119,7 @@ export class InterfaceManager extends DisposableObject {
         this.handleSelectionChange.bind(this)
       )
     );
-    logger.log('Registering path-step navigation commands.');
+    void logger.log('Registering path-step navigation commands.');
     this.push(
       commandRunner(
         'codeQLQueryResults.nextPathStep',
@@ -137,16 +137,22 @@ export class InterfaceManager extends DisposableObject {
       this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
         if (kind === DatabaseEventKind.Remove) {
           this._diagnosticCollection.clear();
-          this.postMessage({
-            t: 'untoggleShowProblems'
-          });
+          if (this.isShowingPanel()) {
+            void this.postMessage({
+              t: 'untoggleShowProblems'
+            });
+          }
         }
       })
     );
   }
 
   async navigatePathStep(direction: number): Promise<void> {
-    this.postMessage({ t: 'navigatePath', direction });
+    await this.postMessage({ t: 'navigatePath', direction });
+  }
+
+  private isShowingPanel() {
+    return !!this._panel;
   }
 
   // Returns the webview panel, creating it if it doesn't already
@@ -168,6 +174,7 @@ export class InterfaceManager extends DisposableObject {
           ]
         }
       ));
+
       this._panel.onDidDispose(
         () => {
           this._panel = undefined;
@@ -200,14 +207,14 @@ export class InterfaceManager extends DisposableObject {
     sortState: InterpretedResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         'Failed to sort results since evaluation info was unknown.'
       );
       return;
     }
     // Notify the webview that it should expect new results.
     await this.postMessage({ t: 'resultsUpdating' });
-    this._displayedQuery.updateInterpretedSortState(sortState);
+    await this._displayedQuery.updateInterpretedSortState(sortState);
     await this.showResults(this._displayedQuery, WebviewReveal.NotForced, true);
   }
 
@@ -216,7 +223,7 @@ export class InterfaceManager extends DisposableObject {
     sortState: RawResultsSortState | undefined
   ): Promise<void> {
     if (this._displayedQuery === undefined) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         'Failed to sort results since evaluation info was unknown.'
       );
       return;
@@ -236,61 +243,67 @@ export class InterfaceManager extends DisposableObject {
   }
 
   private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
-    switch (msg.t) {
-      case 'viewSourceFile': {
-        await jumpToLocation(msg, this.databaseManager, this.logger);
-        break;
-      }
-      case 'toggleDiagnostics': {
-        if (msg.visible) {
-          const databaseItem = this.databaseManager.findDatabaseItem(
-            Uri.parse(msg.databaseUri)
-          );
-          if (databaseItem !== undefined) {
-            await this.showResultsAsDiagnostics(
-              msg.origResultsPaths,
-              msg.metadata,
-              databaseItem
+    try {
+      switch (msg.t) {
+        case 'viewSourceFile': {
+          await jumpToLocation(msg, this.databaseManager, this.logger);
+          break;
+        }
+        case 'toggleDiagnostics': {
+          if (msg.visible) {
+            const databaseItem = this.databaseManager.findDatabaseItem(
+              Uri.parse(msg.databaseUri)
+            );
+            if (databaseItem !== undefined) {
+              await this.showResultsAsDiagnostics(
+                msg.origResultsPaths,
+                msg.metadata,
+                databaseItem
+              );
+            }
+          } else {
+            // TODO: Only clear diagnostics on the same database.
+            this._diagnosticCollection.clear();
+          }
+          break;
+        }
+        case 'resultViewLoaded':
+          this._panelLoaded = true;
+          this._panelLoadedCallBacks.forEach((cb) => cb());
+          this._panelLoadedCallBacks = [];
+          break;
+        case 'changeSort':
+          await this.changeRawSortState(msg.resultSetName, msg.sortState);
+          break;
+        case 'changeInterpretedSort':
+          await this.changeInterpretedSortState(msg.sortState);
+          break;
+        case 'changePage':
+          if (msg.selectedTable === ALERTS_TABLE_NAME) {
+            await this.showPageOfInterpretedResults(msg.pageNumber);
+          }
+          else {
+            await this.showPageOfRawResults(
+              msg.selectedTable,
+              msg.pageNumber,
+              // When we are in an unsorted state, we guarantee that
+              // sortedResultsInfo doesn't have an entry for the current
+              // result set. Use this to determine whether or not we use
+              // the sorted bqrs file.
+              this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
             );
           }
-        } else {
-          // TODO: Only clear diagnostics on the same database.
-          this._diagnosticCollection.clear();
-        }
-        break;
+          break;
+        case 'openFile':
+          await this.openFile(msg.filePath);
+          break;
+        default:
+          assertNever(msg);
       }
-      case 'resultViewLoaded':
-        this._panelLoaded = true;
-        this._panelLoadedCallBacks.forEach((cb) => cb());
-        this._panelLoadedCallBacks = [];
-        break;
-      case 'changeSort':
-        await this.changeRawSortState(msg.resultSetName, msg.sortState);
-        break;
-      case 'changeInterpretedSort':
-        await this.changeInterpretedSortState(msg.sortState);
-        break;
-      case 'changePage':
-        if (msg.selectedTable === ALERTS_TABLE_NAME) {
-          await this.showPageOfInterpretedResults(msg.pageNumber);
-        }
-        else {
-          await this.showPageOfRawResults(
-            msg.selectedTable,
-            msg.pageNumber,
-            // When we are in an unsorted state, we guarantee that
-            // sortedResultsInfo doesn't have an entry for the current
-            // result set. Use this to determine whether or not we use
-            // the sorted bqrs file.
-            this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
-          );
-        }
-        break;
-      case 'openFile':
-        await this.openFile(msg.filePath);
-        break;
-      default:
-        assertNever(msg);
+    } catch (e) {
+      void showAndLogErrorMessage(e.message, {
+        fullMessage: e.stack
+      });
     }
   }
 
@@ -352,14 +365,13 @@ export class InterfaceManager extends DisposableObject {
       const showButton = 'View Results';
       const queryName = results.queryName;
       const resultPromise = vscode.window.showInformationMessage(
-        `Finished running query ${
-        queryName.length > 0 ? ` "${queryName}"` : ''
+        `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ''
         }.`,
         showButton
       );
       // Address this click asynchronously so we still update the
       // query history immediately.
-      resultPromise.then((result) => {
+      void resultPromise.then((result) => {
         if (result === showButton) {
           panel.reveal();
         }
@@ -394,6 +406,7 @@ export class InterfaceManager extends DisposableObject {
       }
     );
     const resultSet = transformBqrsResultSet(schema, chunk);
+    results.setResultCount(interpretationPage?.numTotalResults || resultSet.schema.rows);
     const parsedResultSets: ParsedResultSets = {
       pageNumber: 0,
       pageSize,
@@ -488,7 +501,12 @@ export class InterfaceManager extends DisposableObject {
     );
 
     const resultSetSchemas = await this.getResultSetSchemas(results, sorted ? selectedTable : '');
-    const resultSetNames = resultSetSchemas.map(schema => schema.name);
+
+    // If there is a specific sorted table selected, a different bqrs file is loaded that doesn't have all the result set names.
+    // Make sure that we load all result set names here.
+    // See https://github.com/github/vscode-codeql/issues/1005
+    const allResultSetSchemas = sorted ? await this.getResultSetSchemas(results, '') : resultSetSchemas;
+    const resultSetNames = allResultSetSchemas.map(schema => schema.name);
 
     const schema = resultSetSchemas.find(
       (resultSet) => resultSet.name == selectedTable
@@ -540,24 +558,26 @@ export class InterfaceManager extends DisposableObject {
     sourceInfo: cli.SourceInfo | undefined,
     sourceLocationPrefix: string,
     sortState: InterpretedResultsSortState | undefined
-  ): Promise<Interpretation> {
+  ): Promise<Interpretation | undefined> {
+    if (!resultsPaths) {
+      void this.logger.log('No results path. Cannot display interpreted results.');
+      return undefined;
+    }
+
     const sarif = await interpretResults(
       this.cliServer,
       metadata,
       resultsPaths,
       sourceInfo
     );
+
     sarif.runs.forEach(run => {
       if (run.results !== undefined) {
         sortInterpretedResults(run.results, sortState);
       }
     });
 
-    const numTotalResults = (() => {
-      if (sarif.runs.length === 0) return 0;
-      if (sarif.runs[0].results === undefined) return 0;
-      return sarif.runs[0].results.length;
-    })();
+    const numTotalResults = sarif.runs[0]?.results?.length || 0;
 
     const interpretation: Interpretation = {
       sarif,
@@ -587,7 +607,7 @@ export class InterfaceManager extends DisposableObject {
       throw new Error('Tried to get interpreted results before interpretation finished');
     }
     if (this._interpretation.sarif.runs.length !== 1) {
-      this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
+      void this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
     }
     const interp = this._interpretation;
     return {
@@ -626,8 +646,8 @@ export class InterfaceManager extends DisposableObject {
       } catch (e) {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
-        this.logger.log(
-          `Exception during results interpretation: ${e.message}. Will show raw results instead.`
+        void showAndLogErrorMessage(
+          `Showing raw results instead of interpreted ones due to an error. ${e.message}`
         );
       }
     }
@@ -659,11 +679,15 @@ export class InterfaceManager extends DisposableObject {
       undefined
     );
 
+    if (!interpretation) {
+      return;
+    }
+
     try {
       await this.showProblemResultsAsDiagnostics(interpretation, database);
     } catch (e) {
       const msg = e instanceof Error ? e.message : e.toString();
-      this.logger.log(
+      void this.logger.log(
         `Exception while computing problem results as diagnostics: ${msg}`
       );
       this._diagnosticCollection.clear();
@@ -677,7 +701,7 @@ export class InterfaceManager extends DisposableObject {
     const { sarif, sourceLocationPrefix } = interpretation;
 
     if (!sarif.runs || !sarif.runs[0].results) {
-      this.logger.log(
+      void this.logger.log(
         'Didn\'t find a run in the sarif results. Error processing sarif?'
       );
       return;
@@ -688,11 +712,11 @@ export class InterfaceManager extends DisposableObject {
     for (const result of sarif.runs[0].results) {
       const message = result.message.text;
       if (message === undefined) {
-        this.logger.log('Sarif had result without plaintext message');
+        void this.logger.log('Sarif had result without plaintext message');
         continue;
       }
       if (!result.locations) {
-        this.logger.log('Sarif had result without location');
+        void this.logger.log('Sarif had result without location');
         continue;
       }
 
@@ -705,7 +729,7 @@ export class InterfaceManager extends DisposableObject {
       }
       const resultLocation = tryResolveLocation(sarifLoc, databaseItem);
       if (!resultLocation) {
-        this.logger.log('Sarif location was not resolvable ' + sarifLoc);
+        void this.logger.log('Sarif location was not resolvable ' + sarifLoc);
         continue;
       }
       const parsedMessage = parseSarifPlainTextMessage(message);

extensions/ql-vscode/src/logging.ts

@@ -1,5 +1,5 @@
 import { window as Window, OutputChannel, Progress, Disposable } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -28,9 +28,16 @@ export interface Logger {
   removeAdditionalLogLocation(location: string | undefined): void;
 
   /**
-   * The base location location where all side log files are stored.
+   * The base location where all side log files are stored.
    */
   getBaseLocation(): string | undefined;
+
+  /**
+   * Sets the location where logs are stored.
+   * @param storagePath The path where logs are stored.
+   * @param isCustomLogDirectory Whether the logs are stored in a custom, user-specified directory.
+   */
+  setLogStoragePath(storagePath: string, isCustomLogDirectory: boolean): Promise<void>;
 }
 
 export type ProgressReporter = Progress<{ message: string }>;
@@ -40,18 +47,24 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   public readonly outputChannel: OutputChannel;
   private readonly additionalLocations = new Map<string, AdditionalLogLocation>();
   private additionalLogLocationPath: string | undefined;
+  isCustomLogDirectory: boolean;
 
   constructor(private title: string) {
     super();
     this.outputChannel = Window.createOutputChannel(title);
     this.push(this.outputChannel);
+    this.isCustomLogDirectory = false;
   }
 
-  init(storagePath: string): void {
+  async setLogStoragePath(storagePath: string, isCustomLogDirectory: boolean): Promise<void> {
     this.additionalLogLocationPath = path.join(storagePath, this.title);
 
-    // clear out any old state from previous runs
-    fs.remove(this.additionalLogLocationPath);
+    this.isCustomLogDirectory = isCustomLogDirectory;
+
+    if (!this.isCustomLogDirectory) {
+      // clear out any old state from previous runs
+      await fs.remove(this.additionalLogLocationPath);
+    }
   }
 
   /**
@@ -80,7 +93,7 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
         this.outputChannel.appendLine(separator);
         this.outputChannel.appendLine(msg);
         this.outputChannel.appendLine(separator);
-        additional = new AdditionalLogLocation(logPath);
+        additional = new AdditionalLogLocation(logPath, !this.isCustomLogDirectory);
         this.additionalLocations.set(logPath, additional);
         this.track(additional);
       }
@@ -112,7 +125,7 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
 }
 
 class AdditionalLogLocation extends Disposable {
-  constructor(private location: string) {
+  constructor(private location: string, private shouldDeleteLogs: boolean) {
     super(() => { /**/ });
   }
 
@@ -128,7 +141,9 @@ class AdditionalLogLocation extends Disposable {
   }
 
   async dispose(): Promise<void> {
-    await fs.remove(this.location);
+    if (this.shouldDeleteLogs) {
+      await fs.remove(this.location);
+    }
   }
 }
 

extensions/ql-vscode/src/pure/bqrs-utils.ts

@@ -83,8 +83,7 @@ export function isLineColumnLoc(loc: UrlValue): loc is LineColumnLocation {
     && 'startLine' in loc
     && 'startColumn' in loc
     && 'endLine' in loc
-    && 'endColumn' in loc
-    && loc.endColumn > 0;
+    && 'endColumn' in loc;
 }
 
 export function isWholeFileLoc(loc: UrlValue): loc is WholeFileLocation {

extensions/ql-vscode/src/pure/disposable-object.ts

@@ -1,4 +1,11 @@
-import { Disposable } from 'vscode';
+
+// Avoid explicitly referencing Disposable type in vscode.
+// This file cannot have dependencies on the vscode API.
+interface Disposable {
+  dispose(): any;
+}
+
+export type DisposeHandler = (disposable: Disposable) => void;
 
 /**
  * Base class to make it easier to implement a `Disposable` that owns other disposable object.
@@ -40,21 +47,39 @@ export abstract class DisposableObject implements Disposable {
    * @param obj The object to stop tracking.
    */
   protected disposeAndStopTracking(obj: Disposable): void {
-    if (obj !== undefined) {
-      this.tracked!.delete(obj);
+    if (obj && this.tracked) {
+      this.tracked.delete(obj);
       obj.dispose();
     }
   }
 
-  public dispose() {
+  /**
+   * Dispose this object and all contained objects
+   *
+   * @param disposeHandler An optional dispose handler that gets
+   *      passed each element to dispose. The dispose handler
+   *      can choose how (and if) to dispose the object. The
+   *      primary usage is for tests that should not dispose
+   *      all items of a disposable.
+   */
+  public dispose(disposeHandler?: DisposeHandler) {
     if (this.tracked !== undefined) {
       for (const trackedObject of this.tracked.values()) {
-        trackedObject.dispose();
+        if (disposeHandler) {
+          disposeHandler(trackedObject);
+        } else {
+          trackedObject.dispose();
+        }
       }
       this.tracked = undefined;
     }
     while (this.disposables.length > 0) {
-      this.disposables.pop()!.dispose();
+      const disposable = this.disposables.pop()!;
+      if (disposeHandler) {
+        disposeHandler(disposable);
+      } else {
+        disposable.dispose();
+      }
     }
   }
 }

extensions/ql-vscode/src/pure/messages.ts

@@ -262,7 +262,7 @@ export interface CompilationTarget {
   /**
    * Compile as a normal query
    */
-  query?: {};
+  query?: Record<string, never>;
   /**
    * Compile as a quick evaluation
    */
@@ -826,7 +826,7 @@ export interface ResultSet {
 /**
  * The type returned when the evaluation is complete
  */
-export type EvaluationComplete = {};
+export type EvaluationComplete = Record<string, never>;
 
 /**
  * The result of a single query

extensions/ql-vscode/src/qltest-discovery.ts

@@ -167,10 +167,12 @@ export class QLTestDiscovery extends Discovery<QLTestDiscoveryResults> {
   protected update(results: QLTestDiscoveryResults): void {
     this._testDirectory = results.testDirectory;
 
-    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
     this.watcher.clear();
+    // Watch for changes to any `.ql` or `.qlref` file in any of the QL packs that contain tests.
     this.watcher.addWatch(new RelativePattern(results.watchPath, '**/*.{ql,qlref}'));
-    this._onDidChangeTests.fire();
+    // need to explicitly watch for changes to directories themselves.
+    this.watcher.addWatch(new RelativePattern(results.watchPath, '**/'));
+    this._onDidChangeTests.fire(undefined);
   }
 
   /**

extensions/ql-vscode/src/query-history.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as vscode from 'vscode';
-import { window as Window } from 'vscode';
+import { window as Window, env } from 'vscode';
 import { CompletedQuery } from './query-results';
 import { QueryHistoryConfig } from './config';
 import { QueryWithResults } from './run-queries';
@@ -13,8 +13,9 @@ import {
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
 import { QueryServerClient } from './queryserver-client';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { commandRunner } from './commandRunner';
+import { assertNever } from './pure/helpers-pure';
 
 /**
  * query-history.ts
@@ -58,10 +59,21 @@ const SHOW_QUERY_TEXT_QUICK_EVAL_MSG = `\
  */
 const FAILED_QUERY_HISTORY_ITEM_ICON = 'media/red-x.svg';
 
+enum SortOrder {
+  NameAsc = 'NameAsc',
+  NameDesc = 'NameDesc',
+  DateAsc = 'DateAsc',
+  DateDesc = 'DateDesc',
+  CountAsc = 'CountAsc',
+  CountDesc = 'CountDesc',
+}
+
 /**
  * Tree data provider for the query history view.
  */
 export class HistoryTreeDataProvider extends DisposableObject {
+  private _sortOrder = SortOrder.DateAsc;
+
   private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
 
   readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
@@ -111,7 +123,24 @@ export class HistoryTreeDataProvider extends DisposableObject {
   getChildren(
     element?: CompletedQuery
   ): vscode.ProviderResult<CompletedQuery[]> {
-    return element ? [] : this.history;
+    return element ? [] : this.history.sort((q1, q2) => {
+      switch (this.sortOrder) {
+        case SortOrder.NameAsc:
+          return q1.toString().localeCompare(q2.toString(), env.language);
+        case SortOrder.NameDesc:
+          return q2.toString().localeCompare(q1.toString(), env.language);
+        case SortOrder.DateAsc:
+          return q1.date.getTime() - q2.date.getTime();
+        case SortOrder.DateDesc:
+          return q2.date.getTime() - q1.date.getTime();
+        case SortOrder.CountAsc:
+          return q1.resultCount - q2.resultCount;
+        case SortOrder.CountDesc:
+          return q2.resultCount - q1.resultCount;
+        default:
+          assertNever(this.sortOrder);
+      }
+    });
   }
 
   getParent(_element: CompletedQuery): vscode.ProviderResult<CompletedQuery> {
@@ -157,6 +186,15 @@ export class HistoryTreeDataProvider extends DisposableObject {
   find(queryId: number): CompletedQuery | undefined {
     return this.allHistory.find((query) => query.query.queryID === queryId);
   }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire(undefined);
+  }
 }
 
 /**
@@ -211,7 +249,7 @@ export class QueryHistoryManager extends DisposableObject {
       })
     );
 
-    logger.log('Registering query history panel commands.');
+    void logger.log('Registering query history panel commands.');
     this.push(
       commandRunner(
         'codeQLQueryHistory.openQuery',
@@ -224,6 +262,24 @@ export class QueryHistoryManager extends DisposableObject {
         this.handleRemoveHistoryItem.bind(this)
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByName',
+        this.handleSortByName.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByDate',
+        this.handleSortByDate.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.sortByCount',
+        this.handleSortByCount.bind(this)
+      )
+    );
     this.push(
       commandRunner(
         'codeQLQueryHistory.setLabel',
@@ -250,8 +306,20 @@ export class QueryHistoryManager extends DisposableObject {
     );
     this.push(
       commandRunner(
-        'codeQLQueryHistory.viewSarif',
-        this.handleViewSarif.bind(this)
+        'codeQLQueryHistory.viewCsvResults',
+        this.handleViewCsvResults.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.viewCsvAlerts',
+        this.handleViewCsvAlerts.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.viewSarifAlerts',
+        this.handleViewSarifAlerts.bind(this)
       )
     );
     this.push(
@@ -340,11 +408,35 @@ export class QueryHistoryManager extends DisposableObject {
     });
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
-      this.treeView.reveal(current);
+      await this.treeView.reveal(current);
       await this.invokeCallbackOn(current);
     }
   }
 
+  async handleSortByName() {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  }
+
+  async handleSortByDate() {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAsc;
+    }
+  }
+
+  async handleSortByCount() {
+    if (this.treeDataProvider.sortOrder === SortOrder.CountAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.CountDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.CountAsc;
+    }
+  }
+
   async handleSetLabel(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
@@ -362,7 +454,12 @@ export class QueryHistoryManager extends DisposableObject {
     if (response !== undefined) {
       // Interpret empty string response as 'go back to using default'
       singleItem.options.label = response === '' ? undefined : response;
-      this.treeDataProvider.refresh(singleItem);
+      if (this.treeDataProvider.sortOrder === SortOrder.NameAsc ||
+        this.treeDataProvider.sortOrder === SortOrder.NameDesc) {
+        this.treeDataProvider.refresh();
+      } else {
+        this.treeDataProvider.refresh(singleItem);
+      }
     }
   }
 
@@ -379,10 +476,10 @@ export class QueryHistoryManager extends DisposableObject {
       const to = await this.findOtherQueryToCompare(from, multiSelect);
 
       if (from && to) {
-        this.doCompareCallback(from, to);
+        await this.doCompareCallback(from, to);
       }
     } catch (e) {
-      showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(e.message);
     }
   }
 
@@ -408,13 +505,13 @@ export class QueryHistoryManager extends DisposableObject {
     if (
       prevItemClick !== undefined &&
       now.valueOf() - prevItemClick.time.valueOf() < DOUBLE_CLICK_TIME &&
-      singleItem == prevItemClick.item
+      finalSingleItem == prevItemClick.item
     ) {
       // show original query file on double click
-      await this.handleOpenQuery(singleItem, [singleItem]);
+      await this.handleOpenQuery(finalSingleItem, [finalSingleItem]);
     } else {
       // show results on single click
-      await this.invokeCallbackOn(singleItem);
+      await this.invokeCallbackOn(finalSingleItem);
     }
   }
 
@@ -429,7 +526,7 @@ export class QueryHistoryManager extends DisposableObject {
     if (singleItem.logFileLocation) {
       await this.tryOpenExternalFile(singleItem.logFileLocation);
     } else {
-      showAndLogWarningMessage('No log file available');
+      void showAndLogWarningMessage('No log file available');
     }
   }
 
@@ -459,7 +556,7 @@ export class QueryHistoryManager extends DisposableObject {
     await vscode.window.showTextDocument(doc, { preview: false });
   }
 
-  async handleViewSarif(
+  async handleViewSarifAlerts(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
   ) {
@@ -474,12 +571,43 @@ export class QueryHistoryManager extends DisposableObject {
       );
     } else {
       const label = singleItem.getLabel();
-      showAndLogInformationMessage(
+      void showAndLogInformationMessage(
         `Query ${label} has no interpreted results.`
       );
     }
   }
 
+  async handleViewCsvResults(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+    if (await singleItem.query.hasCsv()) {
+      void this.tryOpenExternalFile(singleItem.query.csvPath);
+      return;
+    }
+    await singleItem.query.exportCsvResults(this.qs, singleItem.query.csvPath, () => {
+      void this.tryOpenExternalFile(
+        singleItem.query.csvPath
+      );
+    });
+  }
+
+  async handleViewCsvAlerts(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    await this.tryOpenExternalFile(
+      await singleItem.query.ensureCsvProduced(this.qs)
+    );
+  }
+
   async handleViewDil(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[],
@@ -511,11 +639,14 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
-  addQuery(info: QueryWithResults): CompletedQuery {
+  buildCompletedQuery(info: QueryWithResults): CompletedQuery {
     const item = new CompletedQuery(info, this.queryHistoryConfigListener);
+    return item;
+  }
+
+  addCompletedQuery(item: CompletedQuery) {
     this.treeDataProvider.pushQuery(item);
     this.updateTreeViewSelectionIfVisible();
-    return item;
   }
 
   find(queryId: number): CompletedQuery | undefined {
@@ -537,7 +668,7 @@ export class QueryHistoryManager extends DisposableObject {
         // We must fire the onDidChangeTreeData event to ensure the current element can be selected
         // using `reveal` if the tree view was not visible when the current element was added.
         this.treeDataProvider.refresh();
-        this.treeView.reveal(current);
+        void this.treeView.reveal(current);
       }
     }
   }
@@ -564,13 +695,13 @@ the file in the file explorer and dragging it into the workspace.`
           try {
             await vscode.commands.executeCommand('revealFileInOS', uri);
           } catch (e) {
-            showAndLogErrorMessage(e.message);
+            void showAndLogErrorMessage(e.message);
           }
         }
       } else {
-        showAndLogErrorMessage(`Could not open file ${fileLocation}`);
-        logger.log(e.message);
-        logger.log(e.stack);
+        void showAndLogErrorMessage(`Could not open file ${fileLocation}`);
+        void logger.log(e.message);
+        void logger.log(e.stack);
       }
     }
   }
@@ -622,7 +753,7 @@ the file in the file explorer and dragging it into the workspace.`
 
   private assertSingleQuery(multiSelect: CompletedQuery[] = [], message = 'Please select a single query.') {
     if (multiSelect.length > 1) {
-      showAndLogErrorMessage(
+      void showAndLogErrorMessage(
         message
       );
       return false;

extensions/ql-vscode/src/query-results.ts

@@ -11,12 +11,14 @@ import { QueryHistoryConfig } from './config';
 import { QueryHistoryItemOptions } from './query-history';
 
 export class CompletedQuery implements QueryWithResults {
+  readonly date: Date;
   readonly time: string;
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
   readonly database: DatabaseInfo;
   readonly logFileLocation?: string;
   options: QueryHistoryItemOptions;
+  resultCount: number;
   dispose: () => void;
 
   /**
@@ -44,8 +46,14 @@ export class CompletedQuery implements QueryWithResults {
     this.options = evaluation.options;
     this.dispose = evaluation.dispose;
 
-    this.time = new Date().toLocaleString(env.language);
+    this.date = new Date();
+    this.time = this.date.toLocaleString(env.language);
     this.sortedResultsInfo = new Map();
+    this.resultCount = 0;
+  }
+
+  setResultCount(value: number) {
+    this.resultCount = value;
   }
 
   get databaseName(): string {
@@ -54,6 +62,9 @@ export class CompletedQuery implements QueryWithResults {
   get queryName(): string {
     return getQueryName(this.query);
   }
+  get queryFileName(): string {
+    return getQueryFileName(this.query);
+  }
 
   get statusString(): string {
     switch (this.result.resultType) {
@@ -80,12 +91,14 @@ export class CompletedQuery implements QueryWithResults {
   }
 
   interpolate(template: string): string {
-    const { databaseName, queryName, time, statusString } = this;
+    const { databaseName, queryName, time, resultCount, statusString, queryFileName } = this;
     const replacements: { [k: string]: string } = {
       t: time,
       q: queryName,
       d: databaseName,
+      r: resultCount.toString(),
       s: statusString,
+      f: queryFileName,
       '%': '%',
     };
     return template.replace(/%(.)/g, (match, key) => {
@@ -143,17 +156,28 @@ export class CompletedQuery implements QueryWithResults {
  * Uses metadata if it exists, and defaults to the query file name.
  */
 export function getQueryName(query: QueryInfo) {
+  if (query.quickEvalPosition !== undefined) {
+    return 'Quick evaluation of ' + getQueryFileName(query);
+  } else if (query.metadata?.name) {
+    return query.metadata.name;
+  } else {
+    return getQueryFileName(query);
+  }
+}
+
+/**
+ * Gets the file name for an evaluated query.
+ * Defaults to the query file name and may contain position information for quick eval queries.
+ */
+export function getQueryFileName(query: QueryInfo) {
   // Queries run through quick evaluation are not usually the entire query file.
   // Label them differently and include the line numbers.
   if (query.quickEvalPosition !== undefined) {
     const { line, endLine, fileName } = query.quickEvalPosition;
     const lineInfo = line === endLine ? `${line}` : `${line}-${endLine}`;
-    return `Quick evaluation of ${path.basename(fileName)}:${lineInfo}`;
-  } else if (query.metadata?.name) {
-    return query.metadata.name;
-  } else {
-    return path.basename(query.program.queryPath);
+    return `${path.basename(fileName)}:${lineInfo}`;
   }
+  return path.basename(query.program.queryPath);
 }
 
 
@@ -170,17 +194,20 @@ export async function interpretResults(
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }
+  return await server.interpretBqrs(ensureMetadataIsComplete(metadata), resultsPath, interpretedResultsPath, sourceInfo);
+}
+
+export function ensureMetadataIsComplete(metadata: QueryMetadata | undefined) {
   if (metadata === undefined) {
     throw new Error('Can\'t interpret results without query metadata');
   }
-  let { kind, id, scored } = metadata;
-  if (kind === undefined) {
+  if (metadata.kind === undefined) {
     throw new Error('Can\'t interpret results without query metadata including kind');
   }
-  if (id === undefined) {
+  if (metadata.id === undefined) {
     // Interpretation per se doesn't really require an id, but the
     // SARIF format does, so in the absence of one, we use a dummy id.
-    id = 'dummy-id';
+    metadata.id = 'dummy-id';
   }
-  return await server.interpretBqrs({ kind, id, scored }, resultsPath, interpretedResultsPath, sourceInfo);
+  return metadata;
 }

extensions/ql-vscode/src/queryserver-client.ts

@@ -1,6 +1,6 @@
 import * as cp from 'child_process';
 import * as path from 'path';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { Disposable, CancellationToken, commands } from 'vscode';
 import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
 import * as cli from './cli';
@@ -8,11 +8,13 @@ import { QueryServerConfig } from './config';
 import { Logger, ProgressReporter } from './logging';
 import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
 import * as messages from './pure/messages';
-import { SemVer } from 'semver';
 import { ProgressCallback, ProgressTask } from './commandRunner';
+import * as fs from 'fs-extra';
+import * as helpers from './helpers';
 
 type ServerOpts = {
   logger: Logger;
+  contextStoragePath: string;
 }
 
 /** A running query server process and its associated message connection. */
@@ -28,7 +30,7 @@ class ServerProcess implements Disposable {
   }
 
   dispose(): void {
-    this.logger.log('Stopping query server...');
+    void this.logger.log('Stopping query server...');
     this.connection.dispose();
     this.child.stdin!.end();
     this.child.stderr!.destroy();
@@ -36,7 +38,7 @@ class ServerProcess implements Disposable {
 
     // On Windows, we usually have to terminate the process before closing its stdout.
     this.child.stdout!.destroy();
-    this.logger.log('Stopped query server.');
+    void this.logger.log('Stopped query server.');
   }
 }
 
@@ -50,11 +52,6 @@ type WithProgressReporting = (task: (progress: ProgressReporter, token: Cancella
  */
 export class QueryServerClient extends DisposableObject {
 
-  /**
-   * Query Server version where database registration was introduced
-   */
-  private static VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
-
   serverProcess?: ServerProcess;
   evaluationResultCallbacks: { [key: number]: (res: EvaluationResult) => void };
   progressCallbacks: { [key: number]: ((res: ProgressMessage) => void) | undefined };
@@ -92,6 +89,26 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
+  async initLogger() {
+    let storagePath = this.opts.contextStoragePath;
+    let isCustomLogDirectory = false;
+    if (this.config.customLogDirectory) {
+      try {
+        if (!(await fs.pathExists(this.config.customLogDirectory))) {
+          await fs.mkdir(this.config.customLogDirectory);
+        }
+        void this.logger.log(`Saving query server logs to user-specified directory: ${this.config.customLogDirectory}.`);
+        storagePath = this.config.customLogDirectory;
+        isCustomLogDirectory = true;
+      } catch (e) {
+        void helpers.showAndLogErrorMessage(`${this.config.customLogDirectory} is not a valid directory. Logs will be stored in a temporary workspace directory instead.`);
+      }
+    }
+
+    await this.logger.setLogStoragePath(storagePath, isCustomLogDirectory);
+
+  }
+
   get logger(): Logger {
     return this.opts.logger;
   }
@@ -101,7 +118,7 @@ export class QueryServerClient extends DisposableObject {
     if (this.serverProcess !== undefined) {
       this.disposeAndStopTracking(this.serverProcess);
     } else {
-      this.logger.log('No server process to be stopped.');
+      void this.logger.log('No server process to be stopped.');
     }
   }
 
@@ -133,10 +150,20 @@ export class QueryServerClient extends DisposableObject {
 
   /** Starts a new query server process, sending progress messages to the given reporter. */
   private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
+    await this.initLogger();
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
 
-    if (await this.supportsDatabaseRegistration()) {
+    if (this.config.saveCache) {
+      args.push('--save-cache');
+    }
+
+    if (this.config.cacheSize > 0) {
+      args.push('--max-disk-cache');
+      args.push(this.config.cacheSize.toString());
+    }
+
+    if (await this.cliServer.cliConstraints.supportsDatabaseRegistration()) {
       args.push('--require-db-registration');
     }
 
@@ -165,9 +192,8 @@ export class QueryServerClient extends DisposableObject {
     const connection = createMessageConnection(child.stdout, child.stdin);
     connection.onRequest(completeQuery, res => {
       if (!(res.runId in this.evaluationResultCallbacks)) {
-        this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
-      }
-      else {
+        void this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
+      } else {
         const baseLocation = this.logger.getBaseLocation();
         if (baseLocation && this.activeQueryName) {
           res.logFileLocation = path.join(baseLocation, this.activeQueryName);
@@ -182,7 +208,7 @@ export class QueryServerClient extends DisposableObject {
         callback(res);
       }
     });
-    this.serverProcess = new ServerProcess(child, connection, this.opts.logger);
+    this.serverProcess = new ServerProcess(child, connection, this.logger);
     // Ensure the server process is disposed together with this client.
     this.track(this.serverProcess);
     connection.listen();
@@ -193,10 +219,6 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
-  async supportsDatabaseRegistration() {
-    return (await this.cliServer.getVersion()).compare(QueryServerClient.VERSION_WITH_DB_REGISTRATION) >= 0;
-  }
-
   registerCallback(callback: (res: EvaluationResult) => void): number {
     const id = this.nextCallback++;
     this.evaluationResultCallbacks[id] = callback;

extensions/ql-vscode/src/quick-query.ts

@@ -1,16 +1,20 @@
 import * as fs from 'fs-extra';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
-import { CancellationToken, ExtensionContext, window as Window, workspace, Uri } from 'vscode';
+import {
+  CancellationToken,
+  ExtensionContext,
+  window as Window,
+  workspace,
+  Uri
+} from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import { CodeQLCliServer } from './cli';
 import { DatabaseUI } from './databases-ui';
-import { logger } from './logging';
 import {
   getInitialQueryContents,
   getPrimaryDbscheme,
   getQlPackForDbscheme,
-  showAndLogErrorMessage,
   showBinaryChoiceDialog,
 } from './helpers';
 import {
@@ -21,23 +25,35 @@ import {
 const QUICK_QUERIES_DIR_NAME = 'quick-queries';
 const QUICK_QUERY_QUERY_NAME = 'quick-query.ql';
 const QUICK_QUERY_WORKSPACE_FOLDER_NAME = 'Quick Queries';
+const QLPACK_FILE_HEADER = '# This is an automatically generated file.\n\n';
 
 export function isQuickQueryPath(queryPath: string): boolean {
   return path.basename(queryPath) === QUICK_QUERY_QUERY_NAME;
 }
 
-function getQuickQueriesDir(ctx: ExtensionContext): string {
+async function getQuickQueriesDir(ctx: ExtensionContext): Promise<string> {
   const storagePath = ctx.storagePath;
   if (storagePath === undefined) {
     throw new Error('Workspace storage path is undefined');
   }
   const queriesPath = path.join(storagePath, QUICK_QUERIES_DIR_NAME);
-  fs.ensureDir(queriesPath, { mode: 0o700 });
+  await fs.ensureDir(queriesPath, { mode: 0o700 });
   return queriesPath;
 }
 
+function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
+  workspace.updateWorkspaceFolders(
+    index,
+    len,
+    { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
+  );
+}
 
-
+function findExistingQuickQueryEditor() {
+  return Window.visibleTextEditors.find(editor =>
+    path.basename(editor.document.uri.fsPath) === QUICK_QUERY_QUERY_NAME
+  );
+}
 
 /**
  * Show a buffer the user can enter a simple query into.
@@ -50,26 +66,18 @@ export async function displayQuickQuery(
   token: CancellationToken
 ) {
 
-  function updateQuickQueryDir(queriesDir: string, index: number, len: number) {
-    workspace.updateWorkspaceFolders(
-      index,
-      len,
-      { uri: Uri.file(queriesDir), name: QUICK_QUERY_WORKSPACE_FOLDER_NAME }
-    );
-  }
-
   try {
-    const workspaceFolders = workspace.workspaceFolders || [];
-    const queriesDir = await getQuickQueriesDir(ctx);
-
     // If there is already a quick query open, don't clobber it, just
     // show it.
-    const existing = workspace.textDocuments.find(doc => path.basename(doc.uri.fsPath) === QUICK_QUERY_QUERY_NAME);
-    if (existing !== undefined) {
-      Window.showTextDocument(existing);
+    const existing = findExistingQuickQueryEditor();
+    if (existing) {
+      await Window.showTextDocument(existing.document);
       return;
     }
 
+    const workspaceFolders = workspace.workspaceFolders || [];
+    const queriesDir = await getQuickQueriesDir(ctx);
+
     // We need to have a multi-root workspace to make quick query work
     // at all. Changing the workspace from single-root to multi-root
     // causes a restart of the whole extension host environment, so we
@@ -88,10 +96,11 @@ export async function displayQuickQuery(
     }
 
     const index = workspaceFolders.findIndex(folder => folder.name === QUICK_QUERY_WORKSPACE_FOLDER_NAME);
-    if (index === -1)
+    if (index === -1) {
       updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
-    else
+    } else {
       updateQuickQueryDir(queriesDir, index, 1);
+    }
 
     // We're going to infer which qlpack to use from the current database
     const dbItem = await databaseUI.getDatabaseItem(progress, token);
@@ -101,32 +110,39 @@ export async function displayQuickQuery(
 
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
     const dbscheme = await getPrimaryDbscheme(datasetFolder);
-    const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
-
-    const quickQueryQlpackYaml: any = {
-      name: 'quick-query',
-      version: '1.0.0',
-      libraryPathDependencies: [qlpack]
-    };
-
-    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const qlpack = (await getQlPackForDbscheme(cliServer, dbscheme)).dbschemePack;
     const qlPackFile = path.join(queriesDir, 'qlpack.yml');
-    await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
-    await fs.writeFile(qlPackFile, yaml.safeDump(quickQueryQlpackYaml), 'utf8');
-    Window.showTextDocument(await workspace.openTextDocument(qlFile));
-  }
+    const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
+    const shouldRewrite = await checkShouldRewrite(qlPackFile, qlpack);
 
-  // TODO: clean up error handling for top-level commands like this
-  catch (e) {
-    if (e instanceof UserCancellationException) {
-      logger.log(e.message);
+    // Only rewrite the qlpack file if the database has changed
+    if (shouldRewrite) {
+      const quickQueryQlpackYaml: any = {
+        name: 'vscode/quick-query',
+        version: '1.0.0',
+        libraryPathDependencies: [qlpack]
+      };
+      await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.safeDump(quickQueryQlpackYaml), 'utf8');
     }
-    else if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
-      logger.log(e.message);
+
+    if (shouldRewrite || !(await fs.pathExists(qlFile))) {
+      await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
     }
-    else if (e instanceof Error)
-      showAndLogErrorMessage(e.message);
-    else
+
+    await Window.showTextDocument(await workspace.openTextDocument(qlFile));
+  } catch (e) {
+    if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+      throw new UserCancellationException(e.message);
+    } else {
       throw e;
+    }
   }
 }
+
+async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
+  if (!(await fs.pathExists(qlPackFile))) {
+    return true;
+  }
+  const qlPackContents: any = yaml.safeLoad(await fs.readFile(qlPackFile, 'utf8'));
+  return qlPackContents.libraryPathDependencies?.[0] !== newDependency;
+}

extensions/ql-vscode/src/run-queries.ts

@@ -24,6 +24,9 @@ import { QueryHistoryItemOptions } from './query-history';
 import * as qsClient from './queryserver-client';
 import { isQuickQueryPath } from './quick-query';
 import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
+import { ensureMetadataIsComplete } from './query-results';
+import { SELECT_QUERY_NAME } from './contextual/locationFinder';
+import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
 
 /**
  * run-queries.ts
@@ -53,6 +56,7 @@ export class QueryInfo {
 
   readonly compiledQueryPath: string;
   readonly dilPath: string;
+  readonly csvPath: string;
   readonly resultsPaths: ResultsPaths;
   readonly dataset: Uri; // guarantee the existence of a well-defined dataset dir at this point
   readonly queryID: number;
@@ -68,6 +72,7 @@ export class QueryInfo {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
     this.dilPath = path.join(tmpDir.name, `results${this.queryID}.dil`);
+    this.csvPath = path.join(tmpDir.name, `results${this.queryID}.csv`);
     this.resultsPaths = {
       resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`),
       interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`)
@@ -153,7 +158,7 @@ export class QueryInfo {
 
       compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
     } finally {
-      qs.logger.log(' - - - COMPILATION DONE - - - ');
+      void qs.logger.log(' - - - COMPILATION DONE - - - ');
     }
     return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
   }
@@ -164,9 +169,17 @@ export class QueryInfo {
   async canHaveInterpretedResults(): Promise<boolean> {
     const hasMetadataFile = await this.dbItem.hasMetadataFile();
     if (!hasMetadataFile) {
-      logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
+      void logger.log('Cannot produce interpreted results since the database does not have a .dbinfo or codeql-database.yml file.');
     }
-    return hasMetadataFile;
+
+    const hasKind = !!this.metadata?.kind;
+    if (!hasKind) {
+      void logger.log('Cannot produce interpreted results since the query does not have @kind metadata.');
+    }
+
+    const isTable = hasKind && this.metadata?.kind === 'table';
+
+    return hasMetadataFile && hasKind && !isTable;
   }
 
   /**
@@ -183,6 +196,13 @@ export class QueryInfo {
     return fs.pathExists(this.dilPath);
   }
 
+  /**
+   * Holds if this query already has CSV results produced
+   */
+  async hasCsv(): Promise<boolean> {
+    return fs.pathExists(this.csvPath);
+  }
+
   async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
     if (await this.hasDil()) {
       return this.dilPath;
@@ -198,8 +218,50 @@ export class QueryInfo {
     return this.dilPath;
   }
 
+  async exportCsvResults(qs: qsClient.QueryServerClient, csvPath: string, onFinish: () => void): Promise<void> {
+    let stopDecoding = false;
+    const out = fs.createWriteStream(csvPath);
+    out.on('finish', onFinish);
+    out.on('error', () => {
+      if (!stopDecoding) {
+        stopDecoding = true;
+        void showAndLogErrorMessage(`Failed to write CSV results to ${csvPath}`);
+      }
+    });
+    let nextOffset: number | undefined = 0;
+    while (nextOffset !== undefined && !stopDecoding) {
+      const chunk: DecodedBqrsChunk = await qs.cliServer.bqrsDecode(this.resultsPaths.resultsPath, SELECT_QUERY_NAME, {
+        pageSize: 100,
+        offset: nextOffset,
+      });
+      for (const tuple of chunk.tuples)
+        out.write(tuple.join(',') + '\n');
+      nextOffset = chunk.next;
+    }
+    out.end();
+  }
+
+  async ensureCsvProduced(qs: qsClient.QueryServerClient): Promise<string> {
+    if (await this.hasCsv()) {
+      return this.csvPath;
+    }
+
+    let sourceInfo;
+    if (this.dbItem.sourceArchive !== undefined) {
+      sourceInfo = {
+        sourceArchive: this.dbItem.sourceArchive.fsPath,
+        sourceLocationPrefix: await this.dbItem.getSourceLocationPrefix(
+          qs.cliServer
+        ),
+      };
+    }
+
+    await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
+    return this.csvPath;
+  }
 }
 
+
 export interface QueryWithResults {
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;
@@ -294,7 +356,7 @@ async function checkDbschemeCompatibility(
   const searchPath = getOnDiskWorkspaceFolders();
 
   if (query.dbItem.contents !== undefined && query.dbItem.contents.dbSchemeUri !== undefined) {
-    const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath);
+    const { finalDbscheme } = await cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
     const hash = async function(filename: string): Promise<string> {
       return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
     };
@@ -348,7 +410,7 @@ async function compileNonDestructiveUpgrade(
   if (!query.dbItem?.contents?.dbSchemeUri) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
-  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, query.queryDbscheme);
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(query.dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme);
 
   if (!matchesTarget) {
     reportNoUpgradePath(query);
@@ -462,7 +524,7 @@ export async function determineSelectedQuery(selectedResourceUri: Uri | undefine
   // then prompt the user to save it first.
   if (editor !== undefined && editor.document.uri.fsPath === queryPath) {
     if (await promptUserToSaveChanges(editor.document)) {
-      editor.document.save();
+      await editor.document.save();
     }
   }
 
@@ -525,8 +587,8 @@ export async function compileAndRunQueryAgainstDatabase(
   const querySchemaName = path.basename(packConfig.dbscheme);
   const dbSchemaName = path.basename(db.contents.dbSchemeUri.fsPath);
   if (querySchemaName != dbSchemaName) {
-    logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
-    throw new Error(`The query ${path.basename(queryPath)} cannot be run against the selected database: their target languages are different. Please select a different database and try again.`);
+    void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
+    throw new Error(`The query ${path.basename(queryPath)} cannot be run against the selected database (${db.name}): their target languages are different. Please select a different database and try again.`);
   }
 
   const qlProgram: messages.QlProgram = {
@@ -547,7 +609,7 @@ export async function compileAndRunQueryAgainstDatabase(
     metadata = await cliServer.resolveMetadata(qlProgram.queryPath);
   } catch (e) {
     // Ignore errors and provide no metadata.
-    logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
+    void logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
   }
 
   const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
@@ -575,8 +637,8 @@ export async function compileAndRunQueryAgainstDatabase(
       const result = await query.run(qs, upgradeQlo, progress, token);
       if (result.resultType !== messages.QueryResultType.SUCCESS) {
         const message = result.message || 'Failed to run query';
-        logger.log(message);
-        showAndLogErrorMessage(message);
+        void logger.log(message);
+        void showAndLogErrorMessage(message);
       }
       return {
         query,
@@ -596,7 +658,7 @@ export async function compileAndRunQueryAgainstDatabase(
       // so we include a general description of the problem,
       // and direct the user to the output window for the detailed compilation messages.
       // However we don't show quick eval errors there so we need to display them anyway.
-      qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
+      void qs.logger.log(`Failed to compile query ${query.program.queryPath} against database scheme ${query.program.dbschemePath}:`);
 
       const formattedMessages: string[] = [];
 
@@ -604,21 +666,24 @@ export async function compileAndRunQueryAgainstDatabase(
         const message = error.message || '[no error message available]';
         const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
         formattedMessages.push(formatted);
-        qs.logger.log(formatted);
+        void qs.logger.log(formatted);
       }
-      if (quickEval && formattedMessages.length <= 3) {
-        showAndLogErrorMessage('Quick evaluation compilation failed: \n' + formattedMessages.join('\n'));
+      if (quickEval && formattedMessages.length <= 2) {
+        // If there are more than 2 error messages, they will not be displayed well in a popup
+        // and will be trimmed by the function displaying the error popup. Accordingly, we only
+        // try to show the errors if there are 2 or less, otherwise we direct the user to the log.
+        void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
       } else {
-        showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
+        void showAndLogErrorMessage((quickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
       }
 
       return createSyntheticResult(query, db, historyItemOptions, 'Query had compilation errors', messages.QueryResultType.OTHER_ERROR);
     }
   } finally {
     try {
-      upgradeDir.cleanup();
+      await upgradeDir.cleanup();
     } catch (e) {
-      qs.logger.log(`Could not clean up the upgrades dir. Reason: ${e.message || e}`);
+      void qs.logger.log(`Could not clean up the upgrades dir. Reason: ${e.message || e}`);
     }
   }
 }

extensions/ql-vscode/src/run-remote-query.ts

@@ -0,0 +1,415 @@
+import { CancellationToken, QuickPickItem, Uri, window } from 'vscode';
+import * as path from 'path';
+import * as yaml from 'js-yaml';
+import * as fs from 'fs-extra';
+import * as tmp from 'tmp-promise';
+import { askForLanguage, findLanguage, getOnDiskWorkspaceFolders, showAndLogErrorMessage, showAndLogInformationMessage, showInformationMessageWithAction } from './helpers';
+import { Credentials } from './authentication';
+import * as cli from './cli';
+import { logger } from './logging';
+import { getRemoteControllerRepo, getRemoteRepositoryLists, setRemoteControllerRepo } from './config';
+import { tmpDir } from './run-queries';
+import { ProgressCallback, UserCancellationException } from './commandRunner';
+import { OctokitResponse } from '@octokit/types/dist-types';
+
+interface Config {
+  repositories: string[];
+  ref?: string;
+  language?: string;
+}
+
+interface RepoListQuickPickItem extends QuickPickItem {
+  repoList: string[];
+}
+
+interface QueriesResponse {
+  workflow_run_id: number
+}
+
+/**
+ * This regex matches strings of the form `owner/repo` where:
+ * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character
+ * - `repo` is made up of alphanumeric characters, hyphens, or underscores
+ */
+const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/;
+
+/**
+ * Gets the repositories to run the query against.
+ */
+export async function getRepositories(): Promise<string[] | undefined> {
+  const repoLists = getRemoteRepositoryLists();
+  if (repoLists && Object.keys(repoLists).length) {
+    const quickPickItems = Object.entries(repoLists).map<RepoListQuickPickItem>(([key, value]) => (
+      {
+        label: key,       // the name of the repository list
+        repoList: value,  // the actual array of repositories
+      }
+    ));
+    const quickpick = await window.showQuickPick<RepoListQuickPickItem>(
+      quickPickItems,
+      {
+        placeHolder: 'Select a repository list. You can define repository lists in the `codeQL.remoteQueries.repositoryLists` setting.',
+        ignoreFocusOut: true,
+      });
+    if (quickpick?.repoList.length) {
+      void logger.log(`Selected repositories: ${quickpick.repoList.join(', ')}`);
+      return quickpick.repoList;
+    } else {
+      void showAndLogErrorMessage('No repositories selected.');
+      return;
+    }
+  } else {
+    void logger.log('No repository lists defined. Displaying text input box.');
+    const remoteRepo = await window.showInputBox({
+      title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
+      placeHolder: '<owner>/<repo>',
+      prompt: 'Tip: you can save frequently used repositories in the `codeQL.remoteQueries.repositoryLists` setting',
+      ignoreFocusOut: true,
+    });
+    if (!remoteRepo) {
+      void showAndLogErrorMessage('No repositories entered.');
+      return;
+    } else if (!REPO_REGEX.test(remoteRepo)) { // Check if user entered invalid input
+      void showAndLogErrorMessage('Invalid repository format. Must be in the format <owner>/<repo> (e.g. github/codeql)');
+      return;
+    }
+    void logger.log(`Entered repository: ${remoteRepo}`);
+    return [remoteRepo];
+  }
+}
+
+/**
+ * Two possibilities:
+ * 1. There is no qlpack.yml in this directory. Assume this is a lone query and generate a synthetic qlpack for it.
+ * 2. There is a qlpack.yml in this directory. Assume this is a query pack and use the yml to pack the query before uploading it.
+ *
+ * @returns the entire qlpack as a base64 string.
+ */
+async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: string, queryPackDir: string, fallbackLanguage?: string): Promise<{
+  base64Pack: string,
+  language: string
+}> {
+  const originalPackRoot = path.dirname(queryFile);
+  // TODO this assumes that the qlpack.yml is in the same directory as the query file, but in reality,
+  // the file could be in a parent directory.
+  const targetQueryFileName = path.join(queryPackDir, path.basename(queryFile));
+
+  // the server is expecting the query file to be named `query.ql`. Rename it here.
+  const renamedQueryFile = path.join(queryPackDir, 'query.ql');
+
+  let language: string | undefined;
+  if (await fs.pathExists(path.join(originalPackRoot, 'qlpack.yml'))) {
+    // don't include ql files. We only want the queryFile to be copied.
+    const toCopy = await cliServer.packPacklist(originalPackRoot, false);
+
+    // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist.
+    [path.join(originalPackRoot, 'qlpack.lock.yml'), path.join(originalPackRoot, 'codeql-pack.lock.yml'), queryFile]
+      .forEach(absolutePath => {
+        if (absolutePath) {
+          toCopy.push(absolutePath);
+        }
+      });
+
+    let copiedCount = 0;
+    await fs.copy(originalPackRoot, queryPackDir, {
+      filter: (file: string) =>
+        // copy file if it is in the packlist, or it is a parent directory of a file in the packlist
+        !!toCopy.find(f => {
+          // Normalized paths ensure that Windows drive letters are capitalized consistently.
+          const normalizedPath = Uri.file(f).fsPath;
+          const matches = normalizedPath === file || normalizedPath.startsWith(file + path.sep);
+          if (matches) {
+            copiedCount++;
+          }
+          return matches;
+        })
+    });
+
+    // ensure the qlpack.yml has a valid name
+    await ensureQueryPackName(queryPackDir);
+
+    void logger.log(`Copied ${copiedCount} files to ${queryPackDir}`);
+
+    language = await findLanguage(cliServer, Uri.file(targetQueryFileName));
+
+  } else {
+    // open popup to ask for language if not already hardcoded
+    language = fallbackLanguage || await askForLanguage(cliServer);
+
+    // copy only the query file to the query pack directory
+    // and generate a synthetic query pack
+    // TODO this has a limitation that query packs inside of a workspace will not resolve its peer dependencies.
+    // Something to work on later. For now, we will only support query packs that are not in a workspace.
+    void logger.log(`Copying ${queryFile} to ${queryPackDir}`);
+    await fs.copy(queryFile, targetQueryFileName);
+    void logger.log('Generating synthetic query pack');
+    const syntheticQueryPack = {
+      name: 'codeql-remote/query',
+      version: '0.0.0',
+      dependencies: {
+        [`codeql/${language}-all`]: '*',
+      }
+    };
+    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(syntheticQueryPack));
+  }
+  if (!language) {
+    throw new UserCancellationException('Could not determine language.');
+  }
+
+  await fs.rename(targetQueryFileName, renamedQueryFile);
+
+  const bundlePath = await getPackedBundlePath(queryPackDir);
+  void logger.log(`Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`);
+  await cliServer.packInstall(queryPackDir);
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+  await cliServer.packBundle(queryPackDir, workspaceFolders, bundlePath, false);
+  const base64Pack = (await fs.readFile(bundlePath)).toString('base64');
+  return {
+    base64Pack,
+    language
+  };
+}
+
+/**
+ * Ensure that the qlpack.yml has a valid name. For local purposes,
+ * Anonymous packs and names that are not prefixed by a scope (ie `<foo>/`)
+ * are sufficient. But in order to create a pack, the name must be prefixed.
+ *
+ * @param queryPackDir the directory containing the query pack.
+ */
+async function ensureQueryPackName(queryPackDir: string) {
+  const pack = yaml.safeLoad(await fs.readFile(path.join(queryPackDir, 'qlpack.yml'), 'utf8')) as { name: string; };
+  if (!pack.name || !pack.name.includes('/')) {
+    if (!pack.name) {
+      pack.name = 'codeql-remote/query';
+    } else if (!pack.name.includes('/')) {
+      pack.name = `codeql-remote/${pack.name}`;
+    }
+    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(pack));
+  }
+}
+
+async function createRemoteQueriesTempDirectory() {
+  const remoteQueryDir = await tmp.dir({ dir: tmpDir.name, unsafeCleanup: true });
+  const queryPackDir = path.join(remoteQueryDir.path, 'query-pack');
+  await fs.mkdirp(queryPackDir);
+  return { remoteQueryDir, queryPackDir };
+}
+
+async function getPackedBundlePath(queryPackDir: string) {
+  return tmp.tmpName({
+    dir: path.dirname(queryPackDir),
+    postfix: 'generated.tgz',
+    prefix: 'qlpack',
+  });
+}
+
+export async function runRemoteQuery(
+  cliServer: cli.CodeQLCliServer,
+  credentials: Credentials,
+  uri: Uri | undefined,
+  dryRun: boolean,
+  progress: ProgressCallback,
+  token: CancellationToken
+): Promise<void | string> {
+  if (!(await cliServer.cliConstraints.supportsRemoteQueries())) {
+    throw new Error(`Remote queries are not supported by this version of CodeQL. Please upgrade to v${cli.CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
+      } or later.`);
+  }
+
+  const { remoteQueryDir, queryPackDir } = await createRemoteQueriesTempDirectory();
+  try {
+    if (!uri?.fsPath.endsWith('.ql')) {
+      throw new UserCancellationException('Not a CodeQL query file.');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 1,
+      message: 'Determining project list'
+    });
+
+    const queryFile = uri.fsPath;
+    const repositoriesFile = queryFile.substring(0, queryFile.length - '.ql'.length) + '.repositories';
+    let ref: string | undefined;
+    // For the case of single file remote queries, use the language from the config in order to avoid the user having to select it.
+    let fallbackLanguage: string | undefined;
+    let repositories: string[] | undefined;
+
+    progress({
+      maxStep: 5,
+      step: 2,
+      message: 'Determining query target language'
+    });
+
+    // If the user has an explicit `.repositories` file, use that.
+    // Otherwise, prompt user to select repositories from the `codeQL.remoteQueries.repositoryLists` setting.
+    if (await fs.pathExists(repositoriesFile)) {
+      void logger.log(`Found '${repositoriesFile}'. Using information from that file to run ${queryFile}.`);
+
+      const config = yaml.safeLoad(await fs.readFile(repositoriesFile, 'utf8')) as Config;
+
+      ref = config.ref || 'main';
+      fallbackLanguage = config.language;
+      repositories = config.repositories;
+    } else {
+      ref = 'main';
+      repositories = await getRepositories();
+    }
+
+    if (!repositories || repositories.length === 0) {
+      throw new UserCancellationException('No repositories to query.');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 3,
+      message: 'Determining controller repo'
+    });
+
+    // Get the controller repo from the config, if it exists.
+    // If it doesn't exist, prompt the user to enter it, and save that value to the config.
+    let controllerRepo: string | undefined;
+    controllerRepo = getRemoteControllerRepo();
+    if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
+      void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
+      controllerRepo = await window.showInputBox({
+        title: 'Controller repository in which to display progress and results of remote queries',
+        placeHolder: '<owner>/<repo>',
+        prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
+        ignoreFocusOut: true,
+      });
+      if (!controllerRepo) {
+        void showAndLogErrorMessage('No controller repository entered.');
+        return;
+      } else if (!REPO_REGEX.test(controllerRepo)) { // Check if user entered invalid input
+        void showAndLogErrorMessage('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
+        return;
+      }
+      void logger.log(`Setting the controller repository as: ${controllerRepo}`);
+      await setRemoteControllerRepo(controllerRepo);
+    }
+
+    void logger.log(`Using controller repository: ${controllerRepo}`);
+    const [owner, repo] = controllerRepo.split('/');
+
+    progress({
+      maxStep: 5,
+      step: 4,
+      message: 'Bundling the query pack'
+    });
+
+    if (token.isCancellationRequested) {
+      throw new UserCancellationException('Cancelled');
+    }
+
+    const { base64Pack, language } = await generateQueryPack(cliServer, queryFile, queryPackDir, fallbackLanguage);
+
+    if (token.isCancellationRequested) {
+      throw new UserCancellationException('Cancelled');
+    }
+
+    progress({
+      maxStep: 5,
+      step: 5,
+      message: 'Sending request'
+    });
+
+    await runRemoteQueriesApiRequest(credentials, ref, language, repositories, owner, repo, base64Pack, dryRun);
+
+    if (dryRun) {
+      return remoteQueryDir.path;
+    } else {
+      // don't return the path because it has been deleted
+      return;
+    }
+
+  } finally {
+    if (dryRun) {
+      // If we are in a dry run keep the data around for debugging purposes.
+      void logger.log(`[DRY RUN] Not deleting ${queryPackDir}.`);
+    } else {
+      await remoteQueryDir.cleanup();
+    }
+  }
+}
+
+async function runRemoteQueriesApiRequest(
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repositories: string[],
+  owner: string,
+  repo: string,
+  queryPackBase64: string,
+  dryRun = false
+): Promise<void> {
+
+  if (dryRun) {
+    void showAndLogInformationMessage('[DRY RUN] Would have sent request. See extension log for the payload.');
+    void logger.log(JSON.stringify({ ref, language, repositories, owner, repo, queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes' }));
+    return;
+  }
+
+  try {
+    const octokit = await credentials.getOctokit();
+    const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
+      'POST /repos/:owner/:repo/code-scanning/codeql/queries',
+      {
+        owner,
+        repo,
+        data: {
+          ref,
+          language,
+          repositories,
+          query_pack: queryPackBase64,
+        }
+      }
+    );
+    void showAndLogInformationMessage(`Successfully scheduled runs. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${response.data.workflow_run_id}).`);
+
+  } catch (error) {
+    await attemptRerun(error, credentials, ref, language, repositories, owner, repo, queryPackBase64, dryRun);
+  }
+}
+
+/** Attempts to rerun the query on only the valid repositories */
+export async function attemptRerun(
+  error: any,
+  credentials: Credentials,
+  ref: string,
+  language: string,
+  repositories: string[],
+  owner: string,
+  repo: string,
+  queryPackBase64: string,
+  dryRun = false
+) {
+  if (typeof error.message === 'string' && error.message.includes('Some repositories were invalid')) {
+    const invalidRepos = error?.response?.data?.invalid_repos || [];
+    const reposWithoutDbUploads = error?.response?.data?.repos_without_db_uploads || [];
+    void logger.log('Unable to run query on some of the specified repositories');
+    if (invalidRepos.length > 0) {
+      void logger.log(`Invalid repos: ${invalidRepos.join(', ')}`);
+    }
+    if (reposWithoutDbUploads.length > 0) {
+      void logger.log(`Repos without DB uploads: ${reposWithoutDbUploads.join(', ')}`);
+    }
+
+    if (invalidRepos.length + reposWithoutDbUploads.length === repositories.length) {
+      // Every repo is invalid in some way
+      void showAndLogErrorMessage('Unable to run query on any of the specified repositories.');
+      return;
+    }
+
+    const popupMessage = 'Unable to run query on some of the specified repositories. [See logs for more details](command:codeQL.showLogs).';
+    const rerunQuery = await showInformationMessageWithAction(popupMessage, 'Rerun on the valid repositories only');
+    if (rerunQuery) {
+      const validRepositories = repositories.filter(r => !invalidRepos.includes(r) && !reposWithoutDbUploads.includes(r));
+      void logger.log(`Rerunning query on set of valid repositories: ${JSON.stringify(validRepositories)}`);
+      await runRemoteQueriesApiRequest(credentials, ref, language, validRepositories, owner, repo, queryPackBase64, dryRun);
+    }
+  } else {
+    void showAndLogErrorMessage(error);
+  }
+}

extensions/ql-vscode/src/sarif-parser.ts

@@ -0,0 +1,48 @@
+import * as Sarif from 'sarif';
+import * as fs from 'fs-extra';
+import { parser } from 'stream-json';
+import { pick } from 'stream-json/filters/Pick';
+import Assembler = require('stream-json/Assembler');
+import { chain } from 'stream-chain';
+
+const DUMMY_TOOL : Sarif.Tool = {driver: {name: ''}};
+
+export async function sarifParser(interpretedResultsPath: string) : Promise<Sarif.Log> {
+  try {
+    // Parse the SARIF file into token streams, filtering out only the results array.
+    const p = parser();
+    const pipeline = chain([
+      fs.createReadStream(interpretedResultsPath),
+      p,
+      pick({filter: 'runs.0.results'})
+    ]);
+
+    // Creates JavaScript objects from the token stream
+    const asm = Assembler.connectTo(pipeline);
+
+    // Returns a constructed Log object with the results or an empty array if no results were found.
+    // If the parser fails for any reason, it will reject the promise.
+    return await new Promise((resolve, reject) => {
+      pipeline.on('error', (error) => {
+        reject(error);
+      });
+      
+      asm.on('done', (asm) => {
+
+        const log : Sarif.Log = {
+          version: '2.1.0', 
+          runs: [
+            { 
+              tool: DUMMY_TOOL, 
+              results: asm.current ?? []
+            }
+          ]
+        };
+        
+        resolve(log);
+      });
+    });
+  } catch (err) {
+    throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
+  }
+}

extensions/ql-vscode/src/status-bar.ts

@@ -0,0 +1,48 @@
+import { ConfigurationChangeEvent, StatusBarAlignment, StatusBarItem, window, workspace } from 'vscode';
+import { CodeQLCliServer } from './cli';
+import { CANARY_FEATURES, CUSTOM_CODEQL_PATH_SETTING, DistributionConfigListener } from './config';
+import { DisposableObject } from './pure/disposable-object';
+
+/**
+ * Creates and manages a status bar item for codeql. THis item contains
+ * the current codeQL cli version as well as a notification if you are
+ * in canary mode
+ *
+ */
+export class CodeQlStatusBarHandler extends DisposableObject {
+
+  private readonly item: StatusBarItem;
+
+  constructor(private cli: CodeQLCliServer, distributionConfigListener: DistributionConfigListener) {
+    super();
+    this.item = window.createStatusBarItem(StatusBarAlignment.Right);
+    this.push(this.item);
+    this.push(workspace.onDidChangeConfiguration(this.handleDidChangeConfiguration, this));
+    this.push(distributionConfigListener.onDidChangeConfiguration(() => this.updateStatusItem()));
+    this.item.command = 'codeQL.copyVersion';
+    void this.updateStatusItem();
+  }
+
+  private handleDidChangeConfiguration(e: ConfigurationChangeEvent) {
+    if (
+      e.affectsConfiguration(CANARY_FEATURES.qualifiedName) ||
+      e.affectsConfiguration(CUSTOM_CODEQL_PATH_SETTING.qualifiedName)
+    ) {
+      // Wait a few seconds before updating the status item.
+      // This avoids a race condition where the cli's version
+      // is not updated before the status bar is refreshed.
+      setTimeout(() => this.updateStatusItem(), 3000);
+    }
+  }
+
+  private async updateStatusItem() {
+    const canary = CANARY_FEATURES.getValue() ? ' (Canary)' : '';
+    // since getting the version may take a few seconds, initialize with some
+    // meaningful text.
+    this.item.text = `CodeQL${canary}`;
+
+    const version = await this.cli.getVersion();
+    this.item.text = `CodeQL CLI v${version}${canary}`;
+    this.item.show();
+  }
+}

extensions/ql-vscode/src/telemetry.ts

@@ -4,7 +4,7 @@ import { ConfigListener, CANARY_FEATURES, ENABLE_TELEMETRY, GLOBAL_ENABLE_TELEME
 import * as appInsights from 'applicationinsights';
 import { logger } from './logging';
 import { UserCancellationException } from './commandRunner';
-import { showBinaryChoiceDialog } from './helpers';
+import { showBinaryChoiceWithUrlDialog } from './helpers';
 
 // Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
 const key = 'REPLACE-APP-INSIGHTS-KEY';
@@ -119,7 +119,7 @@ export class TelemetryListener extends ConfigListener {
         }
 
         if (LOG_TELEMETRY.getValue<boolean>()) {
-          logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
+          void logger.log(`Telemetry: ${JSON.stringify(envelope)}`);
         }
         return true;
       });
@@ -128,7 +128,7 @@ export class TelemetryListener extends ConfigListener {
 
   dispose() {
     super.dispose();
-    this.reporter?.dispose();
+    void this.reporter?.dispose();
   }
 
   sendCommandUsage(name: string, executionTime: number, error?: Error) {
@@ -164,14 +164,9 @@ export class TelemetryListener extends ConfigListener {
       let result = undefined;
       if (GLOBAL_ENABLE_TELEMETRY.getValue()) {
         // Extension won't start until this completes.
-        result = await showBinaryChoiceDialog(
-          'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?\n\nFor details of what we collect and how we use it, see https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/TELEMETRY.md.',
-          // We make this dialog modal for now.
-          // Note that  non-modal dialogs allow for markdown in their text, but modal dialogs do not.
-          // If we do decide to keep this dialog as modal, then this implementation can change and
-          // we no longer need to call Promise.race. Before committing this PR, we need to make
-          // this decision.
-          true
+        result = await showBinaryChoiceWithUrlDialog(
+          'Does the CodeQL Extension by GitHub have your permission to collect usage data and metrics to help us improve CodeQL for VSCode?',
+          'https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code'
         );
       }
       if (result !== undefined) {
@@ -192,7 +187,7 @@ export class TelemetryListener extends ConfigListener {
 
   private disposeReporter() {
     if (this.reporter) {
-      this.reporter.dispose();
+      void this.reporter.dispose();
       this.reporter = undefined;
     }
   }
@@ -214,6 +209,8 @@ export let telemetryListener: TelemetryListener;
 
 export async function initializeTelemetry(extension: Extension<any>, ctx: ExtensionContext): Promise<void> {
   telemetryListener = new TelemetryListener(extension.id, extension.packageJSON.version, key, ctx);
-  telemetryListener.initialize();
+  // do not await initialization, since doing so will sometimes cause a modal popup.
+  // this is a particular problem during integration tests, which will hang if a modal popup is displayed.
+  void telemetryListener.initialize();
   ctx.subscriptions.push(telemetryListener);
 }

extensions/ql-vscode/src/test-adapter.ts

@@ -1,3 +1,4 @@
+import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as vscode from 'vscode';
 import {
@@ -15,10 +16,11 @@ import {
 import { TestAdapterRegistrar } from 'vscode-test-adapter-util';
 import { QLTestFile, QLTestNode, QLTestDirectory, QLTestDiscovery } from './qltest-discovery';
 import { Event, EventEmitter, CancellationTokenSource, CancellationToken } from 'vscode';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { CodeQLCliServer } from './cli';
-import { getOnDiskWorkspaceFolders } from './helpers';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
 import { testLogger } from './logging';
+import { DatabaseItem, DatabaseManager } from './databases';
 
 /**
  * Get the full path of the `.expected` file for the specified QL test.
@@ -57,13 +59,13 @@ function getTestOutputFile(testPath: string, extension: string): string {
  * A factory service that creates `QLTestAdapter` objects for workspace folders on demand.
  */
 export class QLTestAdapterFactory extends DisposableObject {
-  constructor(testHub: TestHub, cliServer: CodeQLCliServer) {
+  constructor(testHub: TestHub, cliServer: CodeQLCliServer, databaseManager: DatabaseManager) {
     super();
 
     // this will register a QLTestAdapter for each WorkspaceFolder
     this.push(new TestAdapterRegistrar(
       testHub,
-      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer)
+      workspaceFolder => new QLTestAdapter(workspaceFolder, cliServer, databaseManager)
     ));
   }
 }
@@ -91,7 +93,8 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
 
   constructor(
     public readonly workspaceFolder: vscode.WorkspaceFolder,
-    private readonly cliServer: CodeQLCliServer
+    private readonly cliServer: CodeQLCliServer,
+    private readonly databaseManager: DatabaseManager
   ) {
     super();
 
@@ -182,19 +185,86 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
     testLogger.outputChannel.show(true);
 
     this.runningTask = this.track(new CancellationTokenSource());
+    const token = this.runningTask.token;
 
     this._testStates.fire({ type: 'started', tests: tests } as TestRunStartedEvent);
 
+    const currentDatabaseUri = this.databaseManager.currentDatabaseItem?.databaseUri;
+    const databasesUnderTest: DatabaseItem[] = [];
+    for (const database of this.databaseManager.databaseItems) {
+      for (const test of tests) {
+        if (await database.isAffectedByTest(test)) {
+          databasesUnderTest.push(database);
+          break;
+        }
+      }
+    }
+
+    await this.removeDatabasesBeforeTests(databasesUnderTest, token);
     try {
-      await this.runTests(tests, this.runningTask.token);
-    }
-    catch (e) {
-      /**/
+      await this.runTests(tests, token);
+    } catch (e) {
+      // CodeQL testing can throw exception even in normal scenarios. For example, if the test run
+      // produces no output (which is normal), the testing command would throw an exception on
+      // unexpected EOF during json parsing. So nothing needs to be done here - all the relevant
+      // error information (if any) should have already been written to the test logger.
     }
+    await this.reopenDatabasesAfterTests(databasesUnderTest, currentDatabaseUri, token);
+
     this._testStates.fire({ type: 'finished' } as TestRunFinishedEvent);
     this.clearTask();
   }
 
+  private async removeDatabasesBeforeTests(
+    databasesUnderTest: DatabaseItem[], token: vscode.CancellationToken): Promise<void> {
+    for (const database of databasesUnderTest) {
+      try {
+        await this.databaseManager
+          .removeDatabaseItem(_ => { /* no progress reporting */ }, token, database);
+      } catch (e) {
+        // This method is invoked from Test Explorer UI, and testing indicates that Test
+        // Explorer UI swallows any thrown exception without reporting it to the user.
+        // So we need to display the error message ourselves and then rethrow.
+        void showAndLogErrorMessage(`Cannot remove database ${database.name}: ${e}`);
+        throw e;
+      }
+    }
+  }
+
+  private async reopenDatabasesAfterTests(
+    databasesUnderTest: DatabaseItem[],
+    currentDatabaseUri: vscode.Uri | undefined,
+    token: vscode.CancellationToken): Promise<void> {
+    for (const closedDatabase of databasesUnderTest) {
+      const uri = closedDatabase.databaseUri;
+      if (await this.isFileAccessible(uri)) {
+        try {
+          const reopenedDatabase = await this.databaseManager
+            .openDatabase(_ => { /* no progress reporting */ }, token, uri);
+          await this.databaseManager.renameDatabaseItem(reopenedDatabase, closedDatabase.name);
+          if (currentDatabaseUri == uri) {
+            await this.databaseManager.setCurrentDatabaseItem(reopenedDatabase, true);
+          }
+        } catch (e) {
+          // This method is invoked from Test Explorer UI, and testing indicates that Test
+          // Explorer UI swallows any thrown exception without reporting it to the user.
+          // So we need to display the error message ourselves and then rethrow.
+          void showAndLogWarningMessage(`Cannot reopen database ${uri}: ${e}`);
+          throw e;
+        }
+      }
+    }
+  }
+
+  private async isFileAccessible(uri: vscode.Uri): Promise<boolean> {
+    try {
+      await fs.access(uri.fsPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
   private clearTask(): void {
     if (this.runningTask !== undefined) {
       const runningTask = this.runningTask;
@@ -205,7 +275,7 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
 
   public cancel(): void {
     if (this.runningTask !== undefined) {
-      testLogger.log('Cancelling test run...');
+      void testLogger.log('Cancelling test run...');
       this.runningTask.cancel();
       this.clearTask();
     }
@@ -223,9 +293,11 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
           ? 'errored'
           : 'failed';
       let message: string | undefined;
-      if (event.diff?.length) {
-        message = ['', `${state}: ${event.test}`, ...event.diff, ''].join('\n');
-        testLogger.log(message);
+      if (event.failureDescription || event.diff?.length) {
+        message = event.failureStage === 'RESULT'
+          ? ['', `${state}: ${event.test}`, event.failureDescription || event.diff?.join('\n'), ''].join('\n')
+          : ['', `${event.failureStage?.toLowerCase()} error: ${event.test}`, event.failureDescription || `${event.messages[0].severity}: ${event.messages[0].message}`, ''].join('\n');
+        void testLogger.log(message);
       }
       this._testStates.fire({
         type: 'test',

extensions/ql-vscode/src/test-ui.ts

@@ -13,7 +13,7 @@ import {
 
 import { showAndLogWarningMessage } from './helpers';
 import { TestTreeNode } from './test-tree-node';
-import { DisposableObject } from './vscode-utils/disposable-object';
+import { DisposableObject } from './pure/disposable-object';
 import { UIService } from './vscode-utils/ui-service';
 import { QLTestAdapter, getExpectedFile, getActualFile } from './test-adapter';
 import { logger } from './logging';
@@ -44,7 +44,7 @@ export class TestUIService extends UIService implements TestController {
   constructor(private readonly testHub: TestHub) {
     super();
 
-    logger.log('Registering CodeQL test panel commands.');
+    void logger.log('Registering CodeQL test panel commands.');
     this.registerCommand('codeQLTests.showOutputDifferences', this.showOutputDifferences);
     this.registerCommand('codeQLTests.acceptOutput', this.acceptOutput);
 
@@ -90,7 +90,7 @@ export class TestUIService extends UIService implements TestController {
       };
 
       if (!await fs.pathExists(expectedPath)) {
-        showAndLogWarningMessage(`'${path.basename(expectedPath)}' does not exist. Creating an empty file.`);
+        void showAndLogWarningMessage(`'${path.basename(expectedPath)}' does not exist. Creating an empty file.`);
         await fs.createFile(expectedPath);
       }
 

extensions/ql-vscode/src/upgrades.ts

@@ -25,7 +25,7 @@ const MAX_UPGRADE_MESSAGE_LINES = 10;
  * resolving upgrades. We check for a version of codeql that has all three features.
  */
 export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QueryServerClient): Promise<boolean> {
-  return semver.gte(await qs.cliServer.getVersion(), '2.4.1');
+  return semver.gte(await qs.cliServer.getVersion(), '2.4.2');
 }
 
 
@@ -103,7 +103,7 @@ async function checkAndConfirmDatabaseUpgrade(
     descriptionMessage += `Would perform upgrade: ${script.description}\n`;
     descriptionMessage += `\t-> Compatibility: ${script.compatibility}\n`;
   }
-  logger.log(descriptionMessage);
+  void logger.log(descriptionMessage);
 
 
   // If the quiet flag is set, do the upgrade without a popup.
@@ -171,7 +171,8 @@ export async function upgradeDatabaseExplicit(
   }
   const upgradeInfo = await qs.cliServer.resolveUpgrades(
     db.contents.dbSchemeUri.fsPath,
-    searchPath
+    searchPath,
+    false
   );
 
   const { scripts, finalDbscheme } = upgradeInfo;
@@ -186,35 +187,35 @@ export async function upgradeDatabaseExplicit(
       compileUpgradeResult = await compileDatabaseUpgrade(qs, db, finalDbscheme, scripts, currentUpgradeTmp, progress, token);
     }
     catch (e) {
-      showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
+      void showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
       return;
     }
     finally {
-      qs.logger.log('Done compiling database upgrade.');
+      void qs.logger.log('Done compiling database upgrade.');
     }
 
     if (!compileUpgradeResult.compiledUpgrades) {
       const error = compileUpgradeResult.error || '[no error message available]';
-      showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
+      void showAndLogErrorMessage(`Compilation of database upgrades failed: ${error}`);
       return;
     }
 
     await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, db, qs.cliServer.quiet);
 
     try {
-      qs.logger.log('Running the following database upgrade:');
+      void qs.logger.log('Running the following database upgrade:');
 
       getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n');
       return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
     }
     catch (e) {
-      showAndLogErrorMessage(`Database upgrade failed: ${e}`);
+      void showAndLogErrorMessage(`Database upgrade failed: ${e}`);
       return;
     } finally {
-      qs.logger.log('Done running database upgrade.');
+      void qs.logger.log('Done running database upgrade.');
     }
   } finally {
-    currentUpgradeTmp.cleanup();
+    await currentUpgradeTmp.cleanup();
   }
 }
 

extensions/ql-vscode/src/view/RawTableValue.tsx

@@ -9,14 +9,14 @@ interface Props {
 }
 
 export default function RawTableValue(props: Props): JSX.Element {
-  const v = props.value;
+  const rawValue = props.value;
   if (
-    typeof v === 'string'
-    || typeof v === 'number'
-    || typeof v === 'boolean'
+    typeof rawValue === 'string'
+    || typeof rawValue === 'number'
+    || typeof rawValue === 'boolean'
   ) {
-    return <span>{v.toString()}</span>;
+    return <span>{renderLocation(undefined, rawValue.toString())}</span>;
   }
 
-  return renderLocation(v.url, v.label, props.databaseUri);
+  return renderLocation(rawValue.url, rawValue.label, props.databaseUri);
 }

extensions/ql-vscode/src/view/alert-table.tsx

@@ -3,7 +3,7 @@ import * as React from 'react';
 import * as Sarif from 'sarif';
 import * as Keys from '../pure/result-keys';
 import * as octicons from './octicons';
-import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection } from './result-table-utils';
+import { className, renderLocation, ResultTableProps, zebraStripe, selectableZebraStripe, jumpToLocation, nextSortDirection, emptyQueryResultsMessage } from './result-table-utils';
 import { onNavigation, NavigationEvent } from './results';
 import { PathTableResultSet } from '../pure/interface-types';
 import {
@@ -79,7 +79,7 @@ export class PathTable extends React.Component<PathTableProps, PathTableState> {
     if (this.props.nonemptyRawResults) {
       return <span>No Alerts. See <a href='#' onClick={this.props.showRawResults}>raw results</a>.</span>;
     } else {
-      return <span>No Alerts</span>;
+      return emptyQueryResultsMessage();
     }
   }
 

extensions/ql-vscode/src/view/raw-results-table.tsx

@@ -1,5 +1,5 @@
 import * as React from 'react';
-import { ResultTableProps, className } from './result-table-utils';
+import { ResultTableProps, className, emptyQueryResultsMessage } from './result-table-utils';
 import { RAW_RESULTS_LIMIT, RawResultsSortState } from '../pure/interface-types';
 import { RawTableResultSet } from '../pure/interface-types';
 import RawTableHeader from './RawTableHeader';
@@ -12,7 +12,7 @@ export type RawTableProps = ResultTableProps & {
   offset: number;
 };
 
-export class RawTable extends React.Component<RawTableProps, {}> {
+export class RawTable extends React.Component<RawTableProps, Record<string, never>> {
   constructor(props: RawTableProps) {
     super(props);
   }
@@ -21,6 +21,10 @@ export class RawTable extends React.Component<RawTableProps, {}> {
     const { resultSet, databaseUri } = this.props;
 
     let dataRows = resultSet.rows;
+    if (dataRows.length === 0) {
+      return emptyQueryResultsMessage();
+    }
+
     let numTruncatedResults = 0;
     if (dataRows.length > RAW_RESULTS_LIMIT) {
       numTruncatedResults = dataRows.length - RAW_RESULTS_LIMIT;

extensions/ql-vscode/src/view/result-table-utils.tsx

@@ -37,6 +37,9 @@ export const oddRowClassName = 'vscode-codeql__result-table-row--odd';
 export const pathRowClassName = 'vscode-codeql__result-table-row--path';
 export const selectedRowClassName = 'vscode-codeql__result-table-row--selected';
 
+const CONTROL_CODE = '\u001F'.codePointAt(0)!;
+const CONTROL_LABEL = '\u2400'.codePointAt(0)!;
+
 export function jumpToLocationHandler(
   loc: ResolvableLocationValue,
   databaseUri: string,
@@ -67,24 +70,42 @@ export function openFile(filePath: string): void {
   });
 }
 
+function convertedNonprintableChars(label: string) {
+  // If the label was empty, use a placeholder instead, so the link is still clickable.
+  if (!label) {
+    return '[empty string]';
+  } else if (label.match(/^\s+$/)) {
+    return `[whitespace: "${label}"]`;
+  } else {
+    /**
+     * If the label contains certain non-printable characters, loop through each
+     * character and replace it with the cooresponding unicode control label.
+    */
+    const convertedLabelArray: any[] = [];
+    for (let i = 0; i < label.length; i++) {
+      const labelCheck = label.codePointAt(i)!;
+      if (labelCheck <= CONTROL_CODE) {
+        convertedLabelArray[i] = String.fromCodePoint(labelCheck + CONTROL_LABEL);
+      } else {
+        convertedLabelArray[i] = label.charAt(i);
+      }
+    }
+    return convertedLabelArray.join('');
+  }
+}
+
 /**
  * Render a location as a link which when clicked displays the original location.
  */
 export function renderLocation(
-  loc: UrlValue | undefined,
-  label: string | undefined,
-  databaseUri: string,
+  loc?: UrlValue,
+  label?: string,
+  databaseUri?: string,
   title?: string,
   callback?: () => void
 ): JSX.Element {
 
-  // If the label was empty, use a placeholder instead, so the link is still clickable.
-  let displayLabel = label;
-  if (!label) {
-    displayLabel = '[empty string]';
-  } else if (label.match(/^\s+$/)) {
-    displayLabel = `[whitespace: "${label}"]`;
-  }
+  const displayLabel = convertedNonprintableChars(label!);
 
   if (loc === undefined) {
     return <span>{displayLabel}</span>;
@@ -93,7 +114,7 @@ export function renderLocation(
   }
 
   const resolvableLoc = tryGetResolvableLocation(loc);
-  if (resolvableLoc !== undefined) {
+  if (databaseUri !== undefined && resolvableLoc !== undefined) {
     return (
       <a href="#"
         className="vscode-codeql__result-table-location-link"
@@ -140,3 +161,9 @@ export function nextSortDirection(direction: SortDirection | undefined, includeU
       return assertNever(direction);
   }
 }
+
+export function emptyQueryResultsMessage(): JSX.Element {
+  return <div className='vscode-codeql__empty-query-message'><span>
+    This query returned no results. If this isn&apos;t what you were expecting, and for effective query-writing tips, check out the <a href="https://codeql.github.com/docs/codeql-language-guides/">CodeQL language guides</a>.
+  </span></div>;
+}

extensions/ql-vscode/src/view/result-tables.tsx

@@ -82,7 +82,7 @@ export class ResultTables
 
   private getResultSets(): ResultSet[] {
     const resultSets: ResultSet[] =
-      // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore 2783
       this.props.rawResultSets.map((rs) => ({ t: 'RawResultSet', ...rs }));
 
@@ -328,13 +328,15 @@ export class ResultTables
   }
 
   private vscodeMessageHandler(evt: MessageEvent) {
+    // sanitize origin
+    const origin = evt.origin.replace(/\n|\r/g, '');
     evt.origin === window.origin
       ? this.handleMessage(evt.data as IntoResultsViewMsg)
-      : console.error(`Invalid event origin ${evt.origin}`);
+      : console.error(`Invalid event origin ${origin}`);
   }
 }
 
-class ResultTable extends React.Component<ResultTableProps, {}> {
+class ResultTable extends React.Component<ResultTableProps, Record<string, never>> {
 
   constructor(props: ResultTableProps) {
     super(props);

extensions/ql-vscode/src/view/results.tsx

@@ -71,7 +71,7 @@ export const onNavigation = new EventHandlerList<NavigationEvent>();
 /**
  * A minimal state container for displaying results.
  */
-class App extends React.Component<{}, ResultsViewState> {
+class App extends React.Component<Record<string, never>, ResultsViewState> {
   constructor(props: any) {
     super(props);
     this.state = {
@@ -102,7 +102,7 @@ class App extends React.Component<{}, ResultsViewState> {
           queryPath: msg.queryPath,
         });
 
-        this.loadResults();
+        void this.loadResults();
         break;
       case 'showInterpretedPage':
         this.updateStateWithNewResultsInfo({
@@ -134,7 +134,7 @@ class App extends React.Component<{}, ResultsViewState> {
           queryName: msg.queryName,
           queryPath: msg.queryPath,
         });
-        this.loadResults();
+        void this.loadResults();
         break;
       case 'resultsUpdating':
         this.setState({
@@ -307,9 +307,11 @@ class App extends React.Component<{}, ResultsViewState> {
   }
 
   private vscodeMessageHandler(evt: MessageEvent) {
+    // sanitize origin
+    const origin = evt.origin.replace(/\n|\r/g, '');
     evt.origin === window.origin
       ? this.handleMessage(evt.data as IntoResultsViewMsg)
-      : console.error(`Invalid event origin ${evt.origin}`);
+      : console.error(`Invalid event origin ${origin}`);
   }
 }
 

extensions/ql-vscode/src/view/resultsView.css

@@ -15,6 +15,10 @@
   display: flex;
   padding: 0.5em 0;
   align-items: center;
+  top: 0;
+  background-color: var(--vscode-editorGutter-background);
+  position: sticky;
+  z-index: 1;
 }
 
 .vscode-codeql__table-selection-header-item {
@@ -221,3 +225,16 @@ td.vscode-codeql__path-index-cell {
 .vscode-codeql__compare-body > tbody {
   vertical-align: top;
 }
+
+.vscode-codeql__empty-query-message {
+  height: 300px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.vscode-codeql__empty-query-message > span {
+  max-width: 80%;
+  font-size: 14px;
+  text-align: center;
+}

extensions/ql-vscode/src/vscode-tests/.eslintrc.js

@@ -1,5 +1,19 @@
 module.exports = {
   env: {
     mocha: true
+  },
+  rules: {
+    "@typescript-eslint/ban-types": [
+      "error",
+      { 
+        // For a full list of the default banned types, see:
+        // https://github.com/typescript-eslint/typescript-eslint/blob/master/packages/eslint-plugin/docs/rules/ban-types.md
+        extendDefaults: true,
+        "types": {
+          // Don't complain about the `Function` type in test files. (Default is `true`.)
+          "Function": false,
+        }
+      }
+    ]
   }
 }

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-no-qlpack/in-pack.ql

@@ -0,0 +1,2 @@
+// This file should not be included the remote query pack.
+select 1

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-no-qlpack/lib.qll

@@ -0,0 +1,3 @@
+int number() {
+  result = 1
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-no-qlpack/not-in-pack.ql

@@ -0,0 +1,4 @@
+import javascript
+import lib
+
+select number()

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/in-pack.ql

@@ -0,0 +1,4 @@
+import javascript
+import lib
+
+select number()

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/lib.qll

@@ -0,0 +1,3 @@
+int number() {
+  result = 1
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/not-in-pack.ql

@@ -0,0 +1,2 @@
+// This file should not be included the remote query pack.
+select 1

extensions/ql-vscode/src/vscode-tests/cli-integration/data-remote-qlpack/qlpack.yml

@@ -0,0 +1,5 @@
+name: github/remote-query-pack
+version: 0.0.0
+extractor: javascript
+dependencies:
+  codeql/javascript-all: '*'

extensions/ql-vscode/src/vscode-tests/cli-integration/data/simple-javascript-query.ql

@@ -0,0 +1,3 @@
+import javascript
+
+select 1

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -5,6 +5,7 @@ import { expect } from 'chai';
 import { extensions, CancellationToken, Uri, window } from 'vscode';
 
 import { CodeQLExtensionInterface } from '../../extension';
+import { CodeQLCliServer } from '../../cli';
 import { DatabaseManager } from '../../databases';
 import { promptImportLgtmDatabase, importArchiveDatabase, promptImportInternetDatabase } from '../../databaseFetcher';
 import { ProgressCallback } from '../../commandRunner';
@@ -17,15 +18,16 @@ describe('Databases', function() {
   this.timeout(60000);
 
   const LGTM_URL = 'https://lgtm.com/projects/g/aeisenberg/angular-bind-notifier/';
-
+  
   let databaseManager: DatabaseManager;
   let sandbox: sinon.SinonSandbox;
   let inputBoxStub: sinon.SinonStub;
+  let cli: CodeQLCliServer;
   let progressCallback: ProgressCallback;
 
   beforeEach(async () => {
     try {
-      const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
       if ('databaseManager' in extension) {
         databaseManager = extension.databaseManager;
       } else {
@@ -44,7 +46,6 @@ describe('Databases', function() {
 
   afterEach(() => {
     try {
-      // dispose();
       sandbox.restore();
     } catch (e) {
       fail(e);
@@ -54,7 +55,7 @@ describe('Databases', function() {
   it('should add a database from a folder', async () => {
     const progressCallback = sandbox.spy() as ProgressCallback;
     const uri = Uri.file(dbLoc);
-    let dbItem = await importArchiveDatabase(uri.toString(true), databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    let dbItem = await importArchiveDatabase(uri.toString(true), databaseManager, storagePath, progressCallback, {} as CancellationToken, cli);
     expect(dbItem).to.be.eq(databaseManager.currentDatabaseItem);
     expect(dbItem).to.be.eq(databaseManager.databaseItems[0]);
     expect(dbItem).not.to.be.undefined;
@@ -65,7 +66,7 @@ describe('Databases', function() {
 
   it('should add a database from lgtm with only one language', async () => {
     inputBoxStub.resolves(LGTM_URL);
-    let dbItem = await promptImportLgtmDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    let dbItem = await promptImportLgtmDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken, cli);
     expect(dbItem).not.to.be.undefined;
     dbItem = dbItem!;
     expect(dbItem.name).to.eq('aeisenberg_angular-bind-notifier_106179a');
@@ -75,7 +76,7 @@ describe('Databases', function() {
   it('should add a database from a url', async () => {
     inputBoxStub.resolves(DB_URL);
 
-    let dbItem = await promptImportInternetDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    let dbItem = await promptImportInternetDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken, cli);
     expect(dbItem).not.to.be.undefined;
     dbItem = dbItem!;
     expect(dbItem.name).to.eq('db');

extensions/ql-vscode/src/vscode-tests/cli-integration/global.helper.ts

@@ -16,20 +16,18 @@ export const DB_URL = 'https://github.com/github/vscode-codeql/files/5586722/sim
 export const dbLoc = path.join(fs.realpathSync(path.join(__dirname, '../../../')), 'build/tests/db.zip');
 export let storagePath: string;
 
-// See https://github.com/DefinitelyTyped/DefinitelyTyped/pull/49860
-// Should be of type Mocha
-export default function(mocha: /*Mocha*/ any) {
+export default function(mocha: Mocha) {
   // create an extension storage location
   let removeStorage: tmp.DirResult['removeCallback'] | undefined;
 
-  mocha.globalSetup([
-    // ensure the test database is downloaded
+  // ensure the test database is downloaded
+  (mocha.options as any).globalSetup.push(
     async () => {
       fs.mkdirpSync(path.dirname(dbLoc));
       if (!fs.existsSync(dbLoc)) {
         try {
           await new Promise((resolve, reject) => {
-            fetch(DB_URL).then(response => {
+            return fetch(DB_URL).then(response => {
               const dest = fs.createWriteStream(dbLoc);
               response.body.pipe(dest);
 
@@ -44,14 +42,18 @@ export default function(mocha: /*Mocha*/ any) {
           fail('Failed to download test database: ' + e);
         }
       }
-    },
+    }
+  );
 
-    // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+  // Set the CLI version here before activation to ensure we don't accidentally try to download a cli
+  (mocha.options as any).globalSetup.push(
     async () => {
       await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
-    },
+    }
+  );
 
-    // Create the temp directory to be used as extension local storage.
+  // Create the temp directory to be used as extension local storage.
+  (mocha.options as any).globalSetup.push(
     () => {
       const dir = tmp.dirSync();
       storagePath = fs.realpathSync(dir.name);
@@ -61,21 +63,24 @@ export default function(mocha: /*Mocha*/ any) {
 
       removeStorage = dir.removeCallback;
     }
-  ]);
+  );
 
-  mocha.globalTeardown([
-    // ensure etension is cleaned up.
+  // ensure etension is cleaned up.
+  (mocha.options as any).globalTeardown.push(
     async () => {
-      const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
       // This shuts down the extension and can only be run after all tests have completed.
       // If this is not called, then the test process will hang.
       if ('dispose' in extension) {
         extension.dispose();
       }
-    },
-    // ensure temp directory is cleaned up.
+    }
+  );
+
+  // ensure temp directory is cleaned up.
+  (mocha.options as any).globalTeardown.push(
     () => {
       removeStorage?.();
     }
-  ]);
+  );
 }

extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts

@@ -1,6 +1,10 @@
-import 'mocha';
-import 'sinon-chai';
 import { runTestsInDirectory } from '../index-template';
+import 'mocha';
+import * as sinonChai from 'sinon-chai';
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+chai.use(chaiAsPromised);
+chai.use(sinonChai);
 
 // The simple database used throughout the tests
 export function run(): Promise<void> {

extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts

@@ -1,10 +1,11 @@
 import { fail } from 'assert';
-import { CancellationToken, commands, extensions, Uri } from 'vscode';
+import { CancellationToken, commands, ExtensionContext, extensions, Uri } from 'vscode';
 import * as sinon from 'sinon';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import 'mocha';
 import { expect } from 'chai';
+import * as yaml from 'js-yaml';
 
 import { DatabaseItem, DatabaseManager } from '../../databases';
 import { CodeQLExtensionInterface } from '../../extension';
@@ -34,17 +35,25 @@ describe('Queries', function() {
   let sandbox: sinon.SinonSandbox;
   let progress: sinon.SinonSpy;
   let token: CancellationToken;
+  let ctx: ExtensionContext;
+
+  let qlpackFile: string;
+  let qlFile: string;
+
 
   beforeEach(async () => {
     sandbox = sinon.createSandbox();
 
     try {
-      const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
       if ('databaseManager' in extension) {
         databaseManager = extension.databaseManager;
         cli = extension.cliServer;
         qs = extension.qs;
         cli.quiet = true;
+        ctx = extension.ctx;
+        qlpackFile = `${ctx.storagePath}/quick-queries/qlpack.yml`;
+        qlFile = `${ctx.storagePath}/quick-queries/quick-query.ql`;
       } else {
         throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
       }
@@ -59,7 +68,8 @@ describe('Queries', function() {
         databaseManager,
         storagePath,
         progress,
-        token
+        token,
+        cli
       );
 
       if (!maybeDbItem) {
@@ -126,4 +136,42 @@ describe('Queries', function() {
       fail(e);
     }
   });
+
+  it('should create a quick query', async () => {
+    safeDel(qlFile);
+    safeDel(qlpackFile);
+
+    await commands.executeCommand('codeQL.quickQuery');
+
+    // should have created the quick query file and query pack file
+    expect(fs.pathExistsSync(qlFile)).to.be.true;
+    expect(fs.pathExistsSync(qlpackFile)).to.be.true;
+
+    const qlpackContents: any = await yaml.safeLoad(
+      fs.readFileSync(qlpackFile, 'utf8')
+    );
+    // Should have chosen the js libraries
+    expect(qlpackContents.libraryPathDependencies[0]).to.include('javascript');
+  });
+
+  it('should avoid creating a quick query', async () => {
+    fs.writeFileSync(qlpackFile, yaml.safeDump({
+      name: 'quick-query',
+      version: '1.0.0',
+      libraryPathDependencies: ['codeql-javascript']
+    }));
+    fs.writeFileSync(qlFile, 'xxx');
+    await commands.executeCommand('codeQL.quickQuery');
+
+    // should not have created the quick query file because database schema hasn't changed
+    expect(fs.readFileSync(qlFile, 'utf8')).to.eq('xxx');
+  });
+
+  function safeDel(file: string) {
+    try {
+      fs.unlinkSync(file);
+    } catch (e) {
+      // ignore
+    }
+  }
 });

extensions/ql-vscode/src/vscode-tests/cli-integration/query.test.ts

@@ -34,7 +34,7 @@ class Checkpoint<T> {
     this.res = () => { /**/ };
     this.rej = () => { /**/ };
     this.promise = new Promise((res, rej) => {
-      this.res = res as () => {};
+      this.res = res as () => Record<string, never>;
       this.rej = rej;
     });
   }
@@ -104,7 +104,7 @@ describe('using the query server', function() {
 
   beforeEach(async () => {
     try {
-      const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+      const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
       if ('cliServer' in extension && 'qs' in extension) {
         cliServer = extension.cliServer;
         qs = extension.qs;
@@ -119,7 +119,7 @@ describe('using the query server', function() {
 
   it('should be able to start the query server', async function() {
     await qs.startQueryServer();
-    queryServerStarted.resolve();
+    await queryServerStarted.resolve();
   });
 
   for (const queryTestCase of queryTestCases) {
@@ -129,7 +129,7 @@ describe('using the query server', function() {
     const parsedResults = new Checkpoint<void>();
 
     it('should register the database if necessary', async () => {
-      if (await qs.supportsDatabaseRegistration()) {
+      if (await cliServer.cliConstraints.supportsDatabaseRegistration()) {
         await qs.sendRequest(messages.registerDatabases, { databases: [db] }, token, (() => { /**/ }) as any);
       }
     });
@@ -160,10 +160,10 @@ describe('using the query server', function() {
         };
         const result = await qs.sendRequest(messages.compileQuery, params, token, () => { /**/ });
         expect(result.messages!.length).to.equal(0);
-        compilationSucceeded.resolve();
+        await compilationSucceeded.resolve();
       }
       catch (e) {
-        compilationSucceeded.reject(e);
+        await compilationSucceeded.reject(e);
       }
     });
 
@@ -171,7 +171,7 @@ describe('using the query server', function() {
       try {
         await compilationSucceeded.done();
         const callbackId = qs.registerCallback(_res => {
-          evaluationSucceeded.resolve();
+          void evaluationSucceeded.resolve();
         });
         const queryToRun: messages.QueryToRun = {
           resultsPath: RESULTS_PATH,
@@ -190,7 +190,7 @@ describe('using the query server', function() {
         await qs.sendRequest(messages.runQueries, params, token, () => { /**/ });
       }
       catch (e) {
-        evaluationSucceeded.reject(e);
+        await evaluationSucceeded.reject(e);
       }
     });
 
@@ -203,7 +203,7 @@ describe('using the query server', function() {
         const decoded = await cliServer.bqrsDecode(RESULTS_PATH, resultSet.name);
         actualResultSets[resultSet.name] = decoded.tuples;
       }
-      parsedResults.resolve();
+      await parsedResults.resolve();
     });
 
     it(`should have correct results for query ${queryName}`, async function() {

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -1,11 +1,14 @@
 import { expect } from 'chai';
-import { extensions } from 'vscode';
+import { extensions, Uri } from 'vscode';
+import * as path from 'path';
 import { SemVer } from 'semver';
 
-import { CodeQLCliServer } from '../../cli';
+import { CodeQLCliServer, QueryInfoByLanguage } from '../../cli';
 import { CodeQLExtensionInterface } from '../../extension';
 import { skipIfNoCodeQL } from '../ensureCli';
-import { getOnDiskWorkspaceFolders } from '../../helpers';
+import { getOnDiskWorkspaceFolders, getQlPackForDbscheme, languageToDbScheme } from '../../helpers';
+import { resolveQueries } from '../../contextual/queryResolver';
+import { KeyType } from '../../contextual/keyType';
 
 /**
  * Perform proper integration tests by running the CLI
@@ -14,23 +17,27 @@ describe('Use cli', function() {
   this.timeout(60000);
 
   let cli: CodeQLCliServer;
+  let supportedLanguages: string[];
 
   beforeEach(async () => {
-    const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+    const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
     if ('cliServer' in extension) {
       cli = extension.cliServer;
+      supportedLanguages = await cli.getSupportedLanguages();
     } else {
       throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
     }
   });
 
-  it('should have the correct version of the cli', async () => {
-    expect(
-      (await cli.getVersion()).toString()
-    ).to.eq(
-      new SemVer(process.env.CLI_VERSION || '').toString()
-    );
-  });
+  if (process.env.CLI_VERSION && process.env.CLI_VERSION !== 'nightly') {
+    it('should have the correct version of the cli', async () => {
+      expect(
+        (await cli.getVersion()).toString()
+      ).to.eq(
+        new SemVer(process.env.CLI_VERSION || '').toString()
+      );
+    });
+  }
 
   it('should resolve ram', async () => {
     const result = await (cli as any).resolveRam(8192);
@@ -43,11 +50,48 @@ describe('Use cli', function() {
   it('should resolve query packs', async function() {
     skipIfNoCodeQL(this);
     const qlpacks = await cli.resolveQlpacks(getOnDiskWorkspaceFolders());
-    // should have a bunch of qlpacks. just check that a few known ones exist
-    expect(qlpacks['codeql-cpp']).not.to.be.undefined;
-    expect(qlpacks['codeql-csharp']).not.to.be.undefined;
-    expect(qlpacks['codeql-java']).not.to.be.undefined;
-    expect(qlpacks['codeql-javascript']).not.to.be.undefined;
-    expect(qlpacks['codeql-python']).not.to.be.undefined;
+    // Depending on the version of the CLI, the qlpacks may have different names
+    // (e.g. "codeql/javascript-all" vs "codeql-javascript"),
+    // so we just check that the expected languages are included.
+    for (const expectedLanguage of supportedLanguages) {
+      expect((Object.keys(qlpacks)).includes(expectedLanguage));
+    }
+  });
+
+  it('should support the expected languages', async function() {
+    skipIfNoCodeQL(this);
+    // Just check a few examples that definitely are/aren't supported.
+    expect(supportedLanguages).to.include.members(['go', 'javascript', 'python']);
+    expect(supportedLanguages).to.not.include.members(['xml', 'properties']);
+  });
+
+  it('should resolve query by language', async function() {
+    skipIfNoCodeQL(this);
+    const queryPath = path.join(__dirname, 'data', 'simple-javascript-query.ql');
+    const queryInfo: QueryInfoByLanguage = await cli.resolveQueryByLanguage(getOnDiskWorkspaceFolders(), Uri.file(queryPath));
+    expect((Object.keys(queryInfo.byLanguage))[0]).to.eql('javascript');
+  });
+
+  it('should resolve printAST queries for supported languages', async function() {
+    skipIfNoCodeQL(this);
+    supportedLanguages.forEach(async lang => {
+      if (lang === 'go') {
+        // The codeql-go submodule is not available in the integration tests.
+        return;
+      }
+
+      console.log(`resolving printAST queries for ${lang}`);
+      const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]);
+      expect(pack.dbschemePack).to.contain(lang);
+      if (pack.dbschemePackIsLibraryPack) {
+        expect(pack.queryPack).to.contain(lang);
+      }
+
+      const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery);
+
+      // It doesn't matter what the name or path of the query is, only
+      // that we have found exactly one query.
+      expect(result.length).to.eq(1);
+    });
   });
 });

extensions/ql-vscode/src/vscode-tests/cli-integration/run-remote-query.test.ts

@@ -0,0 +1,194 @@
+import { assert, expect } from 'chai';
+import * as path from 'path';
+import * as sinon from 'sinon';
+import { CancellationToken, extensions, QuickPickItem, Uri, window } from 'vscode';
+import 'mocha';
+import * as fs from 'fs-extra';
+import * as yaml from 'js-yaml';
+
+import { runRemoteQuery } from '../../run-remote-query';
+import { Credentials } from '../../authentication';
+import { CliVersionConstraint, CodeQLCliServer } from '../../cli';
+import { CodeQLExtensionInterface } from '../../extension';
+import { setRemoteControllerRepo, setRemoteRepositoryLists } from '../../config';
+import { UserCancellationException } from '../../commandRunner';
+
+describe('Remote queries', function() {
+  const baseDir = path.join(__dirname, '../../../src/vscode-tests/cli-integration');
+
+  let sandbox: sinon.SinonSandbox;
+
+  // up to 3 minutes per test
+  this.timeout(3 * 60 * 1000);
+
+  let cli: CodeQLCliServer;
+  let credentials: Credentials = {} as unknown as Credentials;
+  let token: CancellationToken;
+  let progress: sinon.SinonSpy;
+  let showQuickPickSpy: sinon.SinonStub;
+
+  // use `function` so we have access to `this`
+  beforeEach(async function() {
+    sandbox = sinon.createSandbox();
+
+    const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
+    if ('cliServer' in extension) {
+      cli = extension.cliServer;
+    } else {
+      throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+    }
+
+    if (!(await cli.cliConstraints.supportsRemoteQueries())) {
+      console.log(`Remote queries are not supported on CodeQL CLI v${CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
+        }. Skipping this test.`);
+      this.skip();
+    }
+    credentials = {} as unknown as Credentials;
+    token = {
+      isCancellationRequested: false
+    } as unknown as CancellationToken;
+
+    progress = sandbox.spy();
+    // Should not have asked for a language
+    showQuickPickSpy = sandbox.stub(window, 'showQuickPick')
+      .onFirstCall().resolves({ repoList: ['github/vscode-codeql'] } as unknown as QuickPickItem)
+      .onSecondCall().resolves('javascript' as unknown as QuickPickItem);
+
+    // always run in the vscode-codeql repo
+    await setRemoteControllerRepo('github/vscode-codeql');
+    await setRemoteRepositoryLists({ 'vscode-codeql': ['github/vscode-codeql'] });
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it('should run a remote query that is part of a qlpack', async () => {
+    const fileUri = getFile('data-remote-qlpack/in-pack.ql');
+
+    const queryPackRootDir = (await runRemoteQuery(cli, credentials, fileUri, true, progress, token))!;
+    printDirectoryContents(queryPackRootDir);
+
+    // to retrieve the list of repositories
+    expect(showQuickPickSpy).to.have.been.calledOnce;
+
+    // check a few files that we know should exist and others that we know should not
+
+    // the tarball to deliver to the server
+    expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
+
+    const queryPackDir = path.join(queryPackRootDir, 'query-pack');
+    printDirectoryContents(queryPackDir);
+
+    // in-pack.ql renamed to query.ql
+    expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
+
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // the compiled pack
+    const compiledPackDir = path.join(queryPackDir, '.codeql/pack/github/remote-query-pack/0.0.0/');
+    printDirectoryContents(compiledPackDir);
+
+    expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // dependencies
+    const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+    const packNames = fs.readdirSync(libraryDir).sort();
+    expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']);
+  });
+
+  it('should run a remote query that is not part of a qlpack', async () => {
+    const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
+
+    const queryPackRootDir = (await runRemoteQuery(cli, credentials, fileUri, true, progress, token))!;
+
+    // to retrieve the list of repositories
+    // and a second time to ask for the language
+    expect(showQuickPickSpy).to.have.been.calledTwice;
+
+    // check a few files that we know should exist and others that we know should not
+
+    // the tarball to deliver to the server
+    printDirectoryContents(queryPackRootDir);
+    expect(fs.readdirSync(queryPackRootDir).find(f => f.startsWith('qlpack-') && f.endsWith('-generated.tgz'))).not.to.be.undefined;
+
+    const queryPackDir = path.join(queryPackRootDir, 'query-pack');
+    printDirectoryContents(queryPackDir);
+    // in-pack.ql renamed to query.ql
+    expect(fs.existsSync(path.join(queryPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(queryPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(queryPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(queryPackDir, 'lib.qll'))).to.be.false;
+    expect(fs.existsSync(path.join(queryPackDir, 'not-in-pack.ql'))).to.be.false;
+
+    // the compiled pack
+    const compiledPackDir = path.join(queryPackDir, '.codeql/pack/codeql-remote/query/0.0.0/');
+    printDirectoryContents(compiledPackDir);
+    expect(fs.existsSync(path.join(compiledPackDir, 'query.ql'))).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'qlpack.yml'))).to.be.true;
+    // depending on the cli version, we should have one of these files
+    expect(
+      fs.existsSync(path.join(compiledPackDir, 'qlpack.lock.yml')) ||
+      fs.existsSync(path.join(compiledPackDir, 'codeql-pack.lock.yml'))
+    ).to.be.true;
+    expect(fs.existsSync(path.join(compiledPackDir, 'lib.qll'))).to.be.false;
+    expect(fs.existsSync(path.join(compiledPackDir, 'not-in-pack.ql'))).to.be.false;
+    // should have generated a correct qlpack file
+    const qlpackContents: any = yaml.safeLoad(fs.readFileSync(path.join(compiledPackDir, 'qlpack.yml'), 'utf8'));
+    expect(qlpackContents.name).to.equal('codeql-remote/query');
+    expect(qlpackContents.version).to.equal('0.0.0');
+    expect(qlpackContents.dependencies?.['codeql/javascript-all']).to.equal('*');
+
+    // dependencies
+    const libraryDir = path.join(compiledPackDir, '.codeql/libraries/codeql');
+    printDirectoryContents(libraryDir);
+    const packNames = fs.readdirSync(libraryDir).sort();
+    expect(packNames).to.deep.equal(['javascript-all', 'javascript-upgrades']);
+  });
+
+  it('should cancel a run before uploading', async () => {
+    const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
+
+    const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+
+    token.isCancellationRequested = true;
+
+    try {
+      await promise;
+      assert.fail('should have thrown');
+    } catch (e) {
+      expect(e).to.be.instanceof(UserCancellationException);
+    }
+  });
+
+  function getFile(file: string): Uri {
+    return Uri.file(path.join(baseDir, file));
+  }
+
+  function printDirectoryContents(dir: string) {
+    console.log(`DIR ${dir}`);
+    if (!fs.existsSync(dir)) {
+      console.log(`DIR ${dir} does not exist`);
+    }
+    fs.readdirSync(dir).sort().forEach(f => console.log(`  ${f}`));
+  }
+});

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -6,10 +6,15 @@ import { workspace } from 'vscode';
 
 /**
  * This module ensures that the proper CLI is available for tests of the extension.
- * There are two environment variables to control this module:
+ * There are three environment variables to control this module:
  *
  *     - CLI_VERSION: The version of the CLI to install. Defaults to the most recent
  *       version. Note that for now, we must maintain the default version by hand.
+ *       This may be set to `nightly`, in which case the `NIGHTLY_URL` variable must
+ *       also be set.
+ *
+ *     - NIGHTLY_URL: The URL for a nightly release of the CodeQL CLI that will be
+ *       used if `CLI_VERSION` is set to `nightly`.
  *
  *     - CLI_BASE_DIR: The base dir where the CLI will be downloaded and unzipped.
  *       The download location is `${CLI_BASE_DIR}/assets` and the unzip loction is
@@ -39,7 +44,7 @@ const _10MB = _1MB * 10;
 
 // CLI version to test. Hard code the latest as default. And be sure
 // to update the env if it is not otherwise set.
-const CLI_VERSION = process.env.CLI_VERSION || 'v2.4.2';
+const CLI_VERSION = process.env.CLI_VERSION || 'v2.7.2';
 process.env.CLI_VERSION = CLI_VERSION;
 
 // Base dir where CLIs will be downloaded into
@@ -53,6 +58,15 @@ export async function ensureCli(useCli: boolean) {
       return;
     }
 
+    if ('CLI_PATH' in process.env) {
+      const executablePath = process.env.CLI_PATH;
+      console.log(`Using existing CLI at ${executablePath}`);
+
+      // The CLI_VERSION env variable is not used when the CLI_PATH is set.
+      delete process.env.CLI_VERSION;
+      return;
+    }
+
     const assetName = DistributionManager.getRequiredAssetName();
     const url = getCliDownloadUrl(assetName);
     const unzipDir = getCliUnzipDir();
@@ -133,6 +147,11 @@ export function skipIfNoCodeQL(context: Mocha.Context) {
  * Url to download from
  */
 function getCliDownloadUrl(assetName: string) {
+  if (CLI_VERSION === 'nightly') {
+    if (!process.env.NIGHTLY_URL)
+      throw new Error('Nightly CLI was specified but no URL to download it from was given!');
+    return process.env.NIGHTLY_URL + `/${assetName}`;
+  }
   return `https://github.com/github/codeql-cli-binaries/releases/download/${CLI_VERSION}/${assetName}`;
 }
 

extensions/ql-vscode/src/vscode-tests/index-template.ts

@@ -2,6 +2,7 @@ import * as path from 'path';
 import * as Mocha from 'mocha';
 import * as glob from 'glob';
 import { ensureCli } from './ensureCli';
+import { env } from 'vscode';
 
 
 // Use this handler to avoid swallowing unhandled rejections.
@@ -42,8 +43,17 @@ export async function runTestsInDirectory(testsRoot: string, useCli = false): Pr
   // Create the mocha test
   const mocha = new Mocha({
     ui: 'bdd',
-    color: true
-  });
+    color: true,
+    globalSetup: [],
+    globalTeardown: [],
+  } as any);
+
+  (mocha.options as any).globalSetup.push(
+    // convert this function into an noop since it should not run during tests.
+    // If it does run during tests, then it can cause some testing environments
+    // to hang.
+    (env as any).openExternal = () => { /**/ }
+  );
 
   await ensureCli(useCli);
 
@@ -69,6 +79,7 @@ export async function runTestsInDirectory(testsRoot: string, useCli = false): Pr
           .filter(f => f.endsWith('.helper.js'))
           .forEach(f => {
             console.log(`Executing helper ${f}`);
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
             const helper = require(path.resolve(testsRoot, f)).default;
             helper(mocha);
           });

extensions/ql-vscode/src/vscode-tests/minimal-workspace/config.test.ts

@@ -15,7 +15,10 @@ import {
 chai.use(chaiAsPromised);
 const expect = chai.expect;
 
-describe('config listeners', () => {
+describe('config listeners', function() {
+  // Because we are adding some extra waiting, need to bump the test timeouts.
+  this.timeout(5000);
+
   let sandbox: Sinon.SinonSandbox;
   beforeEach(() => {
     sandbox = Sinon.createSandbox();
@@ -26,7 +29,7 @@ describe('config listeners', () => {
   });
 
   interface TestConfig<T> {
-    clazz: new() => {};
+    clazz: new () => unknown;
     settings: {
       name: string;
       property: string;
@@ -38,9 +41,17 @@ describe('config listeners', () => {
     {
       clazz: CliConfigListener,
       settings: [{
+        name: 'codeQL.runningQueries.numberOfThreads',
+        property: 'numberThreads',
+        values: [0, 1]
+      }, {
         name: 'codeQL.runningTests.numberOfThreads',
         property: 'numberTestThreads',
         values: [1, 0]
+      }, {
+        name: 'codeQL.runningQueries.maxPaths',
+        property: 'maxPaths',
+        values: [0, 1]
       }]
     },
     {
@@ -57,6 +68,14 @@ describe('config listeners', () => {
         name: 'codeQL.runningQueries.numberOfThreads',
         property: 'numThreads',
         values: [0, 1]
+      }, {
+        name: 'codeQL.runningQueries.saveCache',
+        property: 'saveCache',
+        values: [false, true]
+      }, {
+        name: 'codeQL.runningQueries.cacheSize',
+        property: 'cacheSize',
+        values: [0, 1]
       }, {
         name: 'codeQL.runningQueries.memory',
         property: 'queryMemoryMb',
@@ -84,19 +103,31 @@ describe('config listeners', () => {
         beforeEach(async () => {
           origValue = workspace.getConfiguration().get(setting.name);
           await workspace.getConfiguration().update(setting.name, setting.values[0]);
+          await wait();
           spy.resetHistory();
         });
 
         afterEach(async () => {
           await workspace.getConfiguration().update(setting.name, origValue);
+          await wait();
         });
 
         it(`should listen for changes to '${setting.name}'`, async () => {
           await workspace.getConfiguration().update(setting.name, setting.values[1]);
-          expect(spy.calledOnce).to.be.true;
+          await wait();
           expect(listener[setting.property]).to.eq(setting.values[1]);
+          expect(spy).to.have.been.calledOnce;
         });
       });
     });
   });
+
+  // Need to wait some time since the onDidChangeConfiguration listeners fire
+  // asynchronously and we sometimes need to wait for them to complete in
+  // order to have as successful test.
+  async function wait(ms = 50) {
+    return new Promise(resolve =>
+      setTimeout(resolve, ms)
+    );
+  }
 });

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -20,6 +20,7 @@ import { registerDatabases } from '../../pure/messages';
 import { ProgressCallback } from '../../commandRunner';
 import { CodeQLCliServer } from '../../cli';
 import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 describe('databases', () => {
 
@@ -66,11 +67,13 @@ describe('databases', () => {
       } as unknown as ExtensionContext,
       {
         sendRequest: sendRequestSpy,
-        supportsDatabaseRegistration: supportsDatabaseRegistrationSpy,
         onDidStartQueryServer: () => { /**/ }
       } as unknown as QueryServerClient,
       {
-        supportsLanguageName: supportsLanguageNameSpy,
+        cliConstraints: {
+          supportsLanguageName: supportsLanguageNameSpy,
+          supportsDatabaseRegistration: supportsDatabaseRegistrationSpy,
+        },
         resolveDatabase: resolveDatabaseSpy
       } as unknown as CodeQLCliServer,
       {} as Logger,
@@ -85,7 +88,7 @@ describe('databases', () => {
 
   afterEach(async () => {
     dir.removeCallback();
-    databaseManager.dispose();
+    databaseManager.dispose(testDisposeHandler);
     sandbox.restore();
   });
 
@@ -176,7 +179,7 @@ describe('databases', () => {
       expect(spy).to.have.been.calledWith(mockEvent);
     });
 
-    it('should add a database item source archive', async function() {
+    it('should add a database item source archive', async function () {
       const mockDbItem = createMockDB();
       mockDbItem.name = 'xxx';
       await (databaseManager as any).addDatabaseSourceArchiveFolder(mockDbItem);
@@ -411,6 +414,72 @@ describe('databases', () => {
     expect(result).to.eq('');
   });
 
+  describe('isAffectedByTest', () => {
+    const directoryStats = new fs.Stats();
+    const fileStats = new fs.Stats();
+
+    before(() => {
+      sinon.stub(directoryStats, 'isDirectory').returns(true);
+      sinon.stub(fileStats, 'isDirectory').returns(false);
+    });
+
+    it('should return true for testproj database in test directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.true;
+    });
+
+    it('should return false for non-existent test directory', async () => {
+      sandbox.stub(fs, 'stat').throws('Simulated Error: ENOENT');
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.false;
+    });
+
+    it('should return false for non-testproj database in test directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.proj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.false;
+    });
+
+    it('should return false for testproj database outside test directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/other/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir')).to.false;
+    });
+
+    it('should return false for testproj database for prefix directory', async () => {
+      sandbox.stub(fs, 'stat').resolves(directoryStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      // /path/to/d is a prefix of /path/to/dir/dir.testproj, but
+      // /path/to/dir/dir.testproj is not under /path/to/d
+      expect(await db.isAffectedByTest('/path/to/d')).to.false;
+    });
+
+    it('should return true for testproj database for test file', async () => {
+      sandbox.stub(fs, 'stat').resolves(fileStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir/test.ql')).to.true;
+    });
+
+    it('should return false for non-existent test file', async () => {
+      sandbox.stub(fs, 'stat').throws('Simulated Error: ENOENT');
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/dir/test.ql')).to.false;
+    });
+
+    it('should return false for non-testproj database for test file', async () => {
+      sandbox.stub(fs, 'stat').resolves(fileStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.proj'));
+      expect(await db.isAffectedByTest('/path/to/dir/test.ql')).to.false;
+    });
+
+    it('should return false for testproj database not matching test file', async () => {
+      sandbox.stub(fs, 'stat').resolves(fileStats);
+      const db = createMockDB(sourceLocationUri(), Uri.file('/path/to/dir/dir.testproj'));
+      expect(await db.isAffectedByTest('/path/to/test.ql')).to.false;
+    });
+  });
+
   function createMockDB(
     // source archive location must be a real(-ish) location since
     // tests will add this to the workspace location

extensions/ql-vscode/src/vscode-tests/no-workspace/astViewer.test.ts

@@ -7,6 +7,7 @@ import * as yaml from 'js-yaml';
 import { AstViewer, AstItem } from '../../astViewer';
 import { commands, Range } from 'vscode';
 import { DatabaseItem } from '../../databases';
+import { testDisposeHandler } from '../test-dispose-handler';
 
 chai.use(chaiAsPromised);
 const expect = chai.expect;
@@ -31,7 +32,7 @@ describe('AstViewer', () => {
   afterEach(() => {
     sandbox.restore();
     if (viewer) {
-      viewer.dispose();
+      viewer.dispose(testDisposeHandler);
       viewer = undefined;
     }
   });

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/astBuilder.test.ts

@@ -19,24 +19,23 @@ This test uses an AST generated from this file (already BQRS-decoded in ../data/
 
 int interrupt_init(void)
 {
-	return 0;
+    return 0;
 }
 
 void enable_interrupts(void)
 {
-	return;
+    return;
 }
 int disable_interrupts(void)
 {
-	return 0;
+    return 0;
 }
  */
 
 
 describe('AstBuilder', () => {
   let mockCli: CodeQLCliServer;
-  let overrides: Record<string, object | undefined>;
-
+  let overrides: Record<string, Record<string, unknown> | undefined>;
 
   beforeEach(() => {
     mockCli = {
@@ -136,7 +135,7 @@ describe('AstBuilder', () => {
     };
 
     const astBuilder = createAstBuilder();
-    expect(astBuilder.getRoots()).to.be.rejectedWith('AST is invalid');
+    await expect(astBuilder.getRoots()).to.be.rejectedWith('AST is invalid');
   });
 
   function createAstBuilder() {

extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts

@@ -18,28 +18,48 @@ describe('queryResolver', () => {
   let writeFileSpy: sinon.SinonSpy;
   let getQlPackForDbschemeSpy: sinon.SinonStub;
   let getPrimaryDbschemeSpy: sinon.SinonStub;
-  let mockCli: Record<string, sinon.SinonStub>;
+  let mockCli: Record<string, sinon.SinonStub | Record<string, sinon.SinonStub>>;
   beforeEach(() => {
     mockCli = {
-      resolveQueriesInSuite: sinon.stub()
+      resolveQueriesInSuite: sinon.stub(),
+      cliConstraints: {
+        supportsAllowLibraryPacksInResolveQueries: sinon.stub().returns(true),
+      }
     };
     module = createModule();
   });
 
   describe('resolveQueries', () => {
-
     it('should resolve a query', async () => {
       mockCli.resolveQueriesInSuite.returns(['a', 'b']);
-      const result = await module.resolveQueries(mockCli, 'my-qlpack', KeyType.DefinitionQuery);
+      const result = await module.resolveQueries(mockCli, { dbschemePack: 'my-qlpack' }, KeyType.DefinitionQuery);
       expect(result).to.deep.equal(['a', 'b']);
       expect(writeFileSpy.getCall(0).args[0]).to.match(/.qls$/);
-      expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal({
-        qlpack: 'my-qlpack',
+      expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal([{
+        from: 'my-qlpack',
+        queries: '.',
         include: {
           kind: 'definitions',
           'tags contain': 'ide-contextual-queries/local-definitions'
         }
-      });
+      }]);
+    });
+
+    it('should resolve a query from the queries pack if this is an old CLI', async () => {
+      // pretend this is an older CLI
+      (mockCli.cliConstraints as any).supportsAllowLibraryPacksInResolveQueries.returns(false);
+      mockCli.resolveQueriesInSuite.returns(['a', 'b']);
+      const result = await module.resolveQueries(mockCli, { dbschemePackIsLibraryPack: true, dbschemePack: 'my-qlpack', queryPack: 'my-qlpack2' }, KeyType.DefinitionQuery);
+      expect(result).to.deep.equal(['a', 'b']);
+      expect(writeFileSpy.getCall(0).args[0]).to.match(/.qls$/);
+      expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal([{
+        from: 'my-qlpack2',
+        queries: '.',
+        include: {
+          kind: 'definitions',
+          'tags contain': 'ide-contextual-queries/local-definitions'
+        }
+      }]);
     });
 
     it('should throw an error when there are no queries found', async () => {
@@ -48,12 +68,12 @@ describe('queryResolver', () => {
       // TODO: Figure out why chai-as-promised isn't failing the test on an
       // unhandled rejection.
       try {
-        await module.resolveQueries(mockCli, 'my-qlpack', KeyType.DefinitionQuery);
+        await module.resolveQueries(mockCli, { dbschemePack: 'my-qlpack' }, KeyType.DefinitionQuery);
         // should reject
         expect(true).to.be.false;
       } catch (e) {
         expect(e.message).to.eq(
-          'Couldn\'t find any queries tagged ide-contextual-queries/local-definitions for qlpack my-qlpack'
+          'Couldn\'t find any queries tagged ide-contextual-queries/local-definitions in any of the following packs: my-qlpack.'
         );
       }
     });

extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif

@@ -0,0 +1,34 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ],
+      "results": []
+    }
+  ]
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/invalidSarif.sarif

@@ -0,0 +1,33 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ]
+    }
+  ]
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/validSarif.sarif

@@ -0,0 +1,57 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ],
+      "results": [
+        {
+          "level": "error",
+          "message": {
+            "text": "'x' is assigned a value but never used."
+          },
+          "locations": [
+            {
+              "physicalLocation": {
+                "artifactLocation": {
+                  "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js",
+                  "index": 0
+                },
+                "region": {
+                  "startLine": 1,
+                  "startColumn": 5
+                }
+              }
+            }
+          ],
+          "ruleId": "no-unused-vars",
+          "ruleIndex": 0
+        }
+      ]
+    }
+  ]
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -13,19 +13,23 @@ import {
   looksLikeLgtmUrl,
   findDirWithFile,
 } from '../../databaseFetcher';
+import { ProgressCallback } from '../../commandRunner';
 chai.use(chaiAsPromised);
 const expect = chai.expect;
 
-describe('databaseFetcher', function() {
+describe('databaseFetcher', function () {
   // These tests make API calls and may need extra time to complete.
   this.timeout(10000);
 
   describe('convertToDatabaseUrl', () => {
     let sandbox: sinon.SinonSandbox;
     let quickPickSpy: sinon.SinonStub;
+    let progressSpy: ProgressCallback;
+
     beforeEach(() => {
       sandbox = sinon.createSandbox();
       quickPickSpy = sandbox.stub(window, 'showQuickPick');
+      progressSpy = sandbox.spy();
     });
 
     afterEach(() => {
@@ -35,7 +39,7 @@ describe('databaseFetcher', function() {
     it('should convert a project url to a database url', async () => {
       quickPickSpy.resolves('javascript');
       const lgtmUrl = 'https://lgtm.com/projects/g/github/codeql';
-      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl, progressSpy);
 
       expect(dbUrl).to.equal(
         'https://lgtm.com/api/v1.0/snapshots/1506465042581/javascript'
@@ -48,17 +52,31 @@ describe('databaseFetcher', function() {
       quickPickSpy.resolves('python');
       const lgtmUrl =
         'https://lgtm.com/projects/g/github/codeql/subpage/subpage2?query=xxx';
-      const dbUrl = await convertToDatabaseUrl(lgtmUrl);
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl, progressSpy);
 
       expect(dbUrl).to.equal(
         'https://lgtm.com/api/v1.0/snapshots/1506465042581/python'
       );
+      expect(progressSpy).to.have.been.calledOnce;
     });
 
-    it('should fail on a nonexistant prohect', async () => {
+    it('should convert a raw slug to a database url with extra path segments', async () => {
+      quickPickSpy.resolves('python');
+      const lgtmUrl =
+        'g/github/codeql';
+      const dbUrl = await convertToDatabaseUrl(lgtmUrl, progressSpy);
+
+      expect(dbUrl).to.equal(
+        'https://lgtm.com/api/v1.0/snapshots/1506465042581/python'
+      );
+      expect(progressSpy).to.have.been.calledOnce;
+    });
+
+    it('should fail on a nonexistent project', async () => {
       quickPickSpy.resolves('javascript');
       const lgtmUrl = 'https://lgtm.com/projects/g/github/hucairz';
-      expect(convertToDatabaseUrl(lgtmUrl)).to.rejectedWith(/Invalid LGTM URL/);
+      await expect(convertToDatabaseUrl(lgtmUrl, progressSpy)).to.rejectedWith(/Invalid LGTM URL/);
+      expect(progressSpy).to.have.callCount(0);
     });
   });
 
@@ -71,6 +89,10 @@ describe('databaseFetcher', function() {
         .to.be.false;
       expect(looksLikeLgtmUrl('https://ww.lgtm.com/projects/g/github')).to.be
         .false;
+      expect(looksLikeLgtmUrl('g/github')).to.be
+        .false;
+      expect(looksLikeLgtmUrl('ggg/github/myproj')).to.be
+        .false;
     });
 
     it('should handle valid urls', () => {
@@ -86,6 +108,10 @@ describe('databaseFetcher', function() {
           'https://lgtm.com/projects/g/github/codeql/sub/pages?query=string'
         )
       ).to.be.true;
+      expect(looksLikeLgtmUrl('g/github/myproj')).to.be
+        .true;
+      expect(looksLikeLgtmUrl('git/github/myproj')).to.be
+        .true;
     });
   });