Merge pull request #1736 from github/v1.7.5

v1.7.5

.gitattributes

@@ -18,4 +18,8 @@ yarn.lock                    merge=binary
 # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/gitattributes.html
 # suggests that this might interleave lines arbitrarily, but empirically
 # it keeps added chunks contiguous
-CHANGELOG.md                merge=union
+CHANGELOG.md                merge=union
+
+# Mark some JSON files containing test data as generated so they are not included
+# as part of diffs or language statistics.
+extensions/ql-vscode/src/stories/remote-queries/data/*.json     linguist-generated

.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "extensions/ql-vscode"
+    schedule:
+      interval: "weekly"
+      day: "thursday" # Thursday is arbitrary
+    labels:
+      - "Update dependencies"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-minor", "version-update:semver-patch"]
+  - package-ecosystem: "github-actions"
+    directory: ".github"
+    schedule:
+      interval: "weekly"
+      day: "thursday" # Thursday is arbitrary
+    labels:
+      - "Update dependencies"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-minor", "version-update:semver-patch"]

.github/workflows/codeql.yml

@@ -19,13 +19,14 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@main
         with:
             languages: javascript
             config-file: ./.github/codeql/codeql-config.yml
+            tools: latest
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@main

.github/workflows/dependency-review.yml

@@ -0,0 +1,16 @@
+name: 'Dependency Review'
+on:
+  - pull_request
+  - workflow_dispatch
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v1

.github/workflows/main.yml

@@ -16,13 +16,13 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
-          node-version: '14.14.0'
+          node-version: '16.14.2'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -65,7 +65,7 @@ jobs:
         # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
         run: |
           LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
-          echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST"
+          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
 
   test:
     name: Test
@@ -76,13 +76,13 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
-          node-version: '14.14.0'
+          node-version: '16.14.2'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -103,6 +103,11 @@ jobs:
         run: |
           npm run lint
 
+      - name: Lint scenarios
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run lint:scenarios
+
       - name: Run unit tests (Linux)
         working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
@@ -118,13 +123,17 @@ jobs:
       - name: Run integration tests (Linux)
         if: matrix.os == 'ubuntu-latest'
         working-directory: extensions/ql-vscode
+        env:
+          VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
-          sudo apt-get install xvfb
+          unset DBUS_SESSION_BUS_ADDRESS
           /usr/bin/xvfb-run npm run integration
 
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
         working-directory: extensions/ql-vscode
+        env:
+          VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
           npm run integration
 
@@ -135,7 +144,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.6', 'nightly']
+        version: ['v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.2', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
       NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
@@ -143,11 +152,11 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
-          node-version: '14.14.0'
+          node-version: '16.14.2'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -168,16 +177,13 @@ jobs:
           if [[ "${{ matrix.version }}" == "nightly" ]]
           then
             REF="codeql-cli/latest"
-          elif [[ "${{ matrix.version }}" == "v2.2.6" || "${{ matrix.version }}" == "v2.3.3" ]]
-          then
-            REF="codeql-cli/v2.4.5"
           else
             REF="codeql-cli/${{ matrix.version }}"
           fi
-          echo "::set-output name=ref::$REF"
+          echo "ref=$REF" >> "$GITHUB_OUTPUT"
 
       - name: Checkout QL
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: github/codeql
           ref: ${{ steps.choose-ref.outputs.ref }}
@@ -187,6 +193,7 @@ jobs:
         working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
+          unset DBUS_SESSION_BUS_ADDRESS
           /usr/bin/xvfb-run npm run cli-integration
 
       - name: Run CLI tests (Windows)

.github/workflows/release.yml

@@ -18,11 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
-          node-version: '10.18.1'
+          node-version: '16.14.2'
 
       - name: Install dependencies
         run: |
@@ -47,11 +47,11 @@ jobs:
           # Record the VSIX path as an output of this step.
           # This will be used later when uploading a release asset.
           VSIX_PATH="$(ls dist/*.vsix)"
-          echo "::set-output name=vsix_path::$VSIX_PATH"
+          echo "vsix_path=$VSIX_PATH" >> "$GITHUB_OUTPUT"
           # Transform the GitHub ref so it can be used in a filename.
           # The last sed invocation is used for testing branches that modify this workflow.
           REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
-          echo "::set-output name=ref_name::$REF_NAME"
+          echo "ref_name=$REF_NAME" >> "$GITHUB_OUTPUT"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
@@ -107,7 +107,7 @@ jobs:
           # Bump to the next patch version. Major or minor version bumps will have to be done manually.
           # Record the next version number as an output of this step.
           NEXT_VERSION="$(npm version patch)"
-          echo "::set-output name=next_version::$NEXT_VERSION"
+          echo "next_version=$NEXT_VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Add changelog for next release
         if: success()
@@ -136,7 +136,7 @@ jobs:
       VSCE_TOKEN: ${{ secrets.VSCE_TOKEN }}
     steps:
       - name: Download artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: vscode-codeql-extension
 
@@ -156,7 +156,7 @@ jobs:
       OPEN_VSX_TOKEN: ${{ secrets.OPEN_VSX_TOKEN }}
     steps:
       - name: Download artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: vscode-codeql-extension
 

.vscode/launch.json

@@ -12,7 +12,6 @@
         // Add a reference to a workspace to open. Eg-
         // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -36,17 +35,31 @@
       "runtimeArgs": [
         "--inspect=9229"
       ],
+      "env": {
+        "LANG": "en-US"
+      },
       "args": [
         "--exit",
         "-u",
         "bdd",
         "--colors",
         "--diff",
-        "-r",
-        "ts-node/register",
+        "--config",
+        ".mocharc.json",
         "test/pure-tests/**/*.ts"
       ],
-      "port": 9229,
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
+    },
+    {
+      "name": "Launch Unit Tests - React (vscode-codeql)",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/extensions/ql-vscode/node_modules/jest/bin/jest.js",
+      "showAsyncStacks": true,
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
       "stopOnEntry": false,
       "sourceMaps": true,
       "console": "integratedTerminal",
@@ -60,10 +73,10 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
+        "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -77,11 +90,11 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
+        "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu",
         "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -95,6 +108,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "--disable-workspace-trust",
         "--disable-gpu",
         "--disable-extension",
         "eamodio.gitlens",
@@ -121,11 +135,18 @@
         // This option overrides the CLI_VERSION option.
         // "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
       },
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
+    },
+    {
+      "name": "Launch Storybook",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["run-script", "storybook"]
     }
   ]
 }

.vscode/settings.json

@@ -30,12 +30,11 @@
     "typescript",
     "typescriptreact"
   ],
-  "eslint.options": {
-    // This is necessary so that eslint can properly resolve its plugins
-    "resolvePluginsRelativeTo": "./extensions/ql-vscode"
-  },
+  // This is necessary to ensure that ESLint can find the correct configuration files and plugins.
+  "eslint.workingDirectories": ["./extensions/ql-vscode"],
   "editor.formatOnSave": false,
   "typescript.preferences.quoteStyle": "single",
   "javascript.preferences.quoteStyle": "single",
-  "editor.wordWrapColumn": 100
+  "editor.wordWrapColumn": 100,
+  "jest.rootPath": "./extensions/ql-vscode"
 }

CODEOWNERS

@@ -1 +1,4 @@
 **/* @github/codeql-vscode-reviewers
+**/remote-queries/ @github/code-scanning-secexp-reviewers
+**/variant-analysis/ @github/code-scanning-secexp-reviewers
+**/databases/ @github/code-scanning-secexp-reviewers

CONTRIBUTING.md

@@ -29,7 +29,9 @@ Here are a few things you can do that will increase the likelihood of your pull
 
 ## Setting up a local build
 
-Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
+Make sure you have installed recent versions of vscode, node, and npm. Check the `engines` block in [`package.json`](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/package.json) file for compatible versions. Earlier versions may work, but we no longer test against them.
+
+To automatically switch to the correct version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm), which will pick-up the node version from `.nvmrc`.
 
 ### Installing all packages
 
@@ -56,8 +58,6 @@ We recommend that you keep `npm run watch` running in the backgound and you only
 
 1. on first checkout
 2. whenever any of the non-TypeScript resources have changed
-3. on any change to files included in one of the webviews
-   - **Important**: This is easy to forget. You must explicitly run `npm run build` whenever one of the files in the webview is changed. These are the files in the `src/view` and `src/compare/view` folders.
 
 ### Installing the extension
 
@@ -77,33 +77,131 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
-### Running the unit tests and integration tests that do not require a CLI instance
+### Storybook
 
-Unit tests and many integration tests do not require a copy of the CodeQL CLI.
-
-Outside of vscode, in the `extensions/ql-vscode` directory, run:
+You can use [Storybook](https://storybook.js.org/) to preview React components outside VSCode. Inside the `extensions/ql-vscode` directory, run:
 
 ```shell
-npm run test && npm run integration
+npm run storybook
 ```
 
-Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
+Your browser should automatically open to the Storybook UI. Stories live in the `src/stories` directory.
 
-Only the _With CLI_ tests require a CLI instance to run. See below on how to do that.
+Alternatively, you can start Storybook inside of VSCode. There is a VSCode launch configuration for starting Storybook. It can be found in the debug view.
 
-Running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+More information about Storybook can be found inside the **Overview** page once you have launched Storybook.
 
-### Running the integration tests
+### Testing
 
-The _Launch Integration Tests - With CLI_ tests require a CLI instance in order to run. There are several environment variables you can use to configure this.
+We have several types of tests:
 
-From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ uncomment and change the environment variables appropriate for your purpose.
+* Unit tests: these live in the `tests/pure-tests/` directory
+* View tests: these live in `src/view/variant-analysis/__tests__/`
+* VSCode integration tests: these live in `src/vscode-tests/no-workspace` and `src/vscode-tests/minimal-workspace`
+* CLI integration tests: these live in `src/vscode-tests/cli-integration`
+
+The CLI integration tests require an instance of the CodeQL CLI to run so they will require some extra setup steps. When adding new tests to our test suite, please be mindful of whether they need to be in the cli-integration folder. If the tests don't depend on the CLI, they are better suited to being a VSCode integration test.
+
+Any test data you're using (sample projects, config files, etc.) must go in a `src/vscode-tests/*/data` directory. When you run the tests, the test runner will copy the data directory to `out/vscode-tests/*/data`.
+
+#### Running the tests
+
+Pre-requisites:
+1. Run `npm run build`.
+2. You will need to have `npm run watch` running in the background.
+
+##### 1. From the terminal
+
+Then, from the `extensions/ql-vscode` directory, use the appropriate command to run the tests:
+
+* Unit tests: `npm run test:unit`
+* View Tests: `npm test:view`
+* VSCode integration tests: `npm run integration`
+
+###### CLI integration tests
+
+The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
+
+1. Set the `TEST_CODEQL_PATH` environment variable: running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+2. Run your test command:
+
+```shell
+cd extensions/ql-vscode && npm run cli-integration
+```
+
+##### 2. From VSCode
+
+Alternatively, you can run the tests inside of VSCode. There are several VSCode launch configurations defined that run the unit and integration tests.
+
+You will need to run tests using a task from inside of VS Code, under the "Run and Debug" view:
+
+* Unit tests: run the _Launch Unit Tests - React_ task
+* View Tests: run the _Launch Unit Tests_ task
+* VSCode integration tests: run the _Launch Unit Tests - No Workspace_ and _Launch Unit Tests - Minimal Workspace_ tasks
+
+###### CLI integration tests
+
+The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
+
+1. Set the `TEST_CODEQL_PATH` environment variable: running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+2. Set the codeql path in VSCode's launch configuration: open `launch.json` and under the _Launch Integration Tests - With CLI_ section, uncomment the `"${workspaceRoot}/../codeql"` line. If you've cloned the `github/codeql` repo to a different path, replace the value with the correct path.
+
+3. Run the VSCode task from the "Run and Debug" view called _Launch Integration Tests - With CLI_.
+
+#### Using a mock GitHub API server
+
+Multi-Repo Variant Analyses (MRVA) rely on the GitHub API. In order to make development and testing easy, we have functionality that allows us to intercept requests to the GitHub API and provide mock responses. 
+
+##### Using a pre-recorded test scenario
+
+To run a mock MRVA scenario, follow these steps:
+1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
+```json
+"codeQL.mockGitHubApiServer": {
+  "enabled": true
+}
+```
+
+1. Run the `CodeQL: Mock GitHub API Server: Load Scenario` command from the command pallet, and choose one of the scenarios to load.
+1. Execute a normal MRVA. At this point you should see the scenario being played out, rather than an actual MRVA running.
+1. Once you're done, you can stop using the mock scenario with `CodeQL: Mock GitHub API Server: Unload Scenario`
+
+If you want to replay the same scenario you should unload and reload it so requests are replayed from the start.
+
+##### Recording a new test scenario
+To record a new mock MRVA scenario, follow these steps:
+
+1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
+```json
+"codeQL.mockGitHubApiServer": {
+  "enabled": true
+}
+```
+
+1. Run the `CodeQL: Mock GitHub API Server: Start Scenario Recording` VS Code command from the command pallet.
+1. Execute a normal MRVA.
+1. Once what you wanted to record is done (e.g. the MRVA has finished), then run the `CodeQL: Mock GitHub API Server: Save Scenario` command from the command pallet.
+1. The scenario should then be available for replaying.
+
+If you want to cancel recording, run the `CodeQL: Mock GitHub API Server: Cancel Scenario Recording` command.
+
+Once the scenario has been recorded, it's often useful to remove some of the requests to speed up the replay, particularly ones that fetch the variant analysis status. Once some of the request files have manually been removed, the [fix-scenario-file-numbering script](./extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts) can be used to update the number of the files. See the script file for details on how to use.
+
+#### Scenario data location
+
+Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.
 
 ## Releasing (write access required)
 
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
+1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
+    * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
+    * `.github/workflows/main.yml` - all the "node-version: <version>" settings
+    * `.github/workflows/release.yml` - the "node-version: <version>" setting
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
 1. Create a PR for this release:
     * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
@@ -111,19 +209,40 @@ From inside of VSCode, open the `launch.json` file and in the _Launch Integratio
     * Create a new commit with a message the same as the branch name.
     * Create a PR for this branch.
     * Wait for the PR to be merged into `main`
-1. Trigger a release build on Actions by adding a new tag on branch `main` named after the release, as above. Note that when you push to upstream, you will need to fully qualify the ref. A command like this will work:
+1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+    ```bash
+    git checkout main
+    git tag v1.3.6
+    ```
+
+   If you've accidentally created a badly named tag, you can delete it via
+    ```bash
+    git tag -d badly-named-tag
+    ```
+1. Push the new tag up:
+
+   a. If you're using a fork of the repo:
 
     ```bash
     git push upstream refs/tags/v1.3.6
     ```
 
+   b. If you're working straight in this repo:
+
+    ```bash
+    git push origin refs/tags/v1.3.6
+    ```
+
+   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
+
     * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
     * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
-
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
+    * DO NOT approve the "publish" stages of the workflow yet.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
+1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
 1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
     - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
 1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
@@ -143,12 +262,7 @@ To regenerate the Open VSX token:
 1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
 1. Update the secret in the `publish-open-vsx` environment in the project settings.
 
-To regenerate the VSCode Marketplace token:
-
-1. Follow the instructions on [getting a PAT for Azure DevOps](https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token).
-1. Update the secret in the `publish-vscode-marketplace` environment in the project settings.
-
-Not that Azure DevOps PATs expire yearly and must be regenerated.
+To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.
 
 ## Resources
 

extensions/ql-vscode/.eslintignore

@@ -0,0 +1,6 @@
+.vscode-test/
+node_modules/
+out/
+
+# Include the Storybook config
+!.storybook

extensions/ql-vscode/.eslintrc.js

@@ -3,14 +3,14 @@ module.exports = {
   parserOptions: {
     ecmaVersion: 2018,
     sourceType: "module",
-    project: ["tsconfig.json", "./src/**/tsconfig.json", "./gulpfile.ts/tsconfig.json"],
+    project: ["tsconfig.json", "./src/**/tsconfig.json", "./gulpfile.ts/tsconfig.json", "./scripts/tsconfig.json", "./.storybook/tsconfig.json"],
   },
   plugins: ["@typescript-eslint"],
   env: {
     node: true,
     es6: true,
   },
-  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended", "plugin:jest-dom/recommended"],
   rules: {
     "@typescript-eslint/no-use-before-define": 0,
     "@typescript-eslint/no-unused-vars": [

extensions/ql-vscode/.mocharc.json

@@ -0,0 +1,4 @@
+{
+  "exit": true,
+  "require": ["test/mocha.setup.js"]
+}

extensions/ql-vscode/.npmrc

@@ -0,0 +1,2 @@
+# Storybook requires this option to be set. See https://github.com/storybookjs/storybook/issues/18298
+legacy-peer-deps=true

extensions/ql-vscode/.nvmrc

@@ -0,0 +1 @@
+v16.14.2

extensions/ql-vscode/.storybook/main.ts

@@ -0,0 +1,20 @@
+import type { StorybookConfig } from '@storybook/core-common';
+
+const config: StorybookConfig = {
+  stories: [
+    '../src/**/*.stories.mdx',
+    '../src/**/*.stories.@(js|jsx|ts|tsx)'
+  ],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    '@storybook/addon-interactions',
+    './vscode-theme-addon/preset.ts',
+  ],
+  framework: '@storybook/react',
+  core: {
+    builder: '@storybook/builder-webpack5'
+  }
+};
+
+module.exports = config;

extensions/ql-vscode/.storybook/manager.ts

@@ -0,0 +1,7 @@
+import { addons } from '@storybook/addons';
+import { themes } from '@storybook/theming';
+
+addons.setConfig({
+  theme: themes.dark,
+  enableShortcuts: false,
+});

extensions/ql-vscode/.storybook/preview.ts

@@ -0,0 +1,31 @@
+import { themes } from '@storybook/theming';
+import { action } from '@storybook/addon-actions';
+
+// Allow all stories/components to use Codicons
+import '@vscode/codicons/dist/codicon.css';
+
+// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
+export const parameters = {
+  // All props starting with `on` will automatically receive an action as a prop
+  actions: { argTypesRegex: '^on[A-Z].*' },
+  // All props matching these names will automatically get the correct control
+  controls: {
+    matchers: {
+      color: /(background|color)$/i,
+      date: /Date$/,
+    },
+  },
+  // Use a dark theme to be aligned with VSCode
+  docs: {
+    theme: themes.dark,
+  },
+  backgrounds: {
+    // The background is injected by our theme CSS files
+    disable: true,
+  }
+};
+
+(window as any).acquireVsCodeApi = () => ({
+  postMessage: action('post-vscode-message'),
+  setState: action('set-vscode-state'),
+});

extensions/ql-vscode/.storybook/tsconfig.json

@@ -4,7 +4,7 @@
     "moduleResolution": "node",
     "target": "es6",
     "outDir": "out",
-    "lib": ["es6", "dom"],
+    "lib": ["ES2021", "dom"],
     "jsx": "react",
     "sourceMap": true,
     "rootDir": "..",
@@ -12,7 +12,8 @@
     "noUnusedLocals": true,
     "noImplicitReturns": true,
     "noFallthroughCasesInSwitch": true,
-    "experimentalDecorators": true
+    "experimentalDecorators": true,
+    "skipLibCheck": true
   },
   "exclude": ["node_modules"]
 }

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -0,0 +1,49 @@
+import * as React from 'react';
+import { FunctionComponent, useCallback } from 'react';
+
+import { useGlobals } from '@storybook/api';
+import { IconButton, Icons, WithTooltip, TooltipLinkList, Link, WithHideFn } from '@storybook/components';
+
+import { themeNames, VSCodeTheme } from './theme';
+
+export const ThemeSelector: FunctionComponent = () => {
+  const [{ vscodeTheme }, updateGlobals] = useGlobals();
+
+  const changeTheme = useCallback((theme: VSCodeTheme) => {
+    updateGlobals({
+      vscodeTheme: theme,
+    });
+  }, [updateGlobals]);
+
+  const createLinks = useCallback((onHide: () => void): Link[] => Object.values(VSCodeTheme).map((theme) => ({
+    id: theme,
+    onClick() {
+      changeTheme(theme);
+      onHide();
+    },
+    title: themeNames[theme],
+    value: theme,
+    active: vscodeTheme === theme,
+  })), [vscodeTheme, changeTheme]);
+
+  return (
+    <WithTooltip
+      placement="top"
+      trigger="click"
+      closeOnClick
+      tooltip={({ onHide }: WithHideFn) => (
+        <TooltipLinkList
+          links={createLinks(onHide)}
+        />
+      )}
+    >
+      <IconButton
+        key="theme"
+        title="Change the theme of the preview"
+        active={vscodeTheme !== VSCodeTheme.Dark}
+      >
+        <Icons icon="dashboard" />
+      </IconButton>
+    </WithTooltip>
+  );
+};

extensions/ql-vscode/.storybook/vscode-theme-addon/manager.tsx

@@ -0,0 +1,14 @@
+import * as React from 'react';
+import { addons, types } from '@storybook/addons';
+import { ThemeSelector } from './ThemeSelector';
+
+const ADDON_ID = 'vscode-theme-addon';
+
+addons.register(ADDON_ID, () => {
+  addons.add(ADDON_ID, {
+    title: 'VSCode Themes',
+    type: types.TOOL,
+    match: ({ viewMode }) => !!(viewMode && viewMode.match(/^(story|docs)$/)),
+    render: () => <ThemeSelector />,
+  });
+});

extensions/ql-vscode/.storybook/vscode-theme-addon/preset.ts

@@ -0,0 +1,7 @@
+export function config(entry = []) {
+  return [...entry, require.resolve('./preview.ts')];
+}
+
+export function managerEntries(entry = []) {
+  return [...entry, require.resolve('./manager.tsx')];
+}

extensions/ql-vscode/.storybook/vscode-theme-addon/preview.ts

@@ -0,0 +1,8 @@
+import { withTheme } from './withTheme';
+import { VSCodeTheme } from './theme';
+
+export const decorators = [withTheme];
+
+export const globals = {
+  vscodeTheme: VSCodeTheme.Dark,
+};

extensions/ql-vscode/.storybook/vscode-theme-addon/theme.ts

@@ -0,0 +1,9 @@
+export enum VSCodeTheme {
+  Dark = 'dark',
+  Light = 'light',
+}
+
+export const themeNames: { [key in VSCodeTheme]: string } = {
+  [VSCodeTheme.Dark]: 'Dark+',
+  [VSCodeTheme.Light]: 'Light+',
+};

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -0,0 +1,38 @@
+import { useEffect, useGlobals } from '@storybook/addons';
+import type { AnyFramework, PartialStoryFn as StoryFunction, StoryContext } from '@storybook/csf';
+
+import { VSCodeTheme } from './theme';
+
+const themeFiles: { [key in VSCodeTheme]: string } = {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  [VSCodeTheme.Dark]: require('!file-loader?modules!../../src/stories/vscode-theme-dark.css').default,
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  [VSCodeTheme.Light]: require('!file-loader?modules!../../src/stories/vscode-theme-light.css').default,
+};
+
+export const withTheme = (
+  StoryFn: StoryFunction<AnyFramework>,
+  context: StoryContext<AnyFramework>
+) => {
+  const [{ vscodeTheme }] = useGlobals();
+
+  useEffect(() => {
+    const styleSelectorId =
+      context.viewMode === 'docs'
+        ? `addon-vscode-theme-docs-${context.id}`
+        : 'addon-vscode-theme-theme';
+
+    const theme = Object.values(VSCodeTheme).includes(vscodeTheme) ? vscodeTheme as VSCodeTheme : VSCodeTheme.Dark;
+
+    document.getElementById(styleSelectorId)?.remove();
+
+    const link = document.createElement('link');
+    link.id = styleSelectorId;
+    link.href = themeFiles[theme];
+    link.rel = 'stylesheet';
+
+    document.head.appendChild(link);
+  }, [vscodeTheme]);
+
+  return StoryFn();
+};

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,102 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.7.5 - 8 November 2022
+
+- Fix a bug where the AST Viewer was not working unless the associated CodeQL library pack is in the workspace. [#1735](https://github.com/github/vscode-codeql/pull/1735)
+
+## 1.7.4 - 29 October 2022
+
+No user facing changes.
+
+## 1.7.3 - 28 October 2022
+
+- Fix a bug where databases may be lost if VS Code is restarted while the extension is being started up. [#1638](https://github.com/github/vscode-codeql/pull/1638)
+- Add commands for navigating up, down, left, or right in the result viewer. Previously there were only commands for moving up and down the currently-selected path. We suggest binding keyboard shortcuts to these commands, for navigating the result viewer using the keyboard. [#1568](https://github.com/github/vscode-codeql/pull/1568)
+
+## 1.7.2 - 14 October 2022
+
+- Fix a bug where results created in older versions were thought to be unsuccessful. [#1605](https://github.com/github/vscode-codeql/pull/1605)
+
+## 1.7.1 - 12 October 2022
+
+- Fix a bug where it was not possible to add a database folder if the folder name starts with `db-`. [#1565](https://github.com/github/vscode-codeql/pull/1565)
+- Ensure the results view opens in an editor column beside the currently active editor. [#1557](https://github.com/github/vscode-codeql/pull/1557)
+
+## 1.7.0 - 20 September 2022
+
+- Remove ability to download databases from LGTM. [#1467](https://github.com/github/vscode-codeql/pull/1467)
+- Remove the ability to manually upgrade databases from the context menu on databases. Databases are non-destructively upgraded automatically so for most users this was not needed. For advanced users this is still available in the Command Palette. [#1501](https://github.com/github/vscode-codeql/pull/1501)
+- Always restart the query server after a manual database upgrade. This avoids a bug in the query server where an invalid dbscheme was being retained in memory after an upgrade. [#1519](https://github.com/github/vscode-codeql/pull/1519)
+
+## 1.6.12 - 1 September 2022
+
+- Add ability for users to download databases directly from GitHub. [#1485](https://github.com/github/vscode-codeql/pull/1485)
+- Fix a race condition that could cause a failure to open the evaluator log when running a query. [#1490](https://github.com/github/vscode-codeql/pull/1490)
+- Fix an error when running a query with an older version of the CodeQL CLI. [#1490](https://github.com/github/vscode-codeql/pull/1490)
+
+## 1.6.11 - 25 August 2022
+
+No user facing changes.
+
+## 1.6.10 - 9 August 2022
+
+No user facing changes.
+
+## 1.6.9 - 20 July 2022
+
+No user facing changes.
+
+## 1.6.8 - 29 June 2022
+
+- Fix a bug where quick queries cannot be compiled if the core libraries are not in the workspace. [#1411](https://github.com/github/vscode-codeql/pull/1411)
+- Fix a bug where quick evaluation of library files would display an error message when using CodeQL CLI v2.10.0. [#1412](https://github.com/github/vscode-codeql/pull/1412)
+
+## 1.6.7 - 15 June 2022
+
+- Prints end-of-query evaluator log summaries to the Query Log. [#1349](https://github.com/github/vscode-codeql/pull/1349)
+- Be consistent about casing in Query History menu. [#1369](https://github.com/github/vscode-codeql/pull/1369)
+- Fix quoting string columns in exported CSV results. [#1379](https://github.com/github/vscode-codeql/pull/1379)
+
+## 1.6.6 - 17 May 2022
+
+No user facing changes.
+
+## 1.6.5 - 25 April 2022
+
+- Re-enable publishing to open-vsx. [#1285](https://github.com/github/vscode-codeql/pull/1285)
+
+## 1.6.4 - 6 April 2022
+
+No user facing changes.
+
+## 1.6.3 - 4 April 2022
+
+- Fix a bug where the AST viewer was not synchronizing its selected node when the editor selection changes. [#1230](https://github.com/github/vscode-codeql/pull/1230)
+- Avoid synchronizing the `codeQL.cli.executablePath` setting. [#1252](https://github.com/github/vscode-codeql/pull/1252)
+- Open the directory in the finder/explorer (instead of just highlighting it) when running the "Open query directory" command from the query history view. [#1235](https://github.com/github/vscode-codeql/pull/1235)
+- Ensure query label in the query history view changes are persisted across restarts. [#1235](https://github.com/github/vscode-codeql/pull/1235)
+- Prints end-of-query evaluator log summaries to the Query Server Console. [#1264](https://github.com/github/vscode-codeql/pull/1264)
+
+## 1.6.1 - 17 March 2022
+
+No user facing changes.
+
+## 1.6.0 - 7 March 2022
+
+- Fix a bug where database upgrades could not be resolved if some of the target pack's dependencies are outside of the workspace. [#1138](https://github.com/github/vscode-codeql/pull/1138)
+- Open the query server logs for query errors (instead of the extension log). This will make it easier to track down query errors. [#1158](https://github.com/github/vscode-codeql/pull/1158)
+- Fix a bug where queries took a long time to run if there are no folders in the workspace. [#1157](https://github.com/github/vscode-codeql/pull/1157)
+- [BREAKING CHANGE] The `codeQL.runningQueries.customLogDirectory` setting is deprecated and no longer has any function. Instead, all query log files will be stored in the query history directory, next to the query results. [#1178](https://github.com/github/vscode-codeql/pull/1178)
+- Add a _Open query directory_ command for query items. This command opens the directory containing all artifacts for a query. [#1179](https://github.com/github/vscode-codeql/pull/1179)
+- Add options to display evaluator logs for a given query run. Some information that was previously found in the query server output may now be found here. [#1186](https://github.com/github/vscode-codeql/pull/1186)
+
+## 1.5.11 - 10 February 2022
+
+- Fix a bug where invoking _View AST_ from the file explorer would not view the selected file. Instead it would view the active editor. Also, prevent the _View AST_ from appearing if the current selection includes a directory or multiple files. [#1113](https://github.com/github/vscode-codeql/pull/1113)
+- Add query history items as soon as a query is run, including new icons for each history item. [#1094](https://github.com/github/vscode-codeql/pull/1094)
+- Save query history items across restarts. Items will be saved for 30 days and can be overwritten by setting the `codeQL.queryHistory.ttl` configuration setting. [#1130](https://github.com/github/vscode-codeql/pull/1130)
+- Allow in-progress query items to be cancelled from the query history view. [#1105](https://github.com/github/vscode-codeql/pull/1105)
+
 ## 1.5.10 - 25 January 2022
 
 - Fix a bug where the results view moved column even when it was already visible. [#1070](https://github.com/github/vscode-codeql/pull/1070)
@@ -24,7 +121,7 @@
 - Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
 - Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)
 - Allow case-insensitive project slugs for GitHub repositories when adding a CodeQL database from LGTM. [#978](https://github.com/github/vscode-codeql/pull/961)
-- Add a _CodeQL: Preview Query Help_ command to generate Markdown previews of `.qhelp` query help files. This command should only be run in trusted workspaces. See https://codeql.github.com/docs/codeql-cli/testing-query-help-files for more information about query help. [#988](https://github.com/github/vscode-codeql/pull/988)
+- Add a _CodeQL: Preview Query Help_ command to generate Markdown previews of `.qhelp` query help files. This command should only be run in trusted workspaces. See [the CodeQL CLI docs](https://codeql.github.com/docs/codeql-cli/testing-query-help-files) for more information about query help. [#988](https://github.com/github/vscode-codeql/pull/988)
 - Make "Open Referenced File" command accessible from the active editor menu. [#989](https://github.com/github/vscode-codeql/pull/989)
 - Fix a bug where result set names in the result set drop-down were disappearing when viewing a sorted table. [#1007](https://github.com/github/vscode-codeql/pull/1007)
 - Allow query result locations with 0 as the end column value. These are treated as the first column in the line. [#1002](https://github.com/github/vscode-codeql/pull/1002)

extensions/ql-vscode/README.md

@@ -22,7 +22,7 @@ For information about other configurations, see the separate [CodeQL help](https
 
 ### Quick start: Using CodeQL
 
-1. [Import a database from LGTM](#importing-a-database-from-lgtm).
+1. [Import a database from GitHub](#importing-a-database-from-github).
 1. [Run a query](#running-a-query).
 
 ---
@@ -73,18 +73,19 @@ If you're using your own clone of the CodeQL standard libraries, you can do a `g
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.
 
-### Importing a database from LGTM
+### Importing a database from GitHub
 
-While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from LGTM.com.
+While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from GitHub.com.
 
-1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
-1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
-1. Copy the link to that project, for example `https://lgtm.com/projects/g/apache/kafka`.
-1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from LGTM** command.
+1. Find a project that you're interested in on GitHub.com, for example [Apache Kafka](https://github.com/apache/kafka).
+1. Copy the link to that project, for example `https://github.com/apache/kafka`.
+1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from GitHub** command.
 1. Paste the link you copied earlier.
 1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
 1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
+For more information, see [Choosing a database](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#choosing-a-database) on codeql.github.com.
+
 ### Running a query
 
 The instructions below assume that you're using the CodeQL starter workspace, or that you've added the CodeQL libraries and queries repository to your workspace.
@@ -98,6 +99,10 @@ When the results are ready, they're displayed in the CodeQL Query Results view.
 
 If there are any problems running a query, a notification is displayed in the bottom right corner of the application. In addition to the error message, the notification includes details of how to fix the problem.
 
+### Keyboad navigation
+
+If you wish to navigate the query results from your keyboard, you can bind shortcuts to the **CodeQL: Navigate Up/Down/Left/Right in Result Viewer** commands.
+
 ## What next?
 
 For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:

extensions/ql-vscode/gulpfile.ts/appInsights.ts

@@ -1,5 +1,6 @@
 import * as gulp from 'gulp';
-import * as replace from 'gulp-replace';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const replace = require('gulp-replace');
 
 /** Inject the application insights key into the telemetry file */
 export function injectAppInsightsKey() {

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -1,5 +1,4 @@
 import * as fs from 'fs-extra';
-import * as jsonc from 'jsonc-parser';
 import * as path from 'path';
 
 export interface DeployedPackage {
@@ -16,7 +15,8 @@ const packageFiles = [
   'snippets.json',
   'media',
   'node_modules',
-  'out'
+  'out',
+  'workspace-databases-schema.json'
 ];
 
 async function copyPackage(sourcePath: string, destPath: string): Promise<void> {
@@ -28,7 +28,7 @@ async function copyPackage(sourcePath: string, destPath: string): Promise<void>
 
 export async function deployPackage(packageJsonPath: string): Promise<DeployedPackage> {
   try {
-    const packageJson: any = jsonc.parse(await fs.readFile(packageJsonPath, 'utf8'));
+    const packageJson: any = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
 
     // Default to development build; use flag --release to indicate release build.
     const isDevBuild = !process.argv.includes('--release');

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -1,8 +1,8 @@
 import * as gulp from 'gulp';
-import { compileTypeScript, watchTypeScript, copyViewCss, cleanOutput } from './typescript';
+import { compileTypeScript, watchTypeScript, cleanOutput } from './typescript';
 import { compileTextMateGrammar } from './textmate';
-import { copyTestData } from './tests';
-import { compileView } from './webpack';
+import { copyTestData, watchTestData } from './tests';
+import { compileView, watchView } from './webpack';
 import { packageExtension } from './package';
 import { injectAppInsightsKey } from './appInsights';
 
@@ -10,9 +10,19 @@ export const buildWithoutPackage =
   gulp.series(
     cleanOutput,
     gulp.parallel(
-      compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
+      compileTypeScript, compileTextMateGrammar, compileView, copyTestData
     )
   );
 
-export { cleanOutput, compileTextMateGrammar, watchTypeScript, compileTypeScript, copyTestData, injectAppInsightsKey };
+export {
+  cleanOutput,
+  compileTextMateGrammar,
+  watchTypeScript,
+  watchView,
+  compileTypeScript,
+  copyTestData,
+  watchTestData,
+  injectAppInsightsKey,
+  compileView,
+};
 export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

extensions/ql-vscode/gulpfile.ts/tests.ts

@@ -1,9 +1,14 @@
 import * as gulp from 'gulp';
 
 export function copyTestData() {
-  copyNoWorkspaceData();
-  copyCliIntegrationData();
-  return Promise.resolve();
+  return Promise.all([
+    copyNoWorkspaceData(),
+    copyCliIntegrationData()
+  ]);
+}
+
+export function watchTestData() {
+  return gulp.watch(['src/vscode-tests/*/data/**/*'], copyTestData);
 }
 
 function copyNoWorkspaceData() {

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -219,14 +219,14 @@ function transformFile(yaml: any) {
 }
 
 export function transpileTextMateGrammar() {
-  return through.obj((file: Vinyl, _encoding: string, callback: Function): void => {
+  return through.obj((file: Vinyl, _encoding: string, callback: (err: string | null, file: Vinyl | PluginError) => void): void => {
     if (file.isNull()) {
       callback(null, file);
     }
     else if (file.isBuffer()) {
       const buf: Buffer = file.contents;
       const yamlText: string = buf.toString('utf8');
-      const jsonData: any = jsYaml.safeLoad(yamlText);
+      const jsonData: any = jsYaml.load(yamlText);
       transformFile(jsonData);
 
       file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), 'utf8');

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -5,7 +5,7 @@
     "strict": true,
     "module": "commonjs",
     "target": "es2017",
-    "lib": ["es6"],
+    "lib": ["ES2021"],
     "moduleResolution": "node",
     "sourceMap": true,
     "rootDir": ".",
@@ -16,7 +16,8 @@
     "noImplicitReturns": true,
     "experimentalDecorators": true,
     "noUnusedLocals": true,
-    "noUnusedParameters": true
+    "noUnusedParameters": true,
+    "esModuleInterop": true
   },
   "include": ["*.ts"]
 }

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -39,9 +39,3 @@ export function compileTypeScript() {
 export function watchTypeScript() {
   gulp.watch('src/**/*.ts', compileTypeScript);
 }
-
-/** Copy CSS files for the results view into the output directory. */
-export function copyViewCss() {
-  return gulp.src('src/**/view/*.css')
-    .pipe(gulp.dest('out'));
-}

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -1,12 +1,11 @@
 import * as path from 'path';
 import * as webpack from 'webpack';
+import * as MiniCssExtractPlugin from 'mini-css-extract-plugin';
 
 export const config: webpack.Configuration = {
   mode: 'development',
   entry: {
-    resultsView: './src/view/results.tsx',
-    compareView: './src/compare/view/Compare.tsx',
-    remoteQueriesView: './src/remote-queries/view/RemoteQueries.tsx',
+    webview: './src/view/webview.tsx'
   },
   output: {
     path: path.resolve(__dirname, '..', 'out'),
@@ -31,9 +30,7 @@ export const config: webpack.Configuration = {
       {
         test: /\.less$/,
         use: [
-          {
-            loader: 'style-loader'
-          },
+          MiniCssExtractPlugin.loader,
           {
             loader: 'css-loader',
             options: {
@@ -53,17 +50,31 @@ export const config: webpack.Configuration = {
       {
         test: /\.css$/,
         use: [
-          {
-            loader: 'style-loader'
-          },
+          MiniCssExtractPlugin.loader,
           {
             loader: 'css-loader'
           }
         ]
+      },
+      {
+        test: /\.(woff(2)?|ttf|eot)$/,
+        use: [
+          {
+            loader: 'file-loader',
+            options: {
+              name: '[name].[ext]',
+              outputPath: 'fonts/',
+              // We need this to make Webpack use the correct path for the fonts.
+              // Without this, the CSS file will use `url([object Module])`
+              esModule: false
+            }
+          },
+        ],
       }
     ]
   },
   performance: {
     hints: false
-  }
+  },
+  plugins: [new MiniCssExtractPlugin()],
 };

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -2,7 +2,23 @@ import * as webpack from 'webpack';
 import { config } from './webpack.config';
 
 export function compileView(cb: (err?: Error) => void) {
-  webpack(config).run((error, stats) => {
+  doWebpack(config, true, cb);
+}
+
+export function watchView(cb: (err?: Error) => void) {
+  const watchConfig = {
+    ...config,
+    watch: true,
+    watchOptions: {
+      aggregateTimeout: 200,
+      poll: 1000,
+    }
+  };
+  doWebpack(watchConfig, false, cb);
+}
+
+function doWebpack(internalConfig: webpack.Configuration, failOnError: boolean, cb: (err?: Error) => void) {
+  const resultCb = (error: Error | undefined, stats?: webpack.Stats) => {
     if (error) {
       cb(error);
     }
@@ -20,11 +36,16 @@ export function compileView(cb: (err?: Error) => void) {
         errors: true
       }));
       if (stats.hasErrors()) {
-        cb(new Error('Compilation errors detected.'));
-        return;
+        if (failOnError) {
+          cb(new Error('Compilation errors detected.'));
+          return;
+        } else {
+          console.error('Compilation errors detected.');
+        }
       }
+      cb();
     }
+  };
 
-    cb();
-  });
+  webpack(internalConfig, resultCb);
 }

extensions/ql-vscode/jest.config.js

@@ -0,0 +1,214 @@
+/*
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+module.exports = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/6m/1394pht172qgd7dmw1fwjk100000gn/T/jest_dx",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  // clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  // coverageDirectory: undefined,
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: 'v8',
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  moduleFileExtensions: [
+    'js',
+    'mjs',
+    'cjs',
+    'jsx',
+    'ts',
+    'tsx',
+    'json'
+  ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  'moduleNameMapper': {
+    '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': '<rootDir>/test/__mocks__/fileMock.ts',
+    '\\.(css|less)$': '<rootDir>/test/__mocks__/styleMock.ts'
+  },
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  preset: 'ts-jest',
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  setupFilesAfterEnv: ['<rootDir>/test/jest.setup.ts'],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  testEnvironment: 'jsdom',
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  testMatch: [
+    '**/__tests__/**/*.[jt]s?(x)'
+  ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        tsconfig: 'src/view/tsconfig.spec.json',
+      },
+    ],
+    'node_modules': [
+      'babel-jest',
+      {
+        presets: [
+          '@babel/preset-env'
+        ],
+        plugins: [
+          '@babel/plugin-transform-modules-commonjs',
+        ]
+      }
+    ]
+  },
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  'transformIgnorePatterns': [
+    // These use ES modules, so need to be transformed
+    'node_modules/(?!(?:@vscode/webview-ui-toolkit|@microsoft/.+|exenv-es6)/.*)'
+  ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};

extensions/ql-vscode/media/dark/github.svg

@@ -0,0 +1,4 @@
+<!-- From https://github.com/microsoft/vscode-icons -->
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7.97553 0C3.57186 0 0 3.57186 0 7.97553C0 11.4985 2.29969 14.4832 5.43119 15.5596C5.82263 15.6086 5.96942 15.3639 5.96942 15.1682C5.96942 14.9725 5.96942 14.4832 5.96942 13.7982C3.76758 14.2875 3.27829 12.7217 3.27829 12.7217C2.93578 11.792 2.39755 11.5474 2.39755 11.5474C1.66361 11.0581 2.44648 11.0581 2.44648 11.0581C3.22936 11.107 3.66972 11.8899 3.66972 11.8899C4.40367 13.1131 5.52905 12.7706 5.96942 12.5749C6.01835 12.0367 6.263 11.6942 6.45872 11.4985C4.69725 11.3028 2.83792 10.6177 2.83792 7.53517C2.83792 6.65443 3.1315 5.96942 3.66972 5.38226C3.62079 5.23547 3.32722 4.40367 3.76758 3.32722C3.76758 3.32722 4.4526 3.1315 5.96942 4.15902C6.6055 3.9633 7.29052 3.91437 7.97553 3.91437C8.66055 3.91437 9.34557 4.01223 9.98165 4.15902C11.4985 3.1315 12.1835 3.32722 12.1835 3.32722C12.6239 4.40367 12.3303 5.23547 12.2813 5.43119C12.7706 5.96942 13.1131 6.70336 13.1131 7.5841C13.1131 10.6667 11.2538 11.3028 9.49235 11.4985C9.78593 11.7431 10.0306 12.2324 10.0306 12.9664C10.0306 14.0428 10.0306 14.8746 10.0306 15.1682C10.0306 15.3639 10.1774 15.6086 10.5688 15.5596C13.7492 14.4832 16 11.4985 16 7.97553C15.9511 3.57186 12.3792 0 7.97553 0Z" fill="#C5C5C5"/>
+</svg>

extensions/ql-vscode/media/drive.svg

@@ -0,0 +1,7 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M15.5 12.1952C15.5 12.9126 14.9137 13.4996 14.1957 13.4996H1.80435C1.08696 13.4996 0.5 12.9126 0.5 12.1952L0.5 9.80435C0.5 9.08696 1.08696 8.5 1.80435 8.5H14.1956C14.9137 8.5 15.5 9.08696 15.5 9.80435L15.5 12.1952Z" stroke="#959DA5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M2.45654 11.5H13.5435" stroke="#959DA5" stroke-linecap="round" stroke-linejoin="round"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M13.5 9.5C13.224 9.5 13 9.725 13 10C13 10.275 13.224 10.5 13.5 10.5C13.776 10.5 14 10.275 14 10C14 9.725 13.776 9.5 13.5 9.5" fill="#959DA5"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.5 9.5C11.224 9.5 11 9.725 11 10C11 10.275 11.224 10.5 11.5 10.5C11.776 10.5 12 10.275 12 10C12 9.725 11.776 9.5 11.5 9.5" fill="#959DA5"/>
+<path d="M15.5 9.81464L13.8728 2.76261C13.6922 2.06804 12.9572 1.5 12.2391 1.5H3.76087C3.04348 1.5 2.30848 2.06804 2.12783 2.76261L0.5 9.8" stroke="#959DA5" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

extensions/ql-vscode/media/globe.svg

@@ -0,0 +1,16 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<circle cx="7.5" cy="7.5" r="7" stroke="#959DA5"/>
+<mask id="mask0_394_2982" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="0" y="0" width="15" height="15">
+<circle cx="7.5" cy="7.5" r="7.5" fill="#C4C4C4"/>
+</mask>
+<g mask="url(#mask0_394_2982)">
+<path d="M14.5 7.5C14.5 9.42971 13.6822 11.1907 12.5493 12.4721C11.4035 13.7683 10.0054 14.5 8.90625 14.5C7.84644 14.5 6.81131 13.8113 6.01569 12.5383C5.22447 11.2724 4.71875 9.49235 4.71875 7.5C4.71875 5.50765 5.22447 3.72765 6.01569 2.4617C6.81131 1.1887 7.84644 0.5 8.90625 0.5C10.0054 0.5 11.4035 1.23172 12.5493 2.52786C13.6822 3.80934 14.5 5.57029 14.5 7.5Z" stroke="#959DA5"/>
+</g>
+<mask id="mask1_394_2982" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="1" y="0" width="16" height="15">
+<circle cx="9.375" cy="7.5" r="7.5" fill="#C4C4C4"/>
+</mask>
+<g mask="url(#mask1_394_2982)">
+<path d="M10.2812 7.5C10.2812 9.49235 9.77553 11.2724 8.98431 12.5383C8.18869 13.8113 7.15356 14.5 6.09375 14.5C4.99456 14.5 3.5965 13.7683 2.45067 12.4721C1.31781 11.1907 0.5 9.42971 0.5 7.5C0.5 5.57029 1.31781 3.80934 2.45067 2.52786C3.5965 1.23172 4.99456 0.5 6.09375 0.5C7.15356 0.5 8.18869 1.1887 8.98431 2.4617C9.77553 3.72765 10.2812 5.50765 10.2812 7.5Z" stroke="#959DA5"/>
+</g>
+<line y1="7.5" x2="15" y2="7.5" stroke="#959DA5"/>
+</svg>

extensions/ql-vscode/media/light/github.svg

@@ -0,0 +1,11 @@
+<!-- From https://github.com/microsoft/vscode-icons -->
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7.97578 0C3.57211 0 0.000244141 3.57186 0.000244141 7.97553C0.000244141 11.4985 2.29994 14.4832 5.43144 15.5596C5.82287 15.6086 5.96966 15.3639 5.96966 15.1682C5.96966 14.9725 5.96966 14.4832 5.96966 13.7982C3.76783 14.2875 3.27853 12.7217 3.27853 12.7217C2.93602 11.792 2.3978 11.5474 2.3978 11.5474C1.66385 11.0581 2.44673 11.0581 2.44673 11.0581C3.2296 11.107 3.66997 11.8899 3.66997 11.8899C4.40391 13.1131 5.5293 12.7706 5.96966 12.5749C6.01859 12.0367 6.26324 11.6942 6.45896 11.4985C4.69749 11.3028 2.83816 10.6177 2.83816 7.53517C2.83816 6.65443 3.13174 5.96942 3.66997 5.38226C3.62104 5.23547 3.32746 4.40367 3.76783 3.32722C3.76783 3.32722 4.45284 3.1315 5.96966 4.15902C6.60575 3.9633 7.29076 3.91437 7.97578 3.91437C8.66079 3.91437 9.34581 4.01223 9.98189 4.15902C11.4987 3.1315 12.1837 3.32722 12.1837 3.32722C12.6241 4.40367 12.3305 5.23547 12.2816 5.43119C12.7709 5.96942 13.1134 6.70336 13.1134 7.5841C13.1134 10.6667 11.2541 11.3028 9.4926 11.4985C9.78618 11.7431 10.0308 12.2324 10.0308 12.9664C10.0308 14.0428 10.0308 14.8746 10.0308 15.1682C10.0308 15.3639 10.1776 15.6086 10.5691 15.5596C13.7495 14.4832 16.0002 11.4985 16.0002 7.97553C15.9513 3.57186 12.3794 0 7.97578 0Z" fill="#424242"/>
+</g>
+<defs>
+<clipPath id="clip0">
+<rect width="16" height="16" fill="white" transform="translate(0.000244141)"/>
+</clipPath>
+</defs>
+</svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.5.10",
+  "version": "1.7.5",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -13,14 +13,15 @@
     "url": "https://github.com/github/vscode-codeql"
   },
   "engines": {
-    "vscode": "^1.57.0"
+    "vscode": "^1.59.0",
+    "node": "^16.13.0",
+    "npm": ">=7.20.6"
   },
   "categories": [
     "Programming Languages"
   ],
   "extensionDependencies": [
-    "hbenl.vscode-test-explorer",
-    "ms-vscode.test-adapter-converter"
+    "hbenl.vscode-test-explorer"
   ],
   "capabilities": {
     "untrustedWorkspaces": {
@@ -34,29 +35,36 @@
   },
   "activationEvents": [
     "onLanguage:ql",
+    "onLanguage:ql-summary",
     "onView:codeQLDatabases",
+    "onView:codeQLDatabasesExperimental",
     "onView:codeQLQueryHistory",
     "onView:codeQLAstViewer",
+    "onView:codeQLEvalLogViewer",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
     "onCommand:codeQL.authenticateToGitHub",
     "onCommand:codeQLDatabases.chooseDatabaseFolder",
     "onCommand:codeQLDatabases.chooseDatabaseArchive",
     "onCommand:codeQLDatabases.chooseDatabaseInternet",
+    "onCommand:codeQLDatabases.chooseDatabaseGithub",
     "onCommand:codeQLDatabases.chooseDatabaseLgtm",
     "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
+    "onCommand:codeQL.viewCfg",
     "onCommand:codeQL.openReferencedFile",
     "onCommand:codeQL.previewQueryHelp",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
+    "onCommand:codeQL.chooseDatabaseGithub",
     "onCommand:codeQL.chooseDatabaseLgtm",
     "onCommand:codeQLDatabases.chooseDatabase",
     "onCommand:codeQLDatabases.setCurrentDatabase",
     "onCommand:codeQL.quickQuery",
     "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
+    "onWebviewPanel:codeQL.variantAnalysis",
     "onFileSystem:codeql-zip-archive"
   ],
   "main": "./out/extension",
@@ -76,6 +84,12 @@
         "editor.wordBasedSuggestions": false
       }
     },
+    "jsonValidation": [
+      {
+        "fileMatch": "workspace-databases.json",
+        "url": "./workspace-databases-schema.json"
+      }
+    ],
     "languages": [
       {
         "id": "ql",
@@ -106,6 +120,12 @@
         "extensions": [
           ".qhelp"
         ]
+      },
+      {
+        "id": "ql-summary",
+        "filenames": [
+          "evaluator-log.summary"
+        ]
       }
     ],
     "grammars": [
@@ -131,7 +151,7 @@
       "title": "CodeQL",
       "properties": {
         "codeQL.cli.executablePath": {
-          "scope": "window",
+          "scope": "machine-overridable",
           "type": "string",
           "default": "",
           "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
@@ -205,7 +225,8 @@
             null
           ],
           "default": null,
-          "description": "Path to a directory where the CodeQL extension should store query server logs. If empty, the extension stores logs in a temporary workspace folder and deletes the contents after each run."
+          "description": "Path to a directory where the CodeQL extension should store query server logs. If empty, the extension stores logs in a temporary workspace folder and deletes the contents after each run.",
+          "markdownDeprecationMessage": "This property is deprecated and no longer has any effect. All query logs are stored in the query history folder next to the query results."
         },
         "codeQL.runningQueries.quickEvalCodelens": {
           "type": "boolean",
@@ -219,9 +240,15 @@
         },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "%q on %d - %s, %r result count [%t]",
+          "default": "%q on %d - %s %r [%t]",
           "markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
         },
+        "codeQL.queryHistory.ttl": {
+          "type": "number",
+          "default": 30,
+          "description": "Number of days to retain queries in the query history before being automatically deleted.",
+          "scope": "machine"
+        },
         "codeQL.runningTests.additionalTestArguments": {
           "scope": "window",
           "type": "array",
@@ -248,7 +275,7 @@
           "scope": "application",
           "description": "Specifies whether or not to write telemetry events to the extension log."
         },
-        "codeQL.remoteQueries.repositoryLists": {
+        "codeQL.variantAnalysis.repositoryLists": {
           "type": [
             "object",
             null
@@ -262,14 +289,21 @@
             }
           },
           "default": null,
-          "markdownDescription": "[For internal use only] Lists of GitHub repositories that you want to query remotely. This should be a JSON object where each key is a user-specified name for this repository list, and the value is an array of GitHub repositories (of the form `<owner>/<repo>`)."
+          "markdownDescription": "[For internal use only] Lists of GitHub repositories that you want to run variant analysis against. This should be a JSON object where each key is a user-specified name for this repository list, and the value is an array of GitHub repositories (of the form `<owner>/<repo>`)."
         },
-        "codeQL.remoteQueries.controllerRepo": {
+        "codeQL.variantAnalysis.controllerRepo": {
           "type": "string",
           "default": "",
           "pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
           "patternErrorMessage": "Please enter a valid GitHub repository",
-          "markdownDescription": "[For internal use only] The name of the GitHub repository where you can view the progress and results of the \"Run Remote query\" command. The repository should be of the form `<owner>/<repo>`)."
+          "markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
+        },
+        "codeQL.logInsights.joinOrderWarningThreshold": {
+          "type": "number",
+          "default": 50,
+          "scope": "window",
+          "minimum": 0,
+          "description": "Report a warning for any join order whose metric exceeds this value."
         }
       }
     },
@@ -287,12 +321,16 @@
         "title": "CodeQL: Run Query on Multiple Databases"
       },
       {
-        "command": "codeQL.runRemoteQuery",
-        "title": "CodeQL: Run Remote Query"
+        "command": "codeQL.runVariantAnalysis",
+        "title": "CodeQL: Run Variant Analysis"
       },
       {
-        "command": "codeQL.showFakeRemoteQueryResults",
-        "title": "CodeQL: [Internal] Show fake remote query results"
+        "command": "codeQL.exportVariantAnalysisResults",
+        "title": "CodeQL: Export Variant Analysis Results"
+      },
+      {
+        "command": "codeQL.openVariantAnalysis",
+        "title": "CodeQL: Open Variant Analysis"
       },
       {
         "command": "codeQL.runQueries",
@@ -322,6 +360,14 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLDatabasesExperimental.openConfigFile",
+        "title": "Open Database Configuration File",
+        "icon": {
+          "light": "media/light/edit.svg",
+          "dark": "media/dark/edit.svg"
+        }
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
@@ -350,6 +396,14 @@
           "dark": "media/dark/cloud-download.svg"
         }
       },
+      {
+        "command": "codeQLDatabases.chooseDatabaseGithub",
+        "title": "Download Database from GitHub",
+        "icon": {
+          "light": "media/light/github.svg",
+          "dark": "media/dark/github.svg"
+        }
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseLgtm",
         "title": "Download from LGTM",
@@ -366,6 +420,10 @@
         "command": "codeQL.viewAst",
         "title": "CodeQL: View AST"
       },
+      {
+        "command": "codeQL.viewCfg",
+        "title": "CodeQL: View CFG"
+      },
       {
         "command": "codeQL.upgradeCurrentDatabase",
         "title": "CodeQL: Upgrade Current Database"
@@ -418,6 +476,10 @@
         "command": "codeQL.chooseDatabaseInternet",
         "title": "CodeQL: Download Database"
       },
+      {
+        "command": "codeQL.chooseDatabaseGithub",
+        "title": "CodeQL: Download Database from GitHub"
+      },
       {
         "command": "codeQL.chooseDatabaseLgtm",
         "title": "CodeQL: Download Database from LGTM"
@@ -444,7 +506,7 @@
       },
       {
         "command": "codeQLQueryHistory.openQuery",
-        "title": "Open the query that produced these results",
+        "title": "Open the Query that Produced these Results",
         "icon": {
           "light": "media/light/edit.svg",
           "dark": "media/dark/edit.svg"
@@ -494,10 +556,34 @@
         "command": "codeQLQueryHistory.showQueryLog",
         "title": "Show Query Log"
       },
+      {
+        "command": "codeQLQueryHistory.openQueryDirectory",
+        "title": "Open Query Directory"
+      },
+      {
+        "command": "codeQLQueryHistory.showEvalLog",
+        "title": "Show Evaluator Log (Raw JSON)"
+      },
+      {
+        "command": "codeQLQueryHistory.showEvalLogSummary",
+        "title": "Show Evaluator Log (Summary Text)"
+      },
+      {
+        "command": "codeQLQueryHistory.showEvalLogViewer",
+        "title": "Show Evaluator Log (UI)"
+      },
+      {
+        "command": "codeQLQueryHistory.cancel",
+        "title": "Cancel"
+      },
       {
         "command": "codeQLQueryHistory.showQueryText",
         "title": "Show Query Text"
       },
+      {
+        "command": "codeQLQueryHistory.exportResults",
+        "title": "Export Results"
+      },
       {
         "command": "codeQLQueryHistory.viewCsvResults",
         "title": "View Results (CSV)"
@@ -523,12 +609,28 @@
         "title": "Compare Results"
       },
       {
-        "command": "codeQLQueryResults.nextPathStep",
-        "title": "CodeQL: Show Next Step on Path"
+        "command": "codeQLQueryHistory.openOnGithub",
+        "title": "Open Variant Analysis on GitHub"
       },
       {
-        "command": "codeQLQueryResults.previousPathStep",
-        "title": "CodeQL: Show Previous Step on Path"
+        "command": "codeQLQueryHistory.copyRepoList",
+        "title": "Copy Repository List"
+      },
+      {
+        "command": "codeQLQueryResults.down",
+        "title": "CodeQL: Navigate Down in Local Result Viewer"
+      },
+      {
+        "command": "codeQLQueryResults.up",
+        "title": "CodeQL: Navigate Up in Local Result Viewer"
+      },
+      {
+        "command": "codeQLQueryResults.right",
+        "title": "CodeQL: Navigate Right in Local Result Viewer"
+      },
+      {
+        "command": "codeQLQueryResults.left",
+        "title": "CodeQL: Navigate Left in Local Result Viewer"
       },
       {
         "command": "codeQL.restartQueryServer",
@@ -553,6 +655,39 @@
           "light": "media/light/clear-all.svg",
           "dark": "media/dark/clear-all.svg"
         }
+      },
+      {
+        "command": "codeQLEvalLogViewer.clear",
+        "title": "Clear Viewer",
+        "icon": {
+          "light": "media/light/clear-all.svg",
+          "dark": "media/dark/clear-all.svg"
+        }
+      },
+      {
+        "command": "codeQL.gotoQL",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
+      },
+      {
+        "command": "codeQL.mockGitHubApiServer.startRecording",
+        "title": "CodeQL: Mock GitHub API Server: Start Scenario Recording"
+      },
+      {
+        "command": "codeQL.mockGitHubApiServer.saveScenario",
+        "title": "CodeQL: Mock GitHub API Server: Save Scenario"
+      },
+      {
+        "command": "codeQL.mockGitHubApiServer.cancelRecording",
+        "title": "CodeQL: Mock GitHub API Server: Cancel Scenario Recording"
+      },
+      {
+        "command": "codeQL.mockGitHubApiServer.loadScenario",
+        "title": "CodeQL: Mock GitHub API Server: Load Scenario"
+      },
+      {
+        "command": "codeQL.mockGitHubApiServer.unloadScenario",
+        "title": "CodeQL: Mock GitHub API Server: Unload Scenario"
       }
     ],
     "menus": {
@@ -583,10 +718,15 @@
           "group": "navigation"
         },
         {
-          "command": "codeQLDatabases.chooseDatabaseLgtm",
+          "command": "codeQLDatabases.chooseDatabaseGithub",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         },
+        {
+          "command": "codeQLDatabases.chooseDatabaseLgtm",
+          "when": "config.codeQL.canary && view == codeQLDatabases",
+          "group": "navigation"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "when": "view == codeQLQueryHistory",
@@ -621,6 +761,16 @@
           "command": "codeQLAstViewer.clear",
           "when": "view == codeQLAstViewer",
           "group": "navigation"
+        },
+        {
+          "command": "codeQLEvalLogViewer.clear",
+          "when": "view == codeQLEvalLogViewer",
+          "group": "navigation"
+        },
+        {
+          "command": "codeQLDatabasesExperimental.openConfigFile",
+          "when": "view == codeQLDatabasesExperimental",
+          "group": "navigation"
         }
       ],
       "view/item/context": [
@@ -634,11 +784,6 @@
           "group": "9_qlCommands",
           "when": "view == codeQLDatabases"
         },
-        {
-          "command": "codeQLDatabases.upgradeDatabase",
-          "group": "9_qlCommands",
-          "when": "view == codeQLDatabases"
-        },
         {
           "command": "codeQLDatabases.renameDatabase",
           "group": "9_qlCommands",
@@ -662,7 +807,7 @@
         {
           "command": "codeQLQueryHistory.removeHistoryItem",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory"
+          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
         },
         {
           "command": "codeQLQueryHistory.setLabel",
@@ -672,52 +817,87 @@
         {
           "command": "codeQLQueryHistory.compareWith",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showQueryLog",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.openQueryDirectory",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && !hasRemoteServer"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLog",
+          "group": "9_qlCommands",
+          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogSummary",
+          "group": "9_qlCommands",
+          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogViewer",
+          "group": "9_qlCommands",
+          "when": "config.codeQL.canary && codeql.supportsEvalLog && viewItem == rawResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == interpretedResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showQueryText",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
         },
+        {
+          "command": "codeQLQueryHistory.exportResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == remoteResultsItem"
+        },
         {
           "command": "codeQLQueryHistory.viewCsvResults",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory && viewItem != interpretedResultsItem"
+          "when": "viewItem == rawResultsItem"
         },
         {
           "command": "codeQLQueryHistory.viewCsvAlerts",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+          "when": "viewItem == interpretedResultsItem"
         },
         {
           "command": "codeQLQueryHistory.viewSarifAlerts",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+          "when": "viewItem == interpretedResultsItem"
         },
         {
           "command": "codeQLQueryHistory.viewDil",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQLQueryHistory.cancel",
           "group": "9_qlCommands",
-          "when": "view == codeQLQueryHistory && resourceScheme == .qhelp && isWorkspaceTrusted"
+          "when": "viewItem == inProgressResultsItem || viewItem == inProgressRemoteResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.openOnGithub",
+          "group": "9_qlCommands",
+          "when": "viewItem == remoteResultsItem || viewItem == inProgressRemoteResultsItem || viewItem == cancelledRemoteResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.copyRepoList",
+          "group": "9_qlCommands",
+          "when": "viewItem == remoteResultsItem"
         },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
-          "when": "view == test-explorer && viewItem == testWithSource"
+          "when": "viewItem == testWithSource"
         },
         {
           "command": "codeQLTests.acceptOutput",
           "group": "qltest@2",
-          "when": "view == test-explorer && viewItem == testWithSource"
+          "when": "viewItem == testWithSource"
         }
       ],
       "explorer/context": [
@@ -729,7 +909,12 @@
         {
           "command": "codeQL.viewAst",
           "group": "9_qlCommands",
-          "when": "resourceScheme == codeql-zip-archive"
+          "when": "resourceScheme == codeql-zip-archive && !explorerResourceIsFolder && !listMultiSelection"
+        },
+        {
+          "command": "codeQL.viewCfg",
+          "group": "9_qlCommands",
+          "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
         },
         {
           "command": "codeQL.runQueries",
@@ -761,11 +946,15 @@
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
         {
-          "command": "codeQL.runRemoteQuery",
+          "command": "codeQL.runVariantAnalysis",
           "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
         },
         {
-          "command": "codeQL.showFakeRemoteQueryResults",
+          "command": "codeQL.openVariantAnalysis",
+          "when": "config.codeQL.canary && config.codeQL.variantAnalysis.liveResults"
+        },
+        {
+          "command": "codeQL.exportVariantAnalysisResults",
           "when": "config.codeQL.canary"
         },
         {
@@ -792,6 +981,18 @@
           "command": "codeQL.viewAst",
           "when": "resourceScheme == codeql-zip-archive"
         },
+        {
+          "command": "codeQL.viewCfg",
+          "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
+        },
+        {
+          "command": "codeQL.chooseDatabaseLgtm",
+          "when": "config.codeQL.canary"
+        },
+        {
+          "command": "codeQLDatabasesExperimental.openConfigFile",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -836,6 +1037,10 @@
           "command": "codeQLDatabases.chooseDatabaseInternet",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.chooseDatabaseGithub",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.chooseDatabaseLgtm",
           "when": "false"
@@ -860,10 +1065,42 @@
           "command": "codeQLQueryHistory.showQueryLog",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.showEvalLog",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogSummary",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogViewer",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.openQueryDirectory",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.cancel",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.openOnGithub",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.copyRepoList",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.showQueryText",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.exportResults",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.viewCsvResults",
           "when": "false"
@@ -908,6 +1145,10 @@
           "command": "codeQLAstViewer.clear",
           "when": "false"
         },
+        {
+          "command": "codeQLEvalLogViewer.clear",
+          "when": "false"
+        },
         {
           "command": "codeQLTests.acceptOutput",
           "when": "false"
@@ -915,6 +1156,26 @@
         {
           "command": "codeQLTests.showOutputDifferences",
           "when": "false"
+        },
+        {
+          "command": "codeQL.mockGitHubApiServer.startRecording",
+          "when": "config.codeQL.mockGitHubApiServer.enabled && !codeQL.mockGitHubApiServer.recording"
+        },
+        {
+          "command": "codeQL.mockGitHubApiServer.saveScenario",
+          "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.recording"
+        },
+        {
+          "command": "codeQL.mockGitHubApiServer.cancelRecording",
+          "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.recording"
+        },
+        {
+          "command": "codeQL.mockGitHubApiServer.loadScenario",
+          "when": "config.codeQL.mockGitHubApiServer.enabled && !codeQL.mockGitHubApiServer.recording"
+        },
+        {
+          "command": "codeQL.mockGitHubApiServer.unloadScenario",
+          "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.scenarioLoaded"
         }
       ],
       "editor/context": [
@@ -927,13 +1188,17 @@
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
         {
-          "command": "codeQL.runRemoteQuery",
+          "command": "codeQL.runVariantAnalysis",
           "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
         },
         {
           "command": "codeQL.viewAst",
           "when": "resourceScheme == codeql-zip-archive"
         },
+        {
+          "command": "codeQL.viewCfg",
+          "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -945,6 +1210,10 @@
         {
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
+        },
+        {
+          "command": "codeQL.gotoQL",
+          "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
     },
@@ -963,6 +1232,11 @@
           "id": "codeQLDatabases",
           "name": "Databases"
         },
+        {
+          "id": "codeQLDatabasesExperimental",
+          "name": "Databases",
+          "when": "config.codeQL.canary && config.codeQL.newQueryRunExperience"
+        },
         {
           "id": "codeQLQueryHistory",
           "name": "Query History"
@@ -970,6 +1244,11 @@
         {
           "id": "codeQLAstViewer",
           "name": "AST Viewer"
+        },
+        {
+          "id": "codeQLEvalLogViewer",
+          "name": "Evaluator Log Viewer",
+          "when": "config.codeQL.canary"
         }
       ]
     },
@@ -984,7 +1263,11 @@
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)"
+      },
+      {
+        "view": "codeQLEvalLogViewer",
+        "contents": "Run the 'Show Evaluator Log (UI)' command on a CodeQL query run in the Query History view."
       }
     ]
   },
@@ -992,31 +1275,55 @@
     "build": "gulp",
     "watch": "npm-run-all -p watch:*",
     "watch:extension": "tsc --watch",
-    "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
-    "preintegration": "rm -rf ./out/vscode-tests && gulp",
+    "watch:webpack": "gulp watchView",
+    "watch:files": "gulp watchTestData",
+    "test": "npm-run-all -p test:*",
+    "test:unit": "mocha --config .mocharc.json 'test/pure-tests/**/*.ts'",
+    "test:view": "jest",
     "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
-    "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
+    "integration:no-workspace": "node ./out/vscode-tests/run-integration-tests.js no-workspace",
+    "integration:minimal-workspace": "node ./out/vscode-tests/run-integration-tests.js minimal-workspace",
+    "cli-integration": "node ./out/vscode-tests/run-integration-tests.js cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
-    "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
-    "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
-    "format-staged": "lint-staged"
+    "format": "tsfmt -r && eslint . --ext .ts,.tsx --fix",
+    "lint": "eslint . --ext .ts,.tsx --max-warnings=0",
+    "format-staged": "lint-staged",
+    "storybook": "start-storybook -p 6006",
+    "build-storybook": "build-storybook",
+    "lint:scenarios": "ts-node scripts/lint-scenarios.ts"
   },
   "dependencies": {
-    "@octokit/rest": "^18.5.6",
+    "@octokit/plugin-retry": "^3.0.9",
+    "@octokit/rest": "^19.0.4",
+    "@primer/octicons-react": "^17.6.0",
+    "@primer/react": "^35.0.0",
+    "@vscode/codicons": "^0.0.31",
+    "@vscode/webview-ui-toolkit": "^1.0.1",
+    "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
+    "chokidar": "^3.5.3",
     "classnames": "~2.2.6",
-    "fs-extra": "^9.0.1",
-    "glob-promise": "^3.4.0",
-    "js-yaml": "^3.14.0",
-    "minimist": "~1.2.5",
+    "d3": "^7.6.1",
+    "d3-graphviz": "^2.6.1",
+    "fs-extra": "^10.0.1",
+    "glob-promise": "^4.2.2",
+    "immutable": "^4.0.0",
+    "js-yaml": "^4.1.0",
+    "minimist": "~1.2.6",
+    "msw": "^0.47.4",
+    "nanoid": "^3.2.0",
     "node-fetch": "~2.6.7",
+    "p-queue": "^6.0.0",
     "path-browserify": "^1.0.1",
-    "react": "^16.8.6",
-    "react-dom": "^16.8.6",
+    "react": "^17.0.2",
+    "react-dom": "^17.0.2",
     "semver": "~7.3.2",
+    "source-map": "^0.7.4",
+    "source-map-support": "^0.5.21",
     "stream": "^0.0.2",
     "stream-chain": "~2.2.4",
     "stream-json": "~1.7.3",
+    "styled-components": "^5.3.3",
     "tmp": "^0.1.0",
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
@@ -1026,28 +1333,47 @@
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
     "vscode-test-adapter-util": "~0.7.0",
-    "zip-a-folder": "~0.0.12"
+    "zip-a-folder": "~1.1.3"
   },
   "devDependencies": {
+    "@babel/core": "^7.18.13",
+    "@babel/plugin-transform-modules-commonjs": "^7.18.6",
+    "@faker-js/faker": "^7.5.0",
+    "@octokit/plugin-throttling": "^4.3.2",
+    "@storybook/addon-actions": "^6.5.10",
+    "@storybook/addon-essentials": "^6.5.10",
+    "@storybook/addon-interactions": "^6.5.10",
+    "@storybook/addon-links": "^6.5.10",
+    "@storybook/builder-webpack5": "^6.5.10",
+    "@storybook/manager-webpack5": "^6.5.10",
+    "@storybook/react": "^6.5.10",
+    "@storybook/testing-library": "^0.0.13",
+    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/react": "^12.1.5",
+    "@testing-library/user-event": "^14.4.3",
     "@types/chai": "^4.1.7",
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
+    "@types/d3": "^7.4.0",
+    "@types/d3-graphviz": "^2.6.6",
     "@types/del": "^4.0.0",
     "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.9",
-    "@types/gulp-replace": "0.0.31",
+    "@types/gulp-replace": "^1.1.0",
     "@types/gulp-sourcemaps": "0.0.32",
+    "@types/jest": "^29.0.2",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
     "@types/mocha": "^9.0.0",
-    "@types/node": "^12.14.1",
+    "@types/nanoid": "^3.0.0",
+    "@types/node": "^16.11.25",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
-    "@types/react": "^16.8.17",
-    "@types/react-dom": "^16.8.4",
+    "@types/react": "^17.0.2",
+    "@types/react-dom": "^17.0.2",
     "@types/sarif": "~2.1.2",
     "@types/semver": "~7.2.0",
     "@types/sinon": "~7.5.2",
@@ -1057,54 +1383,63 @@
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.57.0",
-    "@types/webpack": "^4.32.1",
+    "@types/vscode": "^1.59.0",
+    "@types/webpack": "^5.28.0",
+    "@types/webpack-env": "^1.18.0",
     "@types/xml2js": "~0.4.4",
     "@typescript-eslint/eslint-plugin": "^4.26.0",
     "@typescript-eslint/parser": "^4.26.0",
+    "@vscode/test-electron": "^2.2.0",
     "ansi-colors": "^4.1.1",
-    "applicationinsights": "^1.8.7",
+    "applicationinsights": "^2.3.5",
+    "babel-loader": "^8.2.5",
     "chai": "^4.2.0",
     "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
     "del": "^6.0.0",
     "eslint": "~6.8.0",
+    "eslint-plugin-jest-dom": "^4.0.2",
     "eslint-plugin-react": "~7.19.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-storybook": "^0.6.4",
+    "file-loader": "^6.2.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
-    "gulp-replace": "^1.0.0",
-    "gulp-sourcemaps": "^2.6.5",
+    "gulp-replace": "^1.1.3",
+    "gulp-sourcemaps": "^3.0.0",
     "gulp-typescript": "^5.0.1",
-    "husky": "~4.2.5",
-    "jsonc-parser": "^2.3.0",
+    "husky": "~4.3.8",
+    "jest": "^29.0.3",
+    "jest-environment-jsdom": "^29.0.3",
     "lint-staged": "~10.2.2",
-    "mocha": "^9.1.3",
+    "mini-css-extract-plugin": "^2.6.1",
+    "mocha": "^10.0.0",
     "mocha-sinon": "~2.1.2",
     "npm-run-all": "^4.1.5",
     "prettier": "~2.0.5",
     "proxyquire": "~2.1.3",
-    "sinon": "~9.0.0",
+    "sinon": "~14.0.0",
     "sinon-chai": "~3.5.0",
-    "style-loader": "~0.23.1",
-    "through2": "^3.0.1",
+    "through2": "^4.0.2",
+    "ts-jest": "^29.0.1",
+    "ts-json-schema-generator": "^1.1.2",
     "ts-loader": "^8.1.0",
-    "ts-node": "^8.3.0",
+    "ts-node": "^10.7.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "^4.3.2",
+    "typescript": "^4.5.5",
     "typescript-formatter": "^7.2.2",
-    "vsce": "^1.65.0",
-    "vscode-test": "^1.4.0",
-    "webpack": "^5.28.0",
+    "vsce": "^2.7.0",
+    "webpack": "^5.62.2",
     "webpack-cli": "^4.6.0"
   },
   "husky": {
     "hooks": {
       "pre-commit": "npm run format-staged",
-      "pre-push": "npm run lint"
+      "pre-push": "npm run lint && scripts/forbid-mocha-only"
     }
   },
   "lint-staged": {
-    "./**/*.{json,css,scss,md}": [
+    "./**/*.{json,css,scss}": [
       "prettier --write"
     ],
     "./**/*.{ts,tsx}": [
@@ -1113,6 +1448,6 @@
     ]
   },
   "resolutions": {
-    "glob-parent": "~6.0.0"
+    "glob-parent": "6.0.0"
   }
 }

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -0,0 +1,128 @@
+/**
+ * This scripts helps after adding a new field in the GitHub API. You will
+ * need to modify this script to add the new field to the scenarios. This
+ * is just a template and should not be used as-is since it has already been
+ * applied.
+ *
+ * Depending on the actual implementation of the script, you might run into
+ * rate limits. If that happens, you can set a `GITHUB_TOKEN` environment
+ * variable. For example, use: ``export GITHUB_TOKEN=`gh auth token```.
+ *
+ * Usage: npx ts-node scripts/add-fields-to-scenarios.ts
+ */
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { Octokit, type RestEndpointMethodTypes } from '@octokit/rest';
+import { throttling } from '@octokit/plugin-throttling';
+
+import { getFiles } from './util/files';
+import type { GitHubApiRequest } from '../src/mocks/gh-api-request';
+import { isGetVariantAnalysisRequest } from '../src/mocks/gh-api-request';
+import { VariantAnalysis } from '../src/remote-queries/gh-api/variant-analysis';
+import { RepositoryWithMetadata } from '../src/remote-queries/gh-api/repository';
+
+const extensionDirectory = path.resolve(__dirname, '..');
+const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
+
+// Make sure we don't run into rate limits by automatically waiting until we can
+// make another request.
+const MyOctokit = Octokit.plugin(throttling);
+
+const auth = process.env.GITHUB_TOKEN;
+
+const octokit = new MyOctokit({
+  auth,
+  throttle: {
+    onRateLimit: (retryAfter: number, options: any, octokit: Octokit): boolean => {
+      octokit.log.warn(
+        `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`
+      );
+
+      return true;
+    },
+    onSecondaryRateLimit: (_retryAfter: number, options: any, octokit: Octokit): void => {
+      octokit.log.warn(
+        `SecondaryRateLimit detected for request ${options.method} ${options.url}`
+      );
+    },
+  }
+});
+const repositories = new Map<number, RestEndpointMethodTypes['repos']['get']['response']['data']>();
+
+async function addFieldsToRepository(repository: RepositoryWithMetadata) {
+  if (!repositories.has(repository.id)) {
+    const [owner, repo] = repository.full_name.split('/');
+
+    const apiRepository = await octokit.repos.get({
+      owner,
+      repo,
+    });
+
+    repositories.set(repository.id, apiRepository.data);
+  }
+
+  const apiRepository = repositories.get(repository.id)!;
+
+  repository.stargazers_count = apiRepository.stargazers_count;
+  repository.updated_at = apiRepository.updated_at;
+}
+
+async function addFieldsToScenarios() {
+  if (!(await fs.pathExists(scenariosDirectory))) {
+    console.error('Scenarios directory does not exist: ' + scenariosDirectory);
+    return;
+  }
+
+  for await (const file of getFiles(scenariosDirectory)) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
+    const data: GitHubApiRequest = await fs.readJson(file);
+
+    if (!isGetVariantAnalysisRequest(data)) {
+      continue;
+    }
+
+    if (!data.response.body || !('controller_repo' in data.response.body)) {
+      continue;
+    }
+
+    console.log(`Adding fields to '${path.relative(scenariosDirectory, file)}'`);
+
+    const variantAnalysis = data.response.body as VariantAnalysis;
+
+    if (variantAnalysis.scanned_repositories) {
+      for (const item of variantAnalysis.scanned_repositories) {
+        await addFieldsToRepository(item.repository);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.access_mismatch_repos) {
+      for (const item of variantAnalysis.skipped_repositories.access_mismatch_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.no_codeql_db_repos) {
+      for (const item of variantAnalysis.skipped_repositories.no_codeql_db_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.over_limit_repos) {
+      for (const item of variantAnalysis.skipped_repositories.over_limit_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    await fs.writeJson(file, data, { spaces: 2 });
+  }
+}
+
+addFieldsToScenarios().catch(e => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts

@@ -0,0 +1,78 @@
+/**
+ * This scripts helps after recording a scenario to be used for replaying
+ * with the mock GitHub API server.
+ * 
+ * Once the scenario has been recorded, it's often useful to remove some of
+ * the requests to speed up the replay, particularly ones that fetch the 
+ * variant analysis status. Once some of the requests have manually been 
+ * removed, this script can be used to update the numbering of the files.
+ * 
+ * Usage: npx ts-node scripts/fix-scenario-file-numbering.ts <scenario-name>
+ */
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+if (process.argv.length !== 3) {
+  console.error('Expected 1 argument - the scenario name');
+}
+
+const scenarioName = process.argv[2];
+
+const extensionDirectory = path.resolve(__dirname, '..');
+const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
+const scenarioDirectory = path.resolve(scenariosDirectory, scenarioName);
+
+async function fixScenarioFiles() {
+  console.log(scenarioDirectory);
+  if (!(await fs.pathExists(scenarioDirectory))) {
+    console.error('Scenario directory does not exist: ' + scenarioDirectory);
+    return;
+  }
+
+  const files = await fs.readdir(scenarioDirectory);
+
+  const orderedFiles = files.sort((a, b) => {
+    const aNum = parseInt(a.split('-')[0]);
+    const bNum = parseInt(b.split('-')[0]);
+    return aNum - bNum;
+  });
+
+  let index = 0;
+  for (const file of orderedFiles) {
+    const ext = path.extname(file);
+    if (ext === '.json') {
+      const fileName = path.basename(file, ext);
+      const fileCurrentIndex = parseInt(fileName.split('-')[0]);
+      const fileNameWithoutIndex = fileName.split('-')[1];
+      if (fileCurrentIndex !== index) {
+        const newFileName = `${index}-${fileNameWithoutIndex}${ext}`;
+        const oldFilePath = path.join(scenarioDirectory, file);
+        const newFilePath = path.join(scenarioDirectory, newFileName);
+        console.log(`Rename: ${oldFilePath} -> ${newFilePath}`);
+        await fs.rename(oldFilePath, newFilePath);
+
+        if (fileNameWithoutIndex === 'getVariantAnalysisRepoResult') {
+          const oldZipFileName = `${fileCurrentIndex}-getVariantAnalysisRepoResult.body.zip`;
+          const newZipFileName = `${index}-getVariantAnalysisRepoResult.body.zip`;
+          const oldZipFilePath = path.join(scenarioDirectory, oldZipFileName);
+          const newZipFilePath = path.join(scenarioDirectory, newZipFileName);
+          console.log(`Rename: ${oldZipFilePath} -> ${newZipFilePath}`);
+          await fs.rename(oldZipFilePath, newZipFilePath);
+
+          const json = await fs.readJson(newFilePath);
+          json.response.body = `file:${newZipFileName}`;
+          console.log(`Response.body change to ${json.response.body}`);
+          await fs.writeJSON(newFilePath, json);
+        }
+      }
+
+      index++;
+    }
+  }
+}
+
+fixScenarioFiles().catch(e => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/forbid-mocha-only

@@ -0,0 +1,6 @@
+if grep -rq --include '*.test.ts' 'it.only\|describe.only' './test' './src'; then
+    echo 'There is a .only() in the tests. Please remove it.'
+    exit 1;
+else
+    exit 0;
+fi

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -0,0 +1,68 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import Ajv from 'ajv';
+import * as tsj from 'ts-json-schema-generator';
+
+import { getFiles } from './util/files';
+
+const extensionDirectory = path.resolve(__dirname, '..');
+const rootDirectory = path.resolve(extensionDirectory, '../..');
+const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
+
+const debug = process.env.RUNNER_DEBUG || process.argv.includes('--debug');
+
+async function lintScenarios() {
+  const schema = tsj.createGenerator({
+    path: path.resolve(extensionDirectory, 'src/mocks/gh-api-request.ts'),
+    tsconfig: path.resolve(extensionDirectory, 'tsconfig.json'),
+    type: 'GitHubApiRequest',
+    skipTypeCheck: true,
+    topRef: true,
+    additionalProperties: true,
+  }).createSchema('GitHubApiRequest');
+
+  const ajv = new Ajv();
+
+  if (!ajv.validateSchema(schema)) {
+    throw new Error('Invalid schema: ' + ajv.errorsText());
+  }
+
+  const validate = await ajv.compile(schema);
+
+  let invalidFiles = 0;
+
+  if (!(await fs.pathExists(scenariosDirectory))) {
+    console.error('Scenarios directory does not exist: ' + scenariosDirectory);
+    // Do not exit with a non-zero status code, as this is not a fatal error.
+    return;
+  }
+
+  for await (const file of getFiles(scenariosDirectory)) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
+    const contents = await fs.readFile(file, 'utf8');
+    const data = JSON.parse(contents);
+
+    if (!validate(data)) {
+      validate.errors?.forEach(error => {
+        // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
+        console.log(`::error file=${path.relative(rootDirectory, file)}::${error.instancePath}: ${error.message}`);
+      });
+      invalidFiles++;
+    } else if (debug) {
+      console.log(`File '${path.relative(rootDirectory, file)}' is valid`);
+    }
+  }
+
+  if (invalidFiles > 0) {
+    process.exit(1);
+  }
+}
+
+lintScenarios().catch(e => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "$schema": "https://json.schemastore.org/tsconfig",
+  "extends": "../tsconfig.json",
+  "include": ["**/*.ts"],
+  "exclude": [],
+  "compilerOptions": {
+    "rootDir": "..",
+    "noEmit": true
+  }
+}

extensions/ql-vscode/scripts/util/files.ts

@@ -0,0 +1,15 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+// https://stackoverflow.com/a/45130990
+export async function* getFiles(dir: string): AsyncGenerator<string> {
+  const dirents = await fs.readdir(dir, { withFileTypes: true });
+  for (const dirent of dirents) {
+    const res = path.resolve(dir, dirent.name);
+    if (dirent.isDirectory()) {
+      yield* getFiles(res);
+    } else {
+      yield res;
+    }
+  }
+}

extensions/ql-vscode/src/abstract-webview.ts

@@ -0,0 +1,156 @@
+import {
+  WebviewPanel,
+  ExtensionContext,
+  window as Window,
+  ViewColumn,
+  Uri,
+  WebviewPanelOptions,
+  WebviewOptions,
+} from 'vscode';
+import * as path from 'path';
+
+import { DisposableObject, DisposeHandler } from './pure/disposable-object';
+import { tmpDir } from './helpers';
+import { getHtmlForWebview, WebviewMessage, WebviewView } from './interface-utils';
+
+export type WebviewPanelConfig = {
+  viewId: string;
+  title: string;
+  viewColumn: ViewColumn;
+  view: WebviewView;
+  preserveFocus?: boolean;
+  additionalOptions?: WebviewPanelOptions & WebviewOptions;
+}
+
+export abstract class AbstractWebview<ToMessage extends WebviewMessage, FromMessage extends WebviewMessage> extends DisposableObject {
+  protected panel: WebviewPanel | undefined;
+  protected panelLoaded = false;
+  protected panelLoadedCallBacks: (() => void)[] = [];
+
+  private panelResolves?: Array<(panel: WebviewPanel) => void>;
+
+  constructor(
+    protected readonly ctx: ExtensionContext
+  ) {
+    super();
+  }
+
+  public async restoreView(panel: WebviewPanel): Promise<void> {
+    this.panel = panel;
+    const config = await this.getPanelConfig();
+    this.setupPanel(panel, config);
+  }
+
+  protected get isShowingPanel() {
+    return !!this.panel;
+  }
+
+  protected async getPanel(): Promise<WebviewPanel> {
+    if (this.panel == undefined) {
+      const { ctx } = this;
+
+      // This is an async method, so in theory this method can be called concurrently. To ensure that we don't
+      // create two panels, we use a promise that resolves when the panel is created. This way, if the panel is
+      // being created, the promise will resolve when it is done.
+      if (this.panelResolves !== undefined) {
+        return new Promise((resolve) => {
+          if (this.panel !== undefined) {
+            resolve(this.panel);
+            return;
+          }
+
+          this.panelResolves?.push(resolve);
+        });
+      }
+      this.panelResolves = [];
+
+      const config = await this.getPanelConfig();
+
+      const panel = Window.createWebviewPanel(
+        config.viewId,
+        config.title,
+        { viewColumn: config.viewColumn, preserveFocus: config.preserveFocus },
+        {
+          enableScripts: true,
+          enableFindWidget: true,
+          retainContextWhenHidden: true,
+          ...config.additionalOptions,
+          localResourceRoots: [
+            ...(config.additionalOptions?.localResourceRoots ?? []),
+            Uri.file(tmpDir.name),
+            Uri.file(path.join(ctx.extensionPath, 'out'))
+          ],
+        }
+      );
+      this.panel = panel;
+
+      this.setupPanel(panel, config);
+
+      this.panelResolves.forEach((resolve) => resolve(panel));
+      this.panelResolves = undefined;
+    }
+    return this.panel;
+  }
+
+  protected setupPanel(panel: WebviewPanel, config: WebviewPanelConfig): void {
+    this.push(
+      panel.onDidDispose(
+        () => {
+          this.panel = undefined;
+          this.panelLoaded = false;
+          this.onPanelDispose();
+        },
+        null,
+        this.ctx.subscriptions
+      )
+    );
+
+    panel.webview.html = getHtmlForWebview(
+      this.ctx,
+      panel.webview,
+      config.view,
+      {
+        allowInlineStyles: true,
+      }
+    );
+    this.push(
+      panel.webview.onDidReceiveMessage(
+        async (e) => this.onMessage(e),
+        undefined,
+        this.ctx.subscriptions
+      )
+    );
+  }
+
+  protected abstract getPanelConfig(): WebviewPanelConfig | Promise<WebviewPanelConfig>;
+
+  protected abstract onPanelDispose(): void;
+
+  protected abstract onMessage(msg: FromMessage): Promise<void>;
+
+  protected waitForPanelLoaded(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.panelLoaded) {
+        resolve();
+      } else {
+        this.panelLoadedCallBacks.push(resolve);
+      }
+    });
+  }
+
+  protected onWebViewLoaded(): void {
+    this.panelLoaded = true;
+    this.panelLoadedCallBacks.forEach((cb) => cb());
+    this.panelLoadedCallBacks = [];
+  }
+
+  protected async postMessage(msg: ToMessage): Promise<boolean> {
+    const panel = await this.getPanel();
+    return panel.webview.postMessage(msg);
+  }
+
+  public dispose(disposeHandler?: DisposeHandler) {
+    this.panel?.dispose();
+    super.dispose(disposeHandler);
+  }
+}

extensions/ql-vscode/src/additional-typings.d.ts

@@ -0,0 +1,15 @@
+/**
+ * The d3 library is designed to work in both the browser and
+ * node. Consequently their typings files refer to both node
+ * types like `Buffer` (which don't exist in the browser), and browser
+ * types like `Blob` (which don't exist in node). Instead of sticking
+ * all of `dom` in `compilerOptions.lib`, it suffices just to put in a
+ * stub definition of the affected types so that compilation
+ * succeeds.
+ */
+
+declare type RequestInit = Record<string, unknown>;
+declare type ElementTagNameMap = any;
+declare type NodeListOf<T> = Record<string, T>;
+declare type Node = Record<string, unknown>;
+declare type XMLDocument = Record<string, unknown>;

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -167,21 +167,26 @@ type Archive = {
   dirMap: DirectoryHierarchyMap;
 };
 
+async function parse_zip(zipPath: string): Promise<Archive> {
+  if (!await fs.pathExists(zipPath))
+    throw vscode.FileSystemError.FileNotFound(zipPath);
+  const archive: Archive = { unzipped: await unzipper.Open.file(zipPath), dirMap: new Map };
+  archive.unzipped.files.forEach(f => { ensureFile(archive.dirMap, path.resolve('/', f.path)); });
+  return archive;
+}
+
 export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
   private readOnlyError = vscode.FileSystemError.NoPermissions('write operation attempted, but source archive filesystem is readonly');
-  private archives: Map<string, Archive> = new Map;
+  private archives: Map<string, Promise<Archive>> = new Map;
 
   private async getArchive(zipPath: string): Promise<Archive> {
     if (!this.archives.has(zipPath)) {
-      if (!await fs.pathExists(zipPath))
-        throw vscode.FileSystemError.FileNotFound(zipPath);
-      const archive: Archive = { unzipped: await unzipper.Open.file(zipPath), dirMap: new Map };
-      archive.unzipped.files.forEach(f => { ensureFile(archive.dirMap, path.resolve('/', f.path)); });
-      this.archives.set(zipPath, archive);
+      this.archives.set(zipPath, parse_zip(zipPath));
     }
-    return this.archives.get(zipPath)!;
+    return await this.archives.get(zipPath)!;
   }
 
+
   root = new Directory('');
 
   // metadata

extensions/ql-vscode/src/astViewer.ts

@@ -10,7 +10,8 @@ import {
   TextEditorSelectionChangeEvent,
   TextEditorSelectionChangeKind,
   Location,
-  Range
+  Range,
+  Uri
 } from 'vscode';
 import * as path from 'path';
 
@@ -104,7 +105,7 @@ class AstViewerDataProvider extends DisposableObject implements TreeDataProvider
 export class AstViewer extends DisposableObject {
   private treeView: TreeView<AstItem>;
   private treeDataProvider: AstViewerDataProvider;
-  private currentFile: string | undefined;
+  private currentFileUri: Uri | undefined;
 
   constructor() {
     super();
@@ -125,12 +126,12 @@ export class AstViewer extends DisposableObject {
     this.push(window.onDidChangeTextEditorSelection(this.updateTreeSelection, this));
   }
 
-  updateRoots(roots: AstItem[], db: DatabaseItem, fileName: string) {
+  updateRoots(roots: AstItem[], db: DatabaseItem, fileUri: Uri) {
     this.treeDataProvider.roots = roots;
     this.treeDataProvider.db = db;
     this.treeDataProvider.refresh();
-    this.treeView.message = `AST for ${path.basename(fileName)}`;
-    this.currentFile = fileName;
+    this.treeView.message = `AST for ${path.basename(fileUri.fsPath)}`;
+    this.currentFileUri = fileUri;
     // Handle error on reveal. This could happen if
     // the tree view is disposed during the reveal.
     this.treeView.reveal(roots[0], { focus: false })?.then(
@@ -174,7 +175,7 @@ export class AstViewer extends DisposableObject {
 
     if (
       this.treeView.visible &&
-      e.textEditor.document.uri.fsPath === this.currentFile &&
+      e.textEditor.document.uri.fsPath === this.currentFileUri?.fsPath &&
       e.selections.length === 1
     ) {
       const selection = e.selections[0];
@@ -199,6 +200,6 @@ export class AstViewer extends DisposableObject {
     this.treeDataProvider.db = undefined;
     this.treeDataProvider.refresh();
     this.treeView.message = undefined;
-    this.currentFile = undefined;
+    this.currentFileUri = undefined;
   }
 }

extensions/ql-vscode/src/authentication.ts

@@ -1,13 +1,15 @@
 import * as vscode from 'vscode';
 import * as Octokit from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
 
 const GITHUB_AUTH_PROVIDER_ID = 'github';
 
-// 'repo' scope should be enough for triggering workflows. For a comprehensive list, see:
+// We need 'repo' scope for triggering workflows and 'gist' scope for exporting results to Gist.
+// For a comprehensive list of scopes, see:
 // https://docs.github.com/apps/building-oauth-apps/understanding-scopes-for-oauth-apps
-const SCOPES = ['repo'];
+const SCOPES = ['repo', 'gist'];
 
-/** 
+/**
  * Handles authentication to GitHub, using the VS Code [authentication API](https://code.visualstudio.com/api/references/vscode-api#authentication).
  */
 export class Credentials {
@@ -18,6 +20,15 @@ export class Credentials {
   // eslint-disable-next-line @typescript-eslint/no-empty-function
   private constructor() { }
 
+  /**
+   * Initializes an instance of credentials with an octokit instance.
+   *
+   * Do not call this method until you know you actually need an instance of credentials.
+   * since calling this method will require the user to log in.
+   *
+   * @param context The extension context.
+   * @returns An instance of credentials.
+   */
   static async initialize(context: vscode.ExtensionContext): Promise<Credentials> {
     const c = new Credentials();
     c.registerListeners(context);
@@ -25,12 +36,31 @@ export class Credentials {
     return c;
   }
 
-  private async createOctokit(createIfNone: boolean): Promise<Octokit.Octokit | undefined> {
+  /**
+   * Initializes an instance of credentials with an octokit instance using
+   * a token from the user's GitHub account. This method is meant to be
+   * used non-interactive environments such as tests.
+   *
+   * @param overrideToken The GitHub token to use for authentication.
+   * @returns An instance of credentials.
+   */
+  static async initializeWithToken(overrideToken: string) {
+    const c = new Credentials();
+    c.octokit = await c.createOctokit(false, overrideToken);
+    return c;
+  }
+
+  private async createOctokit(createIfNone: boolean, overrideToken?: string): Promise<Octokit.Octokit | undefined> {
+    if (overrideToken) {
+      return new Octokit.Octokit({ auth: overrideToken, retry });
+    }
+
     const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
 
     if (session) {
       return new Octokit.Octokit({
-        auth: session.accessToken
+        auth: session.accessToken,
+        retry
       });
     } else {
       return undefined;
@@ -46,16 +76,27 @@ export class Credentials {
     }));
   }
 
-  async getOctokit(): Promise<Octokit.Octokit> {
+  /**
+   * Creates or returns an instance of Octokit.
+   *
+   * @param requireAuthentication Whether the Octokit instance needs to be authenticated as user.
+   * @returns An instance of Octokit.
+   */
+  async getOctokit(requireAuthentication = true): Promise<Octokit.Octokit> {
     if (this.octokit) {
       return this.octokit;
     }
 
-    this.octokit = await this.createOctokit(true);
-    // octokit shouldn't be undefined, since we've set "createIfNone: true".
-    // The following block is mainly here to prevent a compiler error.
+    this.octokit = await this.createOctokit(requireAuthentication);
+
     if (!this.octokit) {
-      throw new Error('Did not initialize Octokit.');
+      if (requireAuthentication) {
+        throw new Error('Did not initialize Octokit.');
+      }
+
+      // We don't want to set this in this.octokit because that would prevent
+      // authenticating when requireCredentials is true.
+      return new Octokit.Octokit({ retry });
     }
     return this.octokit;
   }

extensions/ql-vscode/src/blob.d.ts

@@ -1,11 +0,0 @@
-/**
- * The npm library jszip is designed to work in both the browser and
- * node. Consequently its typings @types/jszip refers to both node
- * types like `Buffer` (which don't exist in the browser), and browser
- * types like `Blob` (which don't exist in node). Instead of sticking
- * all of `dom` in `compilerOptions.lib`, it suffices just to put in a
- * stub definition of the type `Blob` here so that compilation
- * succeeds.
- */
-
-declare type Blob = string;

extensions/ql-vscode/src/cli-version.ts

@@ -1,6 +1,7 @@
 import * as semver from 'semver';
 import { runCodeQlCliCommand } from './cli';
 import { Logger } from './logging';
+import { getErrorMessage } from './pure/helpers-pure';
 
 /**
  * Get the version of a CodeQL CLI.
@@ -18,7 +19,7 @@ export async function getCodeQlCliVersion(codeQlPath: string, logger: Logger): P
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.
-    void logger.log(`Failed to run 'codeql version'. Reason: ${e.message}`);
+    void logger.log(`Failed to run 'codeql version'. Reason: ${getErrorMessage(e)}`);
     return undefined;
   }
 }

extensions/ql-vscode/src/cli.ts

@@ -1,5 +1,6 @@
 import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
+import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
 import { SemVer } from 'semver';
@@ -7,17 +8,17 @@ import { Readable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as tk from 'tree-kill';
 import { promisify } from 'util';
-import { CancellationToken, Disposable, Uri } from 'vscode';
+import { CancellationToken, commands, Disposable, Uri } from 'vscode';
 
 import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
-import { CliConfig } from './config';
+import { allowCanaryQueryServer, CliConfig } from './config';
 import { DistributionProvider, FindDistributionResultKind } from './distribution';
-import { assertNever } from './pure/helpers-pure';
+import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
-import { CompilationMessage } from './pure/messages';
+import { CompilationMessage } from './pure/legacy-messages';
 import { sarifParser } from './sarif-parser';
-import { dbSchemeToLanguage } from './helpers';
+import { dbSchemeToLanguage, walkDirectory } from './helpers';
 
 /**
  * The version of the SARIF format that we are using.
@@ -167,7 +168,7 @@ export class CodeQLCliServer implements Disposable {
   nullBuffer: Buffer;
 
   /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: SemVer | undefined;
+  private _version: Promise<SemVer> | undefined;
 
   /**
    * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
@@ -239,7 +240,7 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Restart the server when the current command terminates
    */
-  private restartCliServer(): void {
+  restartCliServer(): void {
     const callback = (): void => {
       try {
         this.killProcessIfRunning();
@@ -345,7 +346,7 @@ export class CodeQLCliServer implements Disposable {
           stderrBuffers.length == 0
             ? new Error(`${description} failed: ${err}`)
             : new Error(`${description} failed: ${Buffer.concat(stderrBuffers).toString('utf8')}`);
-        newError.stack += (err.stack || '');
+        newError.stack += getErrorStack(err);
         throw newError;
       } finally {
         void this.logger.log(Buffer.concat(stderrBuffers).toString('utf8'));
@@ -403,7 +404,7 @@ export class CodeQLCliServer implements Disposable {
     try {
       if (cancellationToken !== undefined) {
         cancellationRegistration = cancellationToken.onCancellationRequested(_e => {
-          tk(child.pid);
+          tk(child.pid || 0);
         });
       }
       if (logger !== undefined) {
@@ -447,7 +448,7 @@ export class CodeQLCliServer implements Disposable {
       try {
         yield JSON.parse(event) as EventType;
       } catch (err) {
-        throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`);
+        throw new Error(`Parsing output of ${description} failed: ${(err as any).stderr || getErrorMessage(err)}`);
       }
     }
   }
@@ -502,7 +503,7 @@ export class CodeQLCliServer implements Disposable {
     try {
       return JSON.parse(result) as OutputType;
     } catch (err) {
-      throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`);
+      throw new Error(`Parsing output of ${description} failed: ${(err as any).stderr || getErrorMessage(err)}`);
     }
   }
 
@@ -514,8 +515,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(workspaces: string[], queryPath: string): Promise<QuerySetup> {
     const subcommandArgs = [
       '--query', queryPath,
-      '--additional-packs',
-      workspaces.join(path.delimiter)
+      ...this.getAdditionalPacksArg(workspaces)
     ];
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, 'Resolving library paths');
   }
@@ -528,8 +528,7 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       '--format', 'bylanguage',
       queryUri.fsPath,
-      '--additional-packs',
-      workspaces.join(path.delimiter)
+      ...this.getAdditionalPacksArg(workspaces)
     ];
     return JSON.parse(await this.runCodeQlCliCommand(['resolve', 'queries'], subcommandArgs, 'Resolving query by language'));
   }
@@ -562,6 +561,17 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Issues an internal clear-cache command to the cli server. This
+   * command is used to clear the qlpack cache of the server.
+   *
+   * This cache is generally cleared every 1s. This method is used
+   * to force an early clearing of the cache.
+   */
+  public async clearCache(): Promise<void> {
+    await this.runCodeQlCliCommand(['clear-cache'], [], 'Clearing qlpack cache');
+  }
+
   /**
    * Runs QL tests.
    * @param testPaths Full paths of the tests to run.
@@ -573,7 +583,7 @@ export class CodeQLCliServer implements Disposable {
   ): AsyncGenerator<TestCompleted, void, unknown> {
 
     const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
-      '--additional-packs', workspaces.join(path.delimiter),
+      ...this.getAdditionalPacksArg(workspaces),
       '--threads',
       this.cliConfig.numberTestThreads.toString(),
       ...testPaths
@@ -594,9 +604,17 @@ export class CodeQLCliServer implements Disposable {
   }
 
   /** Resolves the ML models that should be available when evaluating a query. */
-  async resolveMlModels(additionalPacks: string[]): Promise<MlModelsInfo> {
-    return await this.runJsonCodeQlCliCommand<MlModelsInfo>(['resolve', 'ml-models'], ['--additional-packs',
-      additionalPacks.join(path.delimiter)], 'Resolving ML models', false);
+  async resolveMlModels(additionalPacks: string[], queryPath: string): Promise<MlModelsInfo> {
+    const args = await this.cliConstraints.supportsPreciseResolveMlModels()
+      // use the dirname of the path so that we can handle query libraries
+      ? [...this.getAdditionalPacksArg(additionalPacks), path.dirname(queryPath)]
+      : this.getAdditionalPacksArg(additionalPacks);
+    return await this.runJsonCodeQlCliCommand<MlModelsInfo>(
+      ['resolve', 'ml-models'],
+      args,
+      'Resolving ML models',
+      false
+    );
   }
 
   /**
@@ -651,6 +669,44 @@ export class CodeQLCliServer implements Disposable {
     return await this.runCodeQlCliCommand(['generate', 'query-help'], subcommandArgs, `Generating qhelp in markdown format at ${outputDirectory}`);
   }
 
+  /**
+  * Generate a summary of an evaluation log.
+  * @param endSummaryPath The path to write only the end of query part of the human-readable summary to.
+  * @param inputPath The path of an evaluation event log.
+  * @param outputPath The path to write a human-readable summary of it to.
+  */
+  async generateLogSummary(
+    inputPath: string,
+    outputPath: string,
+    endSummaryPath: string,
+  ): Promise<string> {
+    const subcommandArgs = [
+      '--format=text',
+      `--end-summary=${endSummaryPath}`,
+      ...(await this.cliConstraints.supportsSourceMap() ? ['--sourcemap'] : []),
+      inputPath,
+      outputPath
+    ];
+    return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
+  }
+
+  /**
+  * Generate a JSON summary of an evaluation log.
+  * @param inputPath The path of an evaluation event log.
+  * @param outputPath The path to write a JSON summary of it to.
+  */
+  async generateJsonLogSummary(
+    inputPath: string,
+    outputPath: string,
+  ): Promise<string> {
+    const subcommandArgs = [
+      '--format=predicates',
+      inputPath,
+      outputPath
+    ];
+    return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating JSON log summary');
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
@@ -674,20 +730,13 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
+  async runInterpretCommand(format: string, additonalArgs: string[], metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
       '--output', interpretedResultsPath,
       '--format', format,
       // Forward all of the query metadata.
       ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`)
-    ];
-    if (format == SARIF_FORMAT) {
-      // TODO: This flag means that we don't group interpreted results
-      // by primary location. We may want to revisit whether we call
-      // interpretation with and without this flag, or do some
-      // grouping client-side.
-      args.push('--no-group-results');
-    }
+    ].concat(additonalArgs);
     if (sourceInfo !== undefined) {
       args.push(
         '--source-archive', sourceInfo.sourceArchive,
@@ -709,13 +758,47 @@ export class CodeQLCliServer implements Disposable {
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
   }
 
-  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
-    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
+  async interpretBqrsSarif(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    const additionalArgs = [
+      // TODO: This flag means that we don't group interpreted results
+      // by primary location. We may want to revisit whether we call
+      // interpretation with and without this flag, or do some
+      // grouping client-side.
+      '--no-group-results'
+    ];
+
+    await this.runInterpretCommand(SARIF_FORMAT, additionalArgs, metadata, resultsPath, interpretedResultsPath, sourceInfo);
     return await sarifParser(interpretedResultsPath);
   }
 
+  // Warning: this function is untenable for large dot files,
+  async readDotFiles(dir: string): Promise<string[]> {
+    const dotFiles: Promise<string>[] = [];
+    for await (const file of walkDirectory(dir)) {
+      if (file.endsWith('.dot')) {
+        dotFiles.push(fs.readFile(file, 'utf8'));
+      }
+    }
+    return Promise.all(dotFiles);
+  }
+
+  async interpretBqrsGraph(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<string[]> {
+    const additionalArgs = sourceInfo
+      ? ['--dot-location-url-format', 'file://' + sourceInfo.sourceLocationPrefix + '{path}:{start:line}:{start:column}:{end:line}:{end:column}']
+      : [];
+
+    await this.runInterpretCommand('dot', additionalArgs, metadata, resultsPath, interpretedResultsPath, sourceInfo);
+
+    try {
+      const dot = await this.readDotFiles(interpretedResultsPath);
+      return dot;
+    } catch (err) {
+      throw new Error(`Reading output of interpretation failed: ${getErrorMessage(err)}`);
+    }
+  }
+
   async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
-    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+    await this.runInterpretCommand(CSV_FORMAT, [], metadata, resultsPath, csvPath, sourceInfo);
   }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
@@ -761,7 +844,7 @@ export class CodeQLCliServer implements Disposable {
    * @returns A list of database upgrade script directories
    */
   async resolveUpgrades(dbScheme: string, searchPath: string[], allowDowngradesIfPossible: boolean, targetDbScheme?: string): Promise<UpgradesInfo> {
-    const args = ['--additional-packs', searchPath.join(path.delimiter), '--dbscheme', dbScheme];
+    const args = [...this.getAdditionalPacksArg(searchPath), '--dbscheme', dbScheme];
     if (targetDbScheme) {
       args.push('--target-dbscheme', targetDbScheme);
       if (allowDowngradesIfPossible && await this.cliConstraints.supportsDowngrades()) {
@@ -783,7 +866,7 @@ export class CodeQLCliServer implements Disposable {
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
   resolveQlpacks(additionalPacks: string[], searchPath?: string[]): Promise<QlpacksInfo> {
-    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    const args = this.getAdditionalPacksArg(additionalPacks);
     if (searchPath?.length) {
       args.push('--search-path', path.join(...searchPath));
     }
@@ -829,7 +912,7 @@ export class CodeQLCliServer implements Disposable {
    * @returns A list of query files found.
    */
   async resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
-    const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
+    const args = this.getAdditionalPacksArg(additionalPacks);
     if (searchPath !== undefined) {
       args.push('--search-path', path.join(...searchPath));
     }
@@ -853,21 +936,22 @@ export class CodeQLCliServer implements Disposable {
     return this.runJsonCodeQlCliCommand(['pack', 'download'], packs, 'Downloading packs');
   }
 
-  async packInstall(dir: string) {
-    return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
+  async packInstall(dir: string, forceUpdate = false) {
+    const args = [dir];
+    if (forceUpdate) {
+      args.push('--mode', 'update');
+    }
+    return this.runJsonCodeQlCliCommand(['pack', 'install'], args, 'Installing pack dependencies');
   }
 
-  async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
+  async packBundle(dir: string, workspaceFolders: string[], outputPath: string, moreOptions: string[]): Promise<void> {
     const args = [
       '-o',
       outputPath,
       dir,
-      '--additional-packs',
-      workspaceFolders.join(path.delimiter)
+      ...moreOptions,
+      ...this.getAdditionalPacksArg(workspaceFolders)
     ];
-    if (!precompile && await this.cliConstraints.supportsNoPrecompile()) {
-      args.push('--no-precompile');
-    }
 
     return this.runJsonCodeQlCliCommand(['pack', 'bundle'], args, 'Bundling pack');
   }
@@ -886,6 +970,12 @@ export class CodeQLCliServer implements Disposable {
     }
   }
 
+  async packResolveDependencies(dir: string): Promise<{ [pack: string]: string }> {
+    // Uses the default `--mode use-lock`, which creates the lock file if it doesn't exist.
+    const results: { [pack: string]: string } = await this.runJsonCodeQlCliCommand(['pack', 'resolve-dependencies'], [dir], 'Resolving pack dependencies');
+    return results;
+  }
+
   async generateDil(qloFile: string, outFile: string): Promise<void> {
     const extraArgs = await this.cliConstraints.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
@@ -899,9 +989,13 @@ export class CodeQLCliServer implements Disposable {
 
   public async getVersion() {
     if (!this._version) {
-      this._version = await this.refreshVersion();
+      this._version = this.refreshVersion();
+      // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
+      await commands.executeCommand(
+        'setContext', 'codeql.supportsEvalLog', await this.cliConstraints.supportsPerQueryEvalLog()
+      );
     }
-    return this._version;
+    return await this._version;
   }
 
   private async refreshVersion() {
@@ -918,6 +1012,12 @@ export class CodeQLCliServer implements Disposable {
         throw new Error('No distribution found');
     }
   }
+
+  private getAdditionalPacksArg(paths: string[]): string[] {
+    return paths.length
+      ? ['--additional-packs', paths.join(path.delimiter)]
+      : [];
+  }
 }
 
 /**
@@ -1004,7 +1104,7 @@ export async function runCodeQlCliCommand(
     void logger.log('CLI command succeeded.');
     return result.stdout;
   } catch (err) {
-    throw new Error(`${description} failed: ${err.stderr || err}`);
+    throw new Error(`${description} failed: ${(err as any).stderr || getErrorMessage(err)}`);
   }
 }
 
@@ -1060,8 +1160,8 @@ class SplitBuffer {
     while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
       for (const separator of this.separators) {
         if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
-          const line = this.buffer.substr(0, this.searchIndex);
-          this.buffer = this.buffer.substr(this.searchIndex + separator.length);
+          const line = this.buffer.slice(0, this.searchIndex);
+          this.buffer = this.buffer.slice(this.searchIndex + separator.length);
           this.searchIndex = 0;
           return line;
         }
@@ -1152,6 +1252,9 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
 
+
+  public static CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES = new SemVer('2.4.2');
+
   /**
    * CLI version where `codeql resolve upgrades` supports
    * the `--allow-downgrades` flag
@@ -1165,7 +1268,7 @@ export class CliVersionConstraint {
 
   /**
    * CLI version where database registration was introduced
-  */
+   */
   public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
 
   /**
@@ -1185,15 +1288,27 @@ export class CliVersionConstraint {
   public static CLI_VERSION_WITH_NO_PRECOMPILE = new SemVer('2.7.1');
 
   /**
-   * CLI version where remote queries are supported.
+   * CLI version where remote queries (variant analysis) are supported.
    */
   public static CLI_VERSION_REMOTE_QUERIES = new SemVer('2.6.3');
 
+  /**
+   * CLI version where building QLX packs for remote queries is supported.
+   * (The options were _accepted_ by a few earlier versions, but only from
+   * 2.11.3 will it actually use the existing compilation cache correctly).
+   */
+  public static CLI_VERSION_QLX_REMOTE = new SemVer('2.11.3');
+
   /**
    * CLI version where the `resolve ml-models` subcommand was introduced.
    */
   public static CLI_VERSION_WITH_RESOLVE_ML_MODELS = new SemVer('2.7.3');
 
+  /**
+   * CLI version where the `resolve ml-models` subcommand was enhanced to work with packaging.
+   */
+  public static CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS = new SemVer('2.10.0');
+
   /**
    * CLI version where the `--old-eval-stats` option to the query server was introduced.
    */
@@ -1204,6 +1319,33 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_PACKAGING = new SemVer('2.6.0');
 
+  /**
+   * CLI version where the `--evaluator-log` and related options to the query server were introduced,
+   * on a per-query server basis.
+   */
+  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
+
+  /**
+   * CLI version that supports rotating structured logs to produce one per query.
+   *
+   * Note that 2.8.4 supports generating the evaluation logs and summaries,
+   * but 2.9.0 includes a new option to produce the end-of-query summary logs to
+   * the query server console. For simplicity we gate all features behind 2.9.0,
+   * but if a user is tied to the 2.8 release, we can enable evaluator logs
+   * and summaries for them.
+   */
+  public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.9.0');
+
+  /**
+   * CLI version that supports the `--sourcemap` option for log generation.
+   */
+  public static CLI_VERSION_WITH_SOURCEMAP = new SemVer('2.10.3');
+
+  /**
+   * CLI version that supports the new query server.
+   */
+  public static CLI_VERSION_WITH_NEW_QUERY_SERVER = new SemVer('2.11.1');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1220,6 +1362,10 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
   }
 
+  public async supportsNonDestructiveUpgrades() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES);
+  }
+
   public async supportsDowngrades() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
   }
@@ -1248,10 +1394,18 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES);
   }
 
+  async supportsQlxRemote() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_QLX_REMOTE);
+  }
+
   async supportsResolveMlModels() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_ML_MODELS);
   }
 
+  async supportsPreciseResolveMlModels() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS);
+  }
+
   async supportsOldEvalStats() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_OLD_EVAL_STATS);
   }
@@ -1259,4 +1413,27 @@ export class CliVersionConstraint {
   async supportsPackaging() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PACKAGING);
   }
+
+  async supportsStructuredEvalLog() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG);
+  }
+
+  async supportsPerQueryEvalLog() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG);
+  }
+
+  async supportsSourceMap() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_SOURCEMAP);
+  }
+
+  async supportsNewQueryServer() {
+    // TODO while under development, users _must_ opt-in to the new query server
+    // by setting the `codeql.canaryQueryServer` setting to `true`.
+    return allowCanaryQueryServer() &&
+      this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER);
+  }
+
+  async supportsNewQueryServerForTests() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER);
+  }
 }

extensions/ql-vscode/src/commandRunner.ts

@@ -8,6 +8,7 @@ import {
 } from 'vscode';
 import { showAndLogErrorMessage, showAndLogWarningMessage } from './helpers';
 import { logger } from './logging';
+import { getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { telemetryListener } from './telemetry';
 
 export class UserCancellationException extends Error {
@@ -121,8 +122,9 @@ export function commandRunner(
     try {
       return await task(...args);
     } catch (e) {
-      error = e;
-      const errorMessage = `${e.message || e} (${commandId})`;
+      const errorMessage = `${getErrorMessage(e) || e} (${commandId})`;
+      error = e instanceof Error ? e : new Error(errorMessage);
+      const errorStack = getErrorStack(e);
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
@@ -132,8 +134,8 @@ export function commandRunner(
         }
       } else {
         // Include the full stack in the error log only.
-        const fullMessage = e.stack
-          ? `${errorMessage}\n${e.stack}`
+        const fullMessage = errorStack
+          ? `${errorMessage}\n${errorStack}`
           : errorMessage;
         void showAndLogErrorMessage(errorMessage, {
           fullMessage
@@ -160,7 +162,8 @@ export function commandRunner(
 export function commandRunnerWithProgress<R>(
   commandId: string,
   task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>
+  progressOptions: Partial<ProgressOptions>,
+  outputLogger = logger
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     const startTime = Date.now();
@@ -172,21 +175,23 @@ export function commandRunnerWithProgress<R>(
     try {
       return await withProgress(progressOptionsWithDefaults, task, ...args);
     } catch (e) {
-      error = e;
-      const errorMessage = `${e.message || e} (${commandId})`;
+      const errorMessage = `${getErrorMessage(e) || e} (${commandId})`;
+      error = e instanceof Error ? e : new Error(errorMessage);
+      const errorStack = getErrorStack(e);
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          void logger.log(errorMessage);
+          void outputLogger.log(errorMessage);
         } else {
-          void showAndLogWarningMessage(errorMessage);
+          void showAndLogWarningMessage(errorMessage, { outputLogger });
         }
       } else {
         // Include the full stack in the error log only.
-        const fullMessage = e.stack
-          ? `${errorMessage}\n${e.stack}`
+        const fullMessage = errorStack
+          ? `${errorMessage}\n${errorStack}`
           : errorMessage;
         void showAndLogErrorMessage(errorMessage, {
+          outputLogger,
           fullMessage
         });
       }

extensions/ql-vscode/src/compare/compare-view.ts

@@ -1,15 +1,8 @@
-import { DisposableObject } from '../pure/disposable-object';
 import {
-  WebviewPanel,
   ExtensionContext,
-  window as Window,
   ViewColumn,
-  Uri,
 } from 'vscode';
-import * as path from 'path';
 
-import { tmpDir } from '../run-queries';
-import { CompletedQuery } from '../query-results';
 import {
   FromCompareViewMessage,
   ToCompareViewMessage,
@@ -18,40 +11,43 @@ import {
 import { Logger } from '../logging';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
-import { getHtmlForWebview, jumpToLocation } from '../interface-utils';
+import { jumpToLocation } from '../interface-utils';
 import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../pure/bqrs-cli-types';
 import resultsDiff from './resultsDiff';
+import { CompletedLocalQueryInfo } from '../query-results';
+import { getErrorMessage } from '../pure/helpers-pure';
+import { HistoryItemLabelProvider } from '../history-item-label-provider';
+import { AbstractWebview, WebviewPanelConfig } from '../abstract-webview';
 
 interface ComparePair {
-  from: CompletedQuery;
-  to: CompletedQuery;
+  from: CompletedLocalQueryInfo;
+  to: CompletedLocalQueryInfo;
 }
 
-export class CompareInterfaceManager extends DisposableObject {
+export class CompareView extends AbstractWebview<ToCompareViewMessage, FromCompareViewMessage> {
   private comparePair: ComparePair | undefined;
-  private panel: WebviewPanel | undefined;
-  private panelLoaded = false;
-  private panelLoadedCallBacks: (() => void)[] = [];
 
   constructor(
-    private ctx: ExtensionContext,
+    ctx: ExtensionContext,
     private databaseManager: DatabaseManager,
     private cliServer: CodeQLCliServer,
     private logger: Logger,
+    private labelProvider: HistoryItemLabelProvider,
     private showQueryResultsCallback: (
-      item: CompletedQuery
+      item: CompletedLocalQueryInfo
     ) => Promise<void>
   ) {
-    super();
+    super(ctx);
   }
 
   async showResults(
-    from: CompletedQuery,
-    to: CompletedQuery,
+    from: CompletedLocalQueryInfo,
+    to: CompletedLocalQueryInfo,
     selectedResultSetName?: string
   ) {
     this.comparePair = { from, to };
-    this.getPanel().reveal(undefined, true);
+    const panel = await this.getPanel();
+    panel.reveal(undefined, true);
 
     await this.waitForPanelLoaded();
     const [
@@ -70,7 +66,7 @@ export class CompareInterfaceManager extends DisposableObject {
       try {
         rows = this.compareResults(fromResultSet, toResultSet);
       } catch (e) {
-        message = e.message;
+        message = getErrorMessage(e);
       }
 
       await this.postMessage({
@@ -80,18 +76,14 @@ export class CompareInterfaceManager extends DisposableObject {
             // since we split the description into several rows
             // only run interpolation if the label is user-defined
             // otherwise we will wind up with duplicated rows
-            name: from.options.label
-              ? from.interpolate(from.getLabel())
-              : from.queryName,
-            status: from.statusString,
-            time: from.time,
+            name: this.labelProvider.getShortLabel(from),
+            status: from.completedQuery.statusString,
+            time: from.startTime,
           },
           toQuery: {
-            name: to.options.label
-              ? to.interpolate(to.getLabel())
-              : to.queryName,
-            status: to.statusString,
-            time: to.time,
+            name: this.labelProvider.getShortLabel(to),
+            status: to.completedQuery.statusString,
+            time: to.startTime,
           },
         },
         columns: fromResultSet.schema.columns,
@@ -99,77 +91,29 @@ export class CompareInterfaceManager extends DisposableObject {
         currentResultSetName: currentResultSetName,
         rows,
         message,
-        datebaseUri: to.database.databaseUri,
+        databaseUri: to.initialInfo.databaseInfo.databaseUri,
       });
     }
   }
 
-  getPanel(): WebviewPanel {
-    if (this.panel == undefined) {
-      const { ctx } = this;
-      const panel = (this.panel = Window.createWebviewPanel(
-        'compareView',
-        'Compare CodeQL Query Results',
-        { viewColumn: ViewColumn.Active, preserveFocus: true },
-        {
-          enableScripts: true,
-          enableFindWidget: true,
-          retainContextWhenHidden: true,
-          localResourceRoots: [
-            Uri.file(tmpDir.name),
-            Uri.file(path.join(this.ctx.extensionPath, 'out')),
-          ],
-        }
-      ));
-      this.panel.onDidDispose(
-        () => {
-          this.panel = undefined;
-          this.comparePair = undefined;
-        },
-        null,
-        ctx.subscriptions
-      );
-
-      const scriptPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/compareView.js')
-      );
-
-      const stylesheetPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/view/resultsView.css')
-      );
-
-      panel.webview.html = getHtmlForWebview(
-        panel.webview,
-        scriptPathOnDisk,
-        [stylesheetPathOnDisk]
-      );
-      panel.webview.onDidReceiveMessage(
-        async (e) => this.handleMsgFromView(e),
-        undefined,
-        ctx.subscriptions
-      );
-    }
-    return this.panel;
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: 'compareView',
+      title: 'Compare CodeQL Query Results',
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: 'compare',
+    };
   }
 
-  private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve) => {
-      if (this.panelLoaded) {
-        resolve();
-      } else {
-        this.panelLoadedCallBacks.push(resolve);
-      }
-    });
+  protected onPanelDispose(): void {
+    this.comparePair = undefined;
   }
 
-  private async handleMsgFromView(
-    msg: FromCompareViewMessage
-  ): Promise<void> {
+  protected async onMessage(msg: FromCompareViewMessage): Promise<void> {
     switch (msg.t) {
-      case 'compareViewLoaded':
-        this.panelLoaded = true;
-        this.panelLoadedCallBacks.forEach((cb) => cb());
-        this.panelLoadedCallBacks = [];
+      case 'viewLoaded':
+        this.onWebViewLoaded();
         break;
 
       case 'changeCompare':
@@ -186,20 +130,16 @@ export class CompareInterfaceManager extends DisposableObject {
     }
   }
 
-  private postMessage(msg: ToCompareViewMessage): Thenable<boolean> {
-    return this.getPanel().webview.postMessage(msg);
-  }
-
   private async findCommonResultSetNames(
-    from: CompletedQuery,
-    to: CompletedQuery,
+    from: CompletedLocalQueryInfo,
+    to: CompletedLocalQueryInfo,
     selectedResultSetName: string | undefined
   ): Promise<[string[], string, RawResultSet, RawResultSet]> {
     const fromSchemas = await this.cliServer.bqrsInfo(
-      from.query.resultsPaths.resultsPath
+      from.completedQuery.query.resultsPaths.resultsPath
     );
     const toSchemas = await this.cliServer.bqrsInfo(
-      to.query.resultsPaths.resultsPath
+      to.completedQuery.query.resultsPaths.resultsPath
     );
     const fromSchemaNames = fromSchemas['result-sets'].map(
       (schema) => schema.name
@@ -215,12 +155,12 @@ export class CompareInterfaceManager extends DisposableObject {
     const fromResultSet = await this.getResultSet(
       fromSchemas,
       currentResultSetName,
-      from.query.resultsPaths.resultsPath
+      from.completedQuery.query.resultsPaths.resultsPath
     );
     const toResultSet = await this.getResultSet(
       toSchemas,
       currentResultSetName,
-      to.query.resultsPaths.resultsPath
+      to.completedQuery.query.resultsPaths.resultsPath
     );
     return [
       commonResultSetNames,

extensions/ql-vscode/src/compare/view/.eslintrc.js

@@ -1,13 +0,0 @@
-module.exports = {
-  env: {
-    browser: true
-  },
-  extends: [
-    "plugin:react/recommended"
-  ],
-  settings: {
-    react: {
-      version: 'detect'
-    }
-  }
-}

extensions/ql-vscode/src/config.ts

@@ -2,15 +2,27 @@ import { DisposableObject } from './pure/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
+import { ONE_DAY_IN_MS } from './pure/time';
+
+export const ALL_SETTINGS: Setting[] = [];
 
 /** Helper class to look up a labelled (and possibly nested) setting. */
 export class Setting {
   name: string;
   parent?: Setting;
+  private _hasChildren = false;
 
   constructor(name: string, parent?: Setting) {
     this.name = name;
     this.parent = parent;
+    if (parent !== undefined) {
+      parent._hasChildren = true;
+    }
+    ALL_SETTINGS.push(this);
+  }
+
+  get hasChildren() {
+    return this._hasChildren;
   }
 
   get qualifiedName(): string {
@@ -35,6 +47,18 @@ export class Setting {
     return workspace.getConfiguration(this.parent.qualifiedName).update(this.name, value, target);
   }
 
+  inspect<T>(): InspectionResult<T> | undefined {
+    if (this.parent === undefined) {
+      throw new Error('Cannot update the value of a root setting.');
+    }
+    return workspace.getConfiguration(this.parent.qualifiedName).inspect(this.name);
+  }
+}
+
+export interface InspectionResult<T> {
+  globalValue?: T;
+  workspaceValue?: T,
+  workspaceFolderValue?: T,
 }
 
 const ROOT_SETTING = new Setting('codeQL');
@@ -43,6 +67,7 @@ const ROOT_SETTING = new Setting('codeQL');
 const TELEMETRY_SETTING = new Setting('telemetry', ROOT_SETTING);
 const AST_VIEWER_SETTING = new Setting('astViewer', ROOT_SETTING);
 const GLOBAL_TELEMETRY_SETTING = new Setting('telemetry');
+const LOG_INSIGHTS_SETTING = new Setting('logInsights', ROOT_SETTING);
 
 export const LOG_TELEMETRY = new Setting('logTelemetry', TELEMETRY_SETTING);
 export const ENABLE_TELEMETRY = new Setting('enableTelemetry', TELEMETRY_SETTING);
@@ -54,8 +79,11 @@ const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
 export const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
 const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
 const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
+
+// Query History configuration
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
 const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
+const QUERY_HISTORY_TTL = new Setting('ttl', QUERY_HISTORY_SETTING);
 
 /** When these settings change, the distribution should be updated. */
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
@@ -71,7 +99,6 @@ export interface DistributionConfig {
 }
 
 // Query server configuration
-
 const RUNNING_QUERIES_SETTING = new Setting('runningQueries', ROOT_SETTING);
 const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES_SETTING);
 const SAVE_CACHE_SETTING = new Setting('saveCache', RUNNING_QUERIES_SETTING);
@@ -91,7 +118,10 @@ export const PAGE_SIZE = new Setting('pageSize', RESULTS_DISPLAY_SETTING);
 const CUSTOM_LOG_DIRECTORY_SETTING = new Setting('customLogDirectory', RUNNING_QUERIES_SETTING);
 
 /** When these settings change, the running query server should be restarted. */
-const QUERY_SERVER_RESTARTING_SETTINGS = [NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING, DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING];
+const QUERY_SERVER_RESTARTING_SETTINGS = [
+  NUMBER_OF_THREADS_SETTING, SAVE_CACHE_SETTING, CACHE_SIZE_SETTING, MEMORY_SETTING,
+  DEBUG_SETTING, CUSTOM_LOG_DIRECTORY_SETTING,
+];
 
 export interface QueryServerConfig {
   codeQlPath: string;
@@ -106,10 +136,11 @@ export interface QueryServerConfig {
 }
 
 /** When these settings change, the query history should be refreshed. */
-const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING];
+const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING, QUERY_HISTORY_TTL];
 
 export interface QueryHistoryConfig {
   format: string;
+  ttlInMillis: number;
   onDidChangeConfiguration: Event<void>;
 }
 
@@ -251,6 +282,13 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
   public get format(): string {
     return QUERY_HISTORY_FORMAT_SETTING.getValue<string>();
   }
+
+  /**
+   * The configuration value is in days, but return the value in milliseconds to make it easier to use.
+   */
+  public get ttlInMillis(): number {
+    return (QUERY_HISTORY_TTL.getValue<number>() || 30) * ONE_DAY_IN_MS;
+  }
 }
 
 export class CliConfigListener extends ConfigListener implements CliConfig {
@@ -303,16 +341,32 @@ export function isCanary() {
   return !!CANARY_FEATURES.getValue<boolean>();
 }
 
+/**
+ * Enables the experimental query server
+ */
+export const CANARY_QUERY_SERVER = new Setting('canaryQueryServer', ROOT_SETTING);
+
+
+export function allowCanaryQueryServer() {
+  return !!CANARY_QUERY_SERVER.getValue<boolean>();
+}
+
+export const JOIN_ORDER_WARNING_THRESHOLD = new Setting('joinOrderWarningThreshold', LOG_INSIGHTS_SETTING);
+
+export function joinOrderWarningThreshold(): number {
+  return JOIN_ORDER_WARNING_THRESHOLD.getValue<number>();
+}
+
 /**
  * Avoids caching in the AST viewer if the user is also a canary user.
  */
 export const NO_CACHE_AST_VIEWER = new Setting('disableCache', AST_VIEWER_SETTING);
 
-// Settings for remote queries
-const REMOTE_QUERIES_SETTING = new Setting('remoteQueries', ROOT_SETTING);
+// Settings for variant analysis
+const REMOTE_QUERIES_SETTING = new Setting('variantAnalysis', ROOT_SETTING);
 
 /**
- * Lists of GitHub repositories that you want to query remotely via the "Run Remote query" command.
+ * Lists of GitHub repositories that you want to query remotely via the "Run Variant Analysis" command.
  * Note: This command is only available for internal users.
  *
  * This setting should be a JSON object where each key is a user-specified name (string),
@@ -329,7 +383,22 @@ export async function setRemoteRepositoryLists(lists: Record<string, string[]> |
 }
 
 /**
- * The name of the "controller" repository that you want to use with the "Run Remote query" command.
+ * Path to a file that contains lists of GitHub repositories that you want to query remotely via
+ * the "Run Variant Analysis" command.
+ * Note: This command is only available for internal users.
+ *
+ * This setting should be a path to a JSON file that contains a JSON object where each key is a
+ * user-specified name (string), and the value is an array of GitHub repositories
+ * (of the form `<owner>/<repo>`).
+ */
+const REPO_LISTS_PATH = new Setting('repositoryListsPath', REMOTE_QUERIES_SETTING);
+
+export function getRemoteRepositoryListsPath(): string | undefined {
+  return REPO_LISTS_PATH.getValue<string>() || undefined;
+}
+
+/**
+ * The name of the "controller" repository that you want to use with the "Run Variant Analysis" command.
  * Note: This command is only available for internal users.
  *
  * This setting should be a GitHub repository of the form `<owner>/<repo>`.
@@ -345,13 +414,75 @@ export async function setRemoteControllerRepo(repo: string | undefined) {
 }
 
 /**
- * Whether to insecurely load ML models from CodeQL packs.
- *
- * This setting is for internal users only.
+ * The branch of "github/codeql-variant-analysis-action" to use with the "Run Variant Analysis" command.
+ * Default value is "main".
+ * Note: This command is only available for internal users.
  */
-const SHOULD_INSECURELY_LOAD_MODELS_FROM_PACKS =
-  new Setting('shouldInsecurelyLoadModelsFromPacks', RUNNING_QUERIES_SETTING);
+const ACTION_BRANCH = new Setting('actionBranch', REMOTE_QUERIES_SETTING);
 
-export function shouldInsecurelyLoadMlModelsFromPacks(): boolean {
-  return SHOULD_INSECURELY_LOAD_MODELS_FROM_PACKS.getValue<boolean>();
+export function getActionBranch(): string {
+  return ACTION_BRANCH.getValue<string>() || 'main';
+}
+
+export function isIntegrationTestMode() {
+  return process.env.INTEGRATION_TEST_MODE === 'true';
+}
+
+/**
+ * A flag indicating whether to enable the experimental "live results" feature
+ * for multi-repo variant analyses.
+ */
+const LIVE_RESULTS = new Setting('liveResults', REMOTE_QUERIES_SETTING);
+
+export function isVariantAnalysisLiveResultsEnabled(): boolean {
+  return !!LIVE_RESULTS.getValue<boolean>();
+}
+
+/**
+ * A flag indicating whether to use the new query run experience which involves
+ * using a new database panel.
+ */
+const NEW_QUERY_RUN_EXPERIENCE = new Setting('newQueryRunExperience', ROOT_SETTING);
+
+export function isNewQueryRunExperienceEnabled(): boolean {
+  return !!NEW_QUERY_RUN_EXPERIENCE.getValue<boolean>();
+}
+
+// Settings for mocking the GitHub API.
+const MOCK_GH_API_SERVER = new Setting('mockGitHubApiServer', ROOT_SETTING);
+
+/**
+ * A flag indicating whether to enable a mock GitHub API server.
+ */
+const MOCK_GH_API_SERVER_ENABLED = new Setting('enabled', MOCK_GH_API_SERVER);
+
+/**
+ * A path to a directory containing test scenarios. If this setting is not set,
+ * the mock server will a default location for test scenarios in dev mode, and 
+ * will show a menu to select a directory in production mode.
+ */
+const MOCK_GH_API_SERVER_SCENARIOS_PATH = new Setting('scenariosPath', MOCK_GH_API_SERVER);
+
+export interface MockGitHubApiConfig {
+  mockServerEnabled: boolean;
+  mockScenariosPath: string;
+  onDidChangeConfiguration: Event<void>;
+}
+
+export class MockGitHubApiConfigListener extends ConfigListener implements MockGitHubApiConfig {
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings([MOCK_GH_API_SERVER], e);
+  }
+
+  public get mockServerEnabled(): boolean {
+    return !!MOCK_GH_API_SERVER_ENABLED.getValue<boolean>();
+  }
+
+  public get mockScenariosPath(): string {
+    return MOCK_GH_API_SERVER_SCENARIOS_PATH.getValue<string>();
+  }
+}
+
+export function getMockGitHubApiServerScenariosPath(): string | undefined {
+  return MOCK_GH_API_SERVER_SCENARIOS_PATH.getValue<string>();
 }

extensions/ql-vscode/src/contextual/astBuilder.ts

@@ -1,9 +1,10 @@
-import { QueryWithResults } from '../run-queries';
 import { CodeQLCliServer } from '../cli';
 import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
 import { DatabaseItem } from '../databases';
 import { ChildAstItem, AstItem } from '../astViewer';
 import fileRangeFromURI from './fileRangeFromURI';
+import { Uri } from 'vscode';
+import { QueryWithResults } from '../run-queries-shared';
 
 /**
  * A class that wraps a tree of QL results from a query that
@@ -17,7 +18,7 @@ export default class AstBuilder {
     queryResults: QueryWithResults,
     private cli: CodeQLCliServer,
     public db: DatabaseItem,
-    public fileName: string
+    public fileName: Uri
   ) {
     this.bqrsPath = queryResults.query.resultsPaths.resultsPath;
   }

extensions/ql-vscode/src/contextual/keyType.ts

@@ -2,6 +2,7 @@ export enum KeyType {
   DefinitionQuery = 'DefinitionQuery',
   ReferenceQuery = 'ReferenceQuery',
   PrintAstQuery = 'PrintAstQuery',
+  PrintCfgQuery = 'PrintCfgQuery',
 }
 
 export function tagOfKeyType(keyType: KeyType): string {
@@ -12,6 +13,8 @@ export function tagOfKeyType(keyType: KeyType): string {
       return 'ide-contextual-queries/local-references';
     case KeyType.PrintAstQuery:
       return 'ide-contextual-queries/print-ast';
+    case KeyType.PrintCfgQuery:
+      return 'ide-contextual-queries/print-cfg';
   }
 }
 
@@ -23,6 +26,8 @@ export function nameOfKeyType(keyType: KeyType): string {
       return 'references';
     case KeyType.PrintAstQuery:
       return 'print AST';
+    case KeyType.PrintCfgQuery:
+      return 'print CFG';
   }
 }
 
@@ -32,6 +37,7 @@ export function kindOfKeyType(keyType: KeyType): string {
     case KeyType.ReferenceQuery:
       return 'definitions';
     case KeyType.PrintAstQuery:
+    case KeyType.PrintCfgQuery:
       return 'graph';
   }
 }

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -1,22 +1,20 @@
-import * as vscode from 'vscode';
-
 import { decodeSourceArchiveUri, encodeArchiveBasePath } from '../archive-filesystem-provider';
 import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from '../pure/bqrs-cli-types';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager, DatabaseItem } from '../databases';
 import fileRangeFromURI from './fileRangeFromURI';
-import * as messages from '../pure/messages';
-import { QueryServerClient } from '../queryserver-client';
-import { QueryWithResults, compileAndRunQueryAgainstDatabase } from '../run-queries';
 import { ProgressCallback } from '../commandRunner';
 import { KeyType } from './keyType';
-import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+import { qlpackOfDatabase, resolveQueries, runContextualQuery } from './queryResolver';
+import { CancellationToken, LocationLink, Uri } from 'vscode';
+import { QueryWithResults } from '../run-queries-shared';
+import { QueryRunner } from '../queryRunner';
 
 export const SELECT_QUERY_NAME = '#select';
 export const TEMPLATE_NAME = 'selectedSourceFile';
 
-export interface FullLocationLink extends vscode.LocationLink {
-  originUri: vscode.Uri;
+export interface FullLocationLink extends LocationLink {
+  originUri: Uri;
 }
 
 /**
@@ -29,21 +27,23 @@ export interface FullLocationLink extends vscode.LocationLink {
  * @param dbm The database manager
  * @param uriString The selected source file and location
  * @param keyType The contextual query type to run
+ * @param queryStorageDir The directory to store the query results
  * @param progress A progress callback
  * @param token A CancellationToken
  * @param filter A function that will filter extraneous results
  */
 export async function getLocationsForUriString(
   cli: CodeQLCliServer,
-  qs: QueryServerClient,
+  qs: QueryRunner,
   dbm: DatabaseManager,
   uriString: string,
   keyType: KeyType,
+  queryStorageDir: string,
   progress: ProgressCallback,
-  token: vscode.CancellationToken,
+  token: CancellationToken,
   filter: (src: string, dest: string) => boolean
 ): Promise<FullLocationLink[]> {
-  const uri = decodeSourceArchiveUri(vscode.Uri.parse(uriString, true));
+  const uri = decodeSourceArchiveUri(Uri.parse(uriString, true));
   const sourceArchiveUri = encodeArchiveBasePath(uri.sourceArchiveZipPath);
 
   const db = dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
@@ -56,18 +56,8 @@ export async function getLocationsForUriString(
 
   const links: FullLocationLink[] = [];
   for (const query of await resolveQueries(cli, qlpack, keyType)) {
-    const results = await compileAndRunQueryAgainstDatabase(
-      cli,
-      qs,
-      db,
-      false,
-      vscode.Uri.file(query),
-      progress,
-      token,
-      templates
-    );
-
-    if (results.result.resultType == messages.QueryResultType.SUCCESS) {
+    const results = await runContextualQuery(query, db, queryStorageDir, qs, cli, progress, token, templates);
+    if (results.successful) {
       links.push(...await getLinksFromResults(results, cli, db, filter));
     }
   }
@@ -104,15 +94,9 @@ async function getLinksFromResults(
   return localLinks;
 }
 
-function createTemplates(path: string): messages.TemplateDefinitions {
+function createTemplates(path: string): Record<string, string> {
   return {
-    [TEMPLATE_NAME]: {
-      values: {
-        tuples: [[{
-          stringValue: path
-        }]]
-      }
-    }
+    [TEMPLATE_NAME]: path
   };
 }
 

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -1,6 +1,7 @@
 import * as fs from 'fs-extra';
 import * as yaml from 'js-yaml';
 import * as tmp from 'tmp-promise';
+import * as path from 'path';
 
 import * as helpers from '../helpers';
 import {
@@ -12,6 +13,11 @@ import {
 import { CodeQLCliServer } from '../cli';
 import { DatabaseItem } from '../databases';
 import { QlPacksForLanguage } from '../helpers';
+import { logger } from '../logging';
+import { createInitialQueryInfo } from '../run-queries-shared';
+import { CancellationToken, Uri } from 'vscode';
+import { ProgressCallback } from '../commandRunner';
+import { QueryRunner } from '../queryRunner';
 
 export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<QlPacksForLanguage> {
   if (db.contents === undefined) {
@@ -45,7 +51,7 @@ async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[],
       }
     });
   }
-  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
+  await fs.writeFile(suiteFile, yaml.dump(suiteYaml), 'utf8');
 
   const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
   return queries;
@@ -104,3 +110,69 @@ export async function resolveQueries(cli: CodeQLCliServer, qlpacks: QlPacksForLa
   void helpers.showAndLogErrorMessage(errorMessage);
   throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} in any of the following packs: ${packsToSearch.join(', ')}.`);
 }
+
+async function resolveContextualQuery(cli: CodeQLCliServer, query: string): Promise<{ packPath: string, createdTempLockFile: boolean }> {
+  // Contextual queries now live within the standard library packs.
+  // This simplifies distribution (you don't need the standard query pack to use the AST viewer),
+  // but if the library pack doesn't have a lockfile, we won't be able to find
+  // other pack dependencies of the library pack.
+
+  // Work out the enclosing pack.
+  const packContents = await cli.packPacklist(query, false);
+  const packFilePath = packContents.find((p) => ['codeql-pack.yml', 'qlpack.yml'].includes(path.basename(p)));
+  if (packFilePath === undefined) {
+    // Should not happen; we already resolved this query.
+    throw new Error(`Could not find a CodeQL pack file for the pack enclosing the contextual query ${query}`);
+  }
+  const packPath = path.dirname(packFilePath);
+  const lockFilePath = packContents.find((p) => ['codeql-pack.lock.yml', 'qlpack.lock.yml'].includes(path.basename(p)));
+  let createdTempLockFile = false;
+  if (!lockFilePath) {
+    // No lock file, likely because this library pack is in the package cache.
+    // Create a lock file so that we can resolve dependencies and library path
+    // for the contextual query.
+    void logger.log(`Library pack ${packPath} is missing a lock file; creating a temporary lock file`);
+    await cli.packResolveDependencies(packPath);
+    createdTempLockFile = true;
+    // Clear CLI server pack cache before installing dependencies,
+    // so that it picks up the new lock file, not the previously cached pack.
+    void logger.log('Clearing the CodeQL CLI server\'s pack cache');
+    await cli.clearCache();
+    // Install dependencies.
+    void logger.log(`Installing package dependencies for library pack ${packPath}`);
+    await cli.packInstall(packPath);
+  }
+  return { packPath, createdTempLockFile };
+}
+
+async function removeTemporaryLockFile(packPath: string) {
+  const tempLockFilePath = path.resolve(packPath, 'codeql-pack.lock.yml');
+  void logger.log(`Deleting temporary package lock file at ${tempLockFilePath}`);
+  // It's fine if the file doesn't exist.
+  await fs.promises.rm(path.resolve(packPath, 'codeql-pack.lock.yml'), { force: true });
+}
+
+export async function runContextualQuery(query: string, db: DatabaseItem, queryStorageDir: string, qs: QueryRunner, cli: CodeQLCliServer, progress: ProgressCallback, token: CancellationToken, templates: Record<string, string>) {
+  const { packPath, createdTempLockFile } = await resolveContextualQuery(cli, query);
+  const initialInfo = await createInitialQueryInfo(
+    Uri.file(query),
+    {
+      name: db.name,
+      databaseUri: db.databaseUri.toString(),
+    },
+    false
+  );
+  void logger.log(`Running contextual query ${query}; results will be stored in ${queryStorageDir}`);
+  const queryResult = await qs.compileAndRunQueryAgainstDatabase(
+    db,
+    initialInfo,
+    queryStorageDir,
+    progress,
+    token,
+    templates
+  );
+  if (createdTempLockFile) {
+    await removeTemporaryLockFile(packPath);
+  }
+  return queryResult;
+}

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -16,31 +16,30 @@ import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
 import { CachedOperation } from '../helpers';
 import { ProgressCallback, withProgress } from '../commandRunner';
-import * as messages from '../pure/messages';
-import { QueryServerClient } from '../queryserver-client';
-import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';
 import AstBuilder from './astBuilder';
 import {
   KeyType,
 } from './keyType';
 import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
-import { qlpackOfDatabase, resolveQueries } from './queryResolver';
+import { qlpackOfDatabase, resolveQueries, runContextualQuery } from './queryResolver';
 import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
+import { QueryWithResults } from '../run-queries-shared';
+import { QueryRunner } from '../queryRunner';
 
 /**
- * Run templated CodeQL queries to find definitions and references in
+ * Runs templated CodeQL queries to find definitions in
  * source-language files. We may eventually want to find a way to
  * generalize this to other custom queries, e.g. showing dataflow to
  * or from a selected identifier.
  */
-
 export class TemplateQueryDefinitionProvider implements DefinitionProvider {
   private cache: CachedOperation<LocationLink[]>;
 
   constructor(
     private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
+    private qs: QueryRunner,
     private dbm: DatabaseManager,
+    private queryStorageDir: string,
   ) {
     this.cache = new CachedOperation<LocationLink[]>(this.getDefinitions.bind(this));
   }
@@ -68,6 +67,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
         this.dbm,
         uriString,
         KeyType.DefinitionQuery,
+        this.queryStorageDir,
         progress,
         token,
         (src, _dest) => src === uriString
@@ -76,13 +76,20 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
   }
 }
 
+/**
+ * Runs templated CodeQL queries to find references in
+ * source-language files. We may eventually want to find a way to
+ * generalize this to other custom queries, e.g. showing dataflow to
+ * or from a selected identifier.
+ */
 export class TemplateQueryReferenceProvider implements ReferenceProvider {
   private cache: CachedOperation<FullLocationLink[]>;
 
   constructor(
     private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
+    private qs: QueryRunner,
     private dbm: DatabaseManager,
+    private queryStorageDir: string,
   ) {
     this.cache = new CachedOperation<FullLocationLink[]>(this.getReferences.bind(this));
   }
@@ -115,6 +122,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
         this.dbm,
         uriString,
         KeyType.DefinitionQuery,
+        this.queryStorageDir,
         progress,
         token,
         (src, _dest) => src === uriString
@@ -123,33 +131,43 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
   }
 }
 
+type QueryWithDb = {
+  query: QueryWithResults,
+  dbUri: Uri
+};
+
+/**
+ * Run templated CodeQL queries to produce AST information for
+ * source-language files.
+ */
 export class TemplatePrintAstProvider {
-  private cache: CachedOperation<QueryWithResults>;
+  private cache: CachedOperation<QueryWithDb>;
 
   constructor(
     private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
+    private qs: QueryRunner,
     private dbm: DatabaseManager,
+    private queryStorageDir: string,
   ) {
-    this.cache = new CachedOperation<QueryWithResults>(this.getAst.bind(this));
+    this.cache = new CachedOperation<QueryWithDb>(this.getAst.bind(this));
   }
 
   async provideAst(
     progress: ProgressCallback,
     token: CancellationToken,
-    document?: TextDocument
+    fileUri?: Uri
   ): Promise<AstBuilder | undefined> {
-    if (!document) {
+    if (!fileUri) {
       throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
     }
-    const queryResults = this.shouldCache()
-      ? await this.cache.get(document.uri.toString(), progress, token)
-      : await this.getAst(document.uri.toString(), progress, token);
+    const { query, dbUri } = this.shouldCache()
+      ? await this.cache.get(fileUri.toString(), progress, token)
+      : await this.getAst(fileUri.toString(), progress, token);
 
     return new AstBuilder(
-      queryResults, this.cli,
-      this.dbm.findDatabaseItem(Uri.parse(queryResults.database.databaseUri!, true))!,
-      document.fileName
+      query, this.cli,
+      this.dbm.findDatabaseItem(dbUri)!,
+      fileUri,
     );
   }
 
@@ -161,7 +179,7 @@ export class TemplatePrintAstProvider {
     uriString: string,
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<QueryWithResults> {
+  ): Promise<QueryWithDb> {
     const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
       throw new Error('Cannot view the AST. Please select a valid source file inside a CodeQL database.');
@@ -185,25 +203,72 @@ export class TemplatePrintAstProvider {
     }
 
     const query = queries[0];
-    const templates: messages.TemplateDefinitions = {
-      [TEMPLATE_NAME]: {
-        values: {
-          tuples: [[{
-            stringValue: zippedArchive.pathWithinSourceArchive
-          }]]
-        }
-      }
+    const templates: Record<string, string> = {
+      [TEMPLATE_NAME]:
+        zippedArchive.pathWithinSourceArchive
     };
 
-    return await compileAndRunQueryAgainstDatabase(
-      this.cli,
-      this.qs,
-      db,
-      false,
-      Uri.file(query),
-      progress,
-      token,
-      templates
-    );
+    const queryResult = await runContextualQuery(query, db, this.queryStorageDir, this.qs, this.cli, progress, token, templates);
+    return {
+      query: queryResult,
+      dbUri: db.databaseUri
+    };
+  }
+}
+
+/**
+ * Run templated CodeQL queries to produce CFG information for
+ * source-language files.
+ */
+export class TemplatePrintCfgProvider {
+  private cache: CachedOperation<[Uri, Record<string, string>] | undefined>;
+
+  constructor(
+    private cli: CodeQLCliServer,
+    private dbm: DatabaseManager,
+  ) {
+    this.cache = new CachedOperation<[Uri, Record<string, string>] | undefined>(this.getCfgUri.bind(this));
+  }
+
+  async provideCfgUri(document?: TextDocument): Promise<[Uri, Record<string, string>] | undefined> {
+    if (!document) {
+      return;
+    }
+    return await this.cache.get(document.uri.toString());
+  }
+
+  private async getCfgUri(uriString: string): Promise<[Uri, Record<string, string>]> {
+    const uri = Uri.parse(uriString, true);
+    if (uri.scheme !== zipArchiveScheme) {
+      throw new Error('CFG Viewing is only available for databases with zipped source archives.');
+    }
+
+    const zippedArchive = decodeSourceArchiveUri(uri);
+    const sourceArchiveUri = encodeArchiveBasePath(zippedArchive.sourceArchiveZipPath);
+    const db = this.dbm.findDatabaseItemBySourceArchive(sourceArchiveUri);
+
+    if (!db) {
+      throw new Error('Can\'t infer database from the provided source.');
+    }
+
+    const qlpack = await qlpackOfDatabase(this.cli, db);
+    if (!qlpack) {
+      throw new Error('Can\'t infer qlpack from database source archive.');
+    }
+    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintCfgQuery);
+    if (queries.length > 1) {
+      throw new Error(`Found multiple Print CFG queries. Can't continue. Make sure there is exacly one query with the tag ${KeyType.PrintCfgQuery}`);
+    }
+    if (queries.length === 0) {
+      throw new Error(`Did not find any Print CFG queries. Can't continue. Make sure there is exacly one query with the tag ${KeyType.PrintCfgQuery}`);
+    }
+
+    const queryUri = Uri.file(queries[0]);
+
+    const templates: Record<string, string> = {
+      [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive
+    };
+
+    return [queryUri, templates];
   }
 }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -10,6 +10,8 @@ import {
 import { CodeQLCliServer } from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
+import * as Octokit from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
 
 import { DatabaseManager, DatabaseItem } from './databases';
 import {
@@ -20,7 +22,9 @@ import {
   ProgressCallback,
 } from './commandRunner';
 import { logger } from './logging';
-import { tmpDir } from './run-queries';
+import { tmpDir } from './helpers';
+import { Credentials } from './authentication';
+import { REPO_REGEX, getErrorMessage } from './pure/helpers-pure';
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -46,8 +50,10 @@ export async function promptImportInternetDatabase(
 
   const item = await databaseArchiveFetcher(
     databaseUrl,
+    {},
     databaseManager,
     storagePath,
+    undefined,
     progress,
     token,
     cli
@@ -61,6 +67,78 @@ export async function promptImportInternetDatabase(
 
 }
 
+/**
+ * Prompts a user to fetch a database from GitHub.
+ * User enters a GitHub repository and then the user is asked which language
+ * to download (if there is more than one)
+ *
+ * @param databaseManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function promptImportGithubDatabase(
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  credentials: Credentials | undefined,
+  progress: ProgressCallback,
+  token: CancellationToken,
+  cli?: CodeQLCliServer
+): Promise<DatabaseItem | undefined> {
+  progress({
+    message: 'Choose repository',
+    step: 1,
+    maxStep: 2
+  });
+  const githubRepo = await window.showInputBox({
+    title: 'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
+    placeHolder: 'https://github.com/<owner>/<repo> or <owner>/<repo>',
+    ignoreFocusOut: true,
+  });
+  if (!githubRepo) {
+    return;
+  }
+
+  if (!looksLikeGithubRepo(githubRepo)) {
+    throw new Error(`Invalid GitHub repository: ${githubRepo}`);
+  }
+
+  const octokit = credentials ? await credentials.getOctokit(true) : new Octokit.Octokit({ retry });
+
+  const result = await convertGithubNwoToDatabaseUrl(githubRepo, octokit, progress);
+  if (!result) {
+    return;
+  }
+
+  const { databaseUrl, name, owner } = result;
+
+  /**
+   * The 'token' property of the token object returned by `octokit.auth()`.
+   * The object is undocumented, but looks something like this:
+   * {
+   *   token: 'xxxx',
+   *   tokenType: 'oauth',
+   *   type: 'token',
+   * }
+   * We only need the actual token string.
+   */
+  const octokitToken = (await octokit.auth() as { token: string })?.token;
+  const item = await databaseArchiveFetcher(
+    databaseUrl,
+    { 'Accept': 'application/zip', 'Authorization': octokitToken ? `Bearer ${octokitToken}` : '' },
+    databaseManager,
+    storagePath,
+    `${owner}/${name}`,
+    progress,
+    token,
+    cli
+  );
+  if (item) {
+    await commands.executeCommand('codeQLDatabases.focus');
+    void showAndLogInformationMessage('Database downloaded and imported successfully.');
+    return item;
+  }
+  return;
+}
+
 /**
  * Prompts a user to fetch a database from lgtm.
  * User enters a project url and then the user is asked which language
@@ -90,12 +168,14 @@ export async function promptImportLgtmDatabase(
   }
 
   if (looksLikeLgtmUrl(lgtmUrl)) {
-    const databaseUrl = await convertToDatabaseUrl(lgtmUrl, progress);
+    const databaseUrl = await convertLgtmUrlToDatabaseUrl(lgtmUrl, progress);
     if (databaseUrl) {
       const item = await databaseArchiveFetcher(
         databaseUrl,
+        {},
         databaseManager,
         storagePath,
+        undefined,
         progress,
         token,
         cli
@@ -140,8 +220,10 @@ export async function importArchiveDatabase(
   try {
     const item = await databaseArchiveFetcher(
       databaseUrl,
+      {},
       databaseManager,
       storagePath,
+      undefined,
       progress,
       token,
       cli
@@ -152,7 +234,7 @@ export async function importArchiveDatabase(
     }
     return item;
   } catch (e) {
-    if (e.message.includes('unexpected end of file')) {
+    if (getErrorMessage(e).includes('unexpected end of file')) {
       throw new Error('Database is corrupt or too large. Try unzipping outside of VS Code and importing the unzipped folder instead.');
     } else {
       // delegate
@@ -166,15 +248,19 @@ export async function importArchiveDatabase(
  * or in the local filesystem.
  *
  * @param databaseUrl URL from which to grab the database
+ * @param requestHeaders Headers to send with the request
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
+ * @param nameOverride a name for the database that overrides the default
  * @param progress callback to send progress messages to
  * @param token cancellation token
  */
 async function databaseArchiveFetcher(
   databaseUrl: string,
+  requestHeaders: { [key: string]: string },
   databaseManager: DatabaseManager,
   storagePath: string,
+  nameOverride: string | undefined,
   progress: ProgressCallback,
   token: CancellationToken,
   cli?: CodeQLCliServer,
@@ -193,7 +279,7 @@ async function databaseArchiveFetcher(
   if (isFile(databaseUrl)) {
     await readAndUnzip(databaseUrl, unzipPath, cli, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, cli, progress);
+    await fetchAndUnzip(databaseUrl, requestHeaders, unzipPath, cli, progress);
   }
 
   progress({
@@ -216,7 +302,7 @@ async function databaseArchiveFetcher(
     });
     await ensureZippedSourceLocation(dbPath);
 
-    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath));
+    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath), nameOverride);
     await databaseManager.setCurrentDatabaseItem(item);
     return item;
   } else {
@@ -292,6 +378,7 @@ async function readAndUnzip(
 
 async function fetchAndUnzip(
   databaseUrl: string,
+  requestHeaders: { [key: string]: string },
   unzipPath: string,
   cli?: CodeQLCliServer,
   progress?: ProgressCallback
@@ -310,7 +397,10 @@ async function fetchAndUnzip(
     step: 1,
   });
 
-  const response = await checkForFailingResponse(await fetch(databaseUrl), 'Error downloading database');
+  const response = await checkForFailingResponse(
+    await fetch(databaseUrl, { headers: requestHeaders }),
+    'Error downloading database'
+  );
   const archiveFileStream = fs.createWriteStream(archivePath);
 
   const contentLength = response.headers.get('content-length');
@@ -325,7 +415,6 @@ async function fetchAndUnzip(
 
   await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, cli, progress);
 
-
   // remove archivePath eagerly since these archives can be large.
   await fs.remove(archivePath);
 }
@@ -381,6 +470,88 @@ export async function findDirWithFile(
   return;
 }
 
+/**
+ * The URL pattern is https://github.com/{owner}/{name}/{subpages}.
+ *
+ * This function accepts any URL that matches the pattern above. It also accepts just the
+ * name with owner (NWO): `<owner>/<repo>`.
+ *
+ * @param githubRepo The GitHub repository URL or NWO
+ *
+ * @return true if this looks like a valid GitHub repository URL or NWO
+ */
+export function looksLikeGithubRepo(
+  githubRepo: string | undefined
+): githubRepo is string {
+  if (!githubRepo) {
+    return false;
+  }
+  if (REPO_REGEX.test(githubRepo) || convertGitHubUrlToNwo(githubRepo)) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Converts a GitHub repository URL to the corresponding NWO.
+ * @param githubUrl The GitHub repository URL
+ * @return The corresponding NWO, or undefined if the URL is not valid
+ */
+function convertGitHubUrlToNwo(githubUrl: string): string | undefined {
+  try {
+    const uri = Uri.parse(githubUrl, true);
+    if (uri.scheme !== 'https') {
+      return;
+    }
+    if (uri.authority !== 'github.com' && uri.authority !== 'www.github.com') {
+      return;
+    }
+    const paths = uri.path.split('/').filter((segment: string) => segment);
+    const nwo = `${paths[0]}/${paths[1]}`;
+    if (REPO_REGEX.test(nwo)) {
+      return nwo;
+    }
+    return;
+  } catch (e) {
+    // Ignore the error here, since we catch failures at a higher level.
+    // In particular: returning undefined leads to an error in 'promptImportGithubDatabase'.
+    return;
+  }
+}
+
+export async function convertGithubNwoToDatabaseUrl(
+  githubRepo: string,
+  octokit: Octokit.Octokit,
+  progress: ProgressCallback): Promise<{
+    databaseUrl: string,
+    owner: string,
+    name: string
+  } | undefined> {
+  try {
+    const nwo = convertGitHubUrlToNwo(githubRepo) || githubRepo;
+    const [owner, repo] = nwo.split('/');
+
+    const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo });
+
+    const languages = response.data.map((db: any) => db.language);
+
+    const language = await promptForLanguage(languages, progress);
+    if (!language) {
+      return;
+    }
+
+    return {
+      databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
+      owner,
+      name: repo
+    };
+
+  } catch (e) {
+    void logger.log(`Error: ${getErrorMessage(e)}`);
+    throw new Error(`Unable to get database for '${githubRepo}'`);
+  }
+}
+
 /**
  * The URL pattern is https://lgtm.com/projects/{provider}/{org}/{name}/{irrelevant-subpages}.
  * There are several possibilities for the provider: in addition to GitHub.com (g),
@@ -416,7 +587,7 @@ export function looksLikeLgtmUrl(lgtmUrl: string | undefined): lgtmUrl is string
       return false;
     }
 
-    const paths = uri.path.split('/').filter((segment) => segment);
+    const paths = uri.path.split('/').filter((segment: string) => segment);
     return paths.length >= 4 && paths[0] === 'projects';
   } catch (e) {
     return false;
@@ -434,7 +605,7 @@ function convertRawLgtmSlug(maybeSlug: string): string | undefined {
   }
   return;
 }
- 
+
 function extractProjectSlug(lgtmUrl: string): string | undefined {
   // Only matches the '/g/' provider (github)
   const re = new RegExp('https://lgtm.com/projects/g/(.*[^/])');
@@ -446,7 +617,7 @@ function extractProjectSlug(lgtmUrl: string): string | undefined {
 }
 
 // exported for testing
-export async function convertToDatabaseUrl(
+export async function convertLgtmUrlToDatabaseUrl(
   lgtmUrl: string,
   progress: ProgressCallback) {
   try {
@@ -467,7 +638,9 @@ export async function convertToDatabaseUrl(
       }
     }
 
-    const language = await promptForLanguage(projectJson, progress);
+    const languages = projectJson?.languages?.map((lang: { language: string }) => lang.language) || [];
+
+    const language = await promptForLanguage(languages, progress);
     if (!language) {
       return;
     }
@@ -479,7 +652,7 @@ export async function convertToDatabaseUrl(
       language,
     ].join('/')}`;
   } catch (e) {
-    void logger.log(`Error: ${e.message}`);
+    void logger.log(`Error: ${getErrorMessage(e)}`);
     throw new Error(`Invalid LGTM URL: ${lgtmUrl}`);
   }
 }
@@ -487,7 +660,7 @@ export async function convertToDatabaseUrl(
 async function downloadLgtmProjectMetadata(lgtmUrl: string): Promise<any> {
   const uri = Uri.parse(lgtmUrl, true);
   const paths = ['api', 'v1.0'].concat(
-    uri.path.split('/').filter((segment) => segment)
+    uri.path.split('/').filter((segment: string) => segment)
   ).slice(0, 6);
   const projectUrl = `https://lgtm.com/${paths.join('/')}`;
   const projectResponse = await fetch(projectUrl);
@@ -495,7 +668,7 @@ async function downloadLgtmProjectMetadata(lgtmUrl: string): Promise<any> {
 }
 
 async function promptForLanguage(
-  projectJson: any,
+  languages: string[],
   progress: ProgressCallback
 ): Promise<string | undefined> {
   progress({
@@ -503,17 +676,19 @@ async function promptForLanguage(
     step: 2,
     maxStep: 2
   });
-  if (!projectJson?.languages?.length) {
-    return;
+  if (!languages.length) {
+    throw new Error('No databases found');
   }
-  if (projectJson.languages.length === 1) {
-    return projectJson.languages[0].language;
+  if (languages.length === 1) {
+    return languages[0];
   }
 
   return await window.showQuickPick(
-    projectJson.languages.map((lang: { language: string }) => lang.language), {
-    placeHolder: 'Select the database language to download:'
-  }
+    languages,
+    {
+      placeHolder: 'Select the database language to download:',
+      ignoreFocusOut: true,
+    }
   );
 }
 

extensions/ql-vscode/src/databases-ui.ts

@@ -28,16 +28,17 @@ import {
   showAndLogErrorMessage
 } from './helpers';
 import { logger } from './logging';
-import { clearCacheInDatabase } from './run-queries';
-import * as qsClient from './queryserver-client';
-import { upgradeDatabaseExplicit } from './upgrades';
 import {
   importArchiveDatabase,
+  promptImportGithubDatabase,
   promptImportInternetDatabase,
   promptImportLgtmDatabase,
 } from './databaseFetcher';
 import { CancellationToken } from 'vscode';
-import { asyncFilter } from './pure/helpers-pure';
+import { asyncFilter, getErrorMessage } from './pure/helpers-pure';
+import { Credentials } from './authentication';
+import { QueryRunner } from './queryRunner';
+import { isCanary } from './config';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -217,9 +218,10 @@ export class DatabaseUI extends DisposableObject {
 
   public constructor(
     private databaseManager: DatabaseManager,
-    private readonly queryServer: qsClient.QueryServerClient | undefined,
+    private readonly queryServer: QueryRunner | undefined,
     private readonly storagePath: string,
-    readonly extensionPath: string
+    readonly extensionPath: string,
+    private readonly getCredentials: () => Promise<Credentials>
   ) {
     super();
 
@@ -291,6 +293,20 @@ export class DatabaseUI extends DisposableObject {
         }
       )
     );
+    this.push(
+      commandRunnerWithProgress(
+        'codeQLDatabases.chooseDatabaseGithub',
+        async (
+          progress: ProgressCallback,
+          token: CancellationToken
+        ) => {
+          const credentials = isCanary() ? await this.getCredentials() : undefined;
+          await this.handleChooseDatabaseGithub(credentials, progress, token);
+        },
+        {
+          title: 'Adding database from GitHub',
+        })
+    );
     this.push(
       commandRunnerWithProgress(
         'codeQLDatabases.chooseDatabaseLgtm',
@@ -372,12 +388,11 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseFolder = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<DatabaseItem | undefined> => {
+  ): Promise<void> => {
     try {
-      return await this.chooseAndSetDatabase(true, progress, token);
+      await this.chooseAndSetDatabase(true, progress, token);
     } catch (e) {
-      void showAndLogErrorMessage(e.message);
-      return undefined;
+      void showAndLogErrorMessage(getErrorMessage(e));
     }
   };
 
@@ -440,12 +455,11 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseArchive = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<DatabaseItem | undefined> => {
+  ): Promise<void> => {
     try {
-      return await this.chooseAndSetDatabase(false, progress, token);
+      await this.chooseAndSetDatabase(false, progress, token);
     } catch (e) {
-      void showAndLogErrorMessage(e.message);
-      return undefined;
+      void showAndLogErrorMessage(getErrorMessage(e));
     }
   };
 
@@ -462,6 +476,21 @@ export class DatabaseUI extends DisposableObject {
     );
   };
 
+  handleChooseDatabaseGithub = async (
+    credentials: Credentials | undefined,
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<DatabaseItem | undefined> => {
+    return await promptImportGithubDatabase(
+      this.databaseManager,
+      this.storagePath,
+      credentials,
+      progress,
+      token,
+      this.queryServer?.cliServer
+    );
+  };
+
   handleChooseDatabaseLgtm = async (
     progress: ProgressCallback,
     token: CancellationToken
@@ -543,8 +572,7 @@ export class DatabaseUI extends DisposableObject {
 
     // Search for upgrade scripts in any workspace folders available
 
-    await upgradeDatabaseExplicit(
-      this.queryServer,
+    await this.queryServer.upgradeDatabaseExplicit(
       databaseItem,
       progress,
       token
@@ -559,8 +587,7 @@ export class DatabaseUI extends DisposableObject {
       this.queryServer !== undefined &&
       this.databaseManager.currentDatabaseItem !== undefined
     ) {
-      await clearCacheInDatabase(
-        this.queryServer,
+      await this.queryServer.clearCacheInDatabase(
         this.databaseManager.currentDatabaseItem,
         progress,
         token
@@ -590,8 +617,7 @@ export class DatabaseUI extends DisposableObject {
     } catch (e) {
       // rethrow and let this be handled by default error handling.
       throw new Error(
-        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${e.message
-        }`
+        `Could not set database to ${path.basename(uri.fsPath)}. Reason: ${getErrorMessage(e)}`
       );
     }
   };
@@ -724,7 +750,7 @@ export class DatabaseUI extends DisposableObject {
    * Perform some heuristics to ensure a proper database location is chosen.
    *
    * 1. If the selected URI to add is a file, choose the containing directory
-   * 2. If the selected URI is a directory matching db-*, choose the containing directory
+   * 2. If the selected URI appears to be a db language folder, choose the containing directory
    * 3. choose the current directory
    *
    * @param uri a URI that is a database folder or inside it
@@ -737,7 +763,7 @@ export class DatabaseUI extends DisposableObject {
       dbPath = path.dirname(dbPath);
     }
 
-    if (isLikelyDbLanguageFolder(dbPath)) {
+    if (await isLikelyDbLanguageFolder(dbPath)) {
       dbPath = path.dirname(dbPath);
     }
     return Uri.file(dbPath);

extensions/ql-vscode/src/databases.ts

@@ -17,8 +17,8 @@ import {
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
 import { DisposableObject } from './pure/disposable-object';
 import { Logger, logger } from './logging';
-import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
-import { QueryServerClient } from './queryserver-client';
+import { getErrorMessage } from './pure/helpers-pure';
+import { QueryRunner } from './queryRunner';
 
 /**
  * databases.ts
@@ -147,7 +147,7 @@ export async function findSourceArchive(
 }
 
 async function resolveDatabase(
-  databasePath: string
+  databasePath: string,
 ): Promise<DatabaseContents> {
 
   const name = path.basename(databasePath);
@@ -169,7 +169,9 @@ async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
   return await glob('*.dbscheme', { cwd: dbDirectory });
 }
 
-async function resolveDatabaseContents(uri: vscode.Uri): Promise<DatabaseContents> {
+async function resolveDatabaseContents(
+  uri: vscode.Uri,
+): Promise<DatabaseContents> {
   if (uri.scheme !== 'file') {
     throw new Error(`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`);
   }
@@ -356,14 +358,12 @@ export class DatabaseItemImpl implements DatabaseItem {
       try {
         this._contents = await resolveDatabaseContents(this.databaseUri);
         this._error = undefined;
-      }
-      catch (e) {
+      } catch (e) {
         this._contents = undefined;
-        this._error = e;
+        this._error = e instanceof Error ? e : new Error(String(e));
         throw e;
       }
-    }
-    finally {
+    } finally {
       this.onChanged({
         kind: DatabaseEventKind.Refresh,
         item: this
@@ -552,30 +552,28 @@ export class DatabaseManager extends DisposableObject {
 
   constructor(
     private readonly ctx: ExtensionContext,
-    private readonly qs: QueryServerClient,
+    private readonly qs: QueryRunner,
     private readonly cli: cli.CodeQLCliServer,
     public logger: Logger
   ) {
     super();
 
-    qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
-
-    // Let this run async.
-    void this.loadPersistedState();
+    qs.onStart(this.reregisterDatabases.bind(this));
   }
 
   public async openDatabase(
     progress: ProgressCallback,
     token: vscode.CancellationToken,
     uri: vscode.Uri,
+    displayName?: string
   ): Promise<DatabaseItem> {
     const contents = await resolveDatabaseContents(uri);
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj';
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive: isQLTestDatabase,
-      // displayName is only set if a user explicitly renames a database
-      displayName: undefined,
+      // If a displayName is not passed in, the basename of folder containing the database is used.
+      displayName,
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath)
     };
@@ -686,11 +684,13 @@ export class DatabaseManager extends DisposableObject {
         this._onDidChangeDatabaseItem.fire(event);
       });
 
-    await this.addDatabaseItem(progress, token, item);
+    // Avoid persisting the database state after adding since that should happen only after
+    // all databases have been added.
+    await this.addDatabaseItem(progress, token, item, false);
     return item;
   }
 
-  private async loadPersistedState(): Promise<void> {
+  public async loadPersistedState(): Promise<void> {
     return withProgress({
       location: vscode.ProgressLocation.Notification
     },
@@ -704,6 +704,7 @@ export class DatabaseManager extends DisposableObject {
           step
         });
         try {
+          void this.logger.log(`Found ${databases.length} persisted databases: ${databases.map(db => db.uri).join(', ')}`);
           for (const database of databases) {
             progress({
               maxStep: databases.length,
@@ -718,16 +719,20 @@ export class DatabaseManager extends DisposableObject {
               if (currentDatabaseUri === database.uri) {
                 await this.setCurrentDatabaseItem(databaseItem, true);
               }
-            }
-            catch (e) {
+              void this.logger.log(`Loaded database ${databaseItem.name} at URI ${database.uri}.`);
+            } catch (e) {
               // When loading from persisted state, leave invalid databases in the list. They will be
               // marked as invalid, and cannot be set as the current database.
+              void this.logger.log(`Error loading database ${database.uri}: ${e}.`);
             }
           }
+          await this.updatePersistedDatabaseList();
         } catch (e) {
           // database list had an unexpected type - nothing to be done?
-          void showAndLogErrorMessage(`Database list loading failed: ${e.message}`);
+          void showAndLogErrorMessage(`Database list loading failed: ${getErrorMessage(e)}`);
         }
+
+        void this.logger.log('Finished loading persisted databases.');
       });
   }
 
@@ -782,10 +787,14 @@ export class DatabaseManager extends DisposableObject {
   private async addDatabaseItem(
     progress: ProgressCallback,
     token: vscode.CancellationToken,
-    item: DatabaseItem
+    item: DatabaseItem,
+    updatePersistedState = true
   ) {
     this._databaseItems.push(item);
-    await this.updatePersistedDatabaseList();
+
+    if (updatePersistedState) {
+      await this.updatePersistedDatabaseList();
+    }
 
     // Add this database item to the allow-list
     // Database items reconstituted from persisted state
@@ -841,7 +850,7 @@ export class DatabaseManager extends DisposableObject {
       void logger.log('Deleting database from filesystem.');
       fs.remove(item.databaseUri.fsPath).then(
         () => void logger.log(`Deleted '${item.databaseUri.fsPath}'`),
-        e => void logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
+        e => void logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${getErrorMessage(e)}`));
     }
 
     // note that we use undefined as the item in order to reset the entire tree
@@ -856,27 +865,14 @@ export class DatabaseManager extends DisposableObject {
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
-      const databases: Dataset[] = [{
-        dbDir: dbItem.contents.datasetUri.fsPath,
-        workingSet: 'default'
-      }];
-      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
-    }
+    await this.qs.deregisterDatabase(progress, token, dbItem);
   }
-
   private async registerDatabase(
     progress: ProgressCallback,
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
-      const databases: Dataset[] = [{
-        dbDir: dbItem.contents.datasetUri.fsPath,
-        workingSet: 'default'
-      }];
-      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
-    }
+    await this.qs.registerDatabase(progress, token, dbItem);
   }
 
   private updatePersistedCurrentDatabaseItem(): void {

extensions/ql-vscode/src/databases/README.md

@@ -0,0 +1,3 @@
+### Databases
+
+This folder contains code for the new experimental databases panel and new query run experience.

extensions/ql-vscode/src/databases/db-config-store.ts

@@ -0,0 +1,80 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { cloneDbConfig, DbConfig } from './db-config';
+import * as chokidar from 'chokidar';
+import { DisposableObject } from '../pure/disposable-object';
+import { DbConfigValidator } from './db-config-validator';
+
+export class DbConfigStore extends DisposableObject {
+  private readonly configPath: string;
+  private readonly configValidator: DbConfigValidator;
+
+  private config: DbConfig;
+  private configWatcher: chokidar.FSWatcher | undefined;
+
+  public constructor(
+    workspaceStoragePath: string,
+    extensionPath: string) {
+    super();
+
+    this.configPath = path.join(workspaceStoragePath, 'workspace-databases.json');
+
+    this.config = this.createEmptyConfig();
+    this.configWatcher = undefined;
+    this.configValidator = new DbConfigValidator(extensionPath);
+  }
+
+  public async initialize(): Promise<void> {
+    await this.loadConfig();
+    this.watchConfig();
+  }
+
+  public dispose(): void {
+    this.configWatcher?.unwatch(this.configPath);
+  }
+
+  public getConfig(): DbConfig {
+    // Clone the config so that it's not modified outside of this class.
+    return cloneDbConfig(this.config);
+  }
+
+  public getConfigPath(): string {
+    return this.configPath;
+  }
+
+  public validateConfig(): string[] {
+    return this.configValidator.validate(this.config);
+  }
+
+  private async loadConfig(): Promise<void> {
+    if (!await fs.pathExists(this.configPath)) {
+      await fs.writeJSON(this.configPath, this.createEmptyConfig(), { spaces: 2 });
+    }
+
+    await this.readConfig();
+  }
+
+  private async readConfig(): Promise<void> {
+    this.config = await fs.readJSON(this.configPath);
+  }
+
+  private readConfigSync(): void {
+    this.config = fs.readJSONSync(this.configPath);
+  }
+
+  private watchConfig(): void {
+    this.configWatcher = chokidar.watch(this.configPath).on('change', () => {
+      this.readConfigSync();
+    });
+  }
+
+  private createEmptyConfig(): DbConfig {
+    return {
+      remote: {
+        repositoryLists: [],
+        owners: [],
+        repositories: [],
+      }
+    };
+  }
+}

extensions/ql-vscode/src/databases/db-config-validator.ts

@@ -0,0 +1,24 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import Ajv from 'ajv';
+import { DbConfig } from './db-config';
+
+export class DbConfigValidator {
+  private readonly schema: any;
+
+  constructor(extensionPath: string) {
+    const schemaPath = path.resolve(extensionPath, 'workspace-databases-schema.json');
+    this.schema = fs.readJsonSync(schemaPath);
+  }
+
+  public validate(dbConfig: DbConfig): string[] {
+    const ajv = new Ajv({ allErrors: true });
+    ajv.validate(this.schema, dbConfig);
+
+    if (ajv.errors) {
+      return ajv.errors.map((error) => `${error.instancePath} ${error.message}`);
+    }
+
+    return [];
+  }
+}

extensions/ql-vscode/src/databases/db-config.ts

@@ -0,0 +1,29 @@
+// Contains models for the data we want to store in the database config
+
+export interface DbConfig {
+  remote: RemoteDbConfig;
+}
+
+export interface RemoteDbConfig {
+  repositoryLists: RemoteRepositoryList[];
+  owners: string[];
+  repositories: string[];
+}
+
+export interface RemoteRepositoryList {
+  name: string;
+  repositories: string[];
+}
+
+export function cloneDbConfig(config: DbConfig): DbConfig {
+  return {
+    remote: {
+      repositoryLists: config.remote.repositoryLists.map((list) => ({
+        name: list.name,
+        repositories: [...list.repositories],
+      })),
+      owners: [...config.remote.owners],
+      repositories: [...config.remote.repositories],
+    }
+  };
+}

extensions/ql-vscode/src/databases/db-item.ts

@@ -0,0 +1,53 @@
+// This file contains models that are used to represent the databases.
+
+export enum DbItemKind {
+  RootLocal = 'RootLocal',
+  RootRemote = 'RootRemote',
+  RemoteSystemDefinedList = 'RemoteSystemDefinedList',
+  RemoteUserDefinedList = 'RemoteUserDefinedList',
+  RemoteOwner = 'RemoteOwner',
+  RemoteRepo = 'RemoteRepo'
+}
+
+export interface RootLocalDbItem {
+  kind: DbItemKind.RootLocal;
+}
+
+export interface RootRemoteDbItem {
+  kind: DbItemKind.RootRemote;
+  children: RemoteDbItem[];
+}
+
+export type DbItem =
+  | RootLocalDbItem
+  | RootRemoteDbItem
+  | RemoteDbItem
+
+export type RemoteDbItem =
+  | RemoteSystemDefinedListDbItem
+  | RemoteUserDefinedListDbItem
+  | RemoteOwnerDbItem
+  | RemoteRepoDbItem;
+
+export interface RemoteSystemDefinedListDbItem {
+  kind: DbItemKind.RemoteSystemDefinedList;
+  listName: string;
+  listDisplayName: string;
+  listDescription: string;
+}
+
+export interface RemoteUserDefinedListDbItem {
+  kind: DbItemKind.RemoteUserDefinedList;
+  listName: string;
+  repos: RemoteRepoDbItem[];
+}
+
+export interface RemoteOwnerDbItem {
+  kind: DbItemKind.RemoteOwner;
+  ownerName: string;
+}
+
+export interface RemoteRepoDbItem {
+  kind: DbItemKind.RemoteRepo;
+  repoFullName: string;
+}

extensions/ql-vscode/src/databases/db-manager.ts

@@ -0,0 +1,23 @@
+import { DbConfigStore } from './db-config-store';
+import { DbItem } from './db-item';
+import { createLocalTree, createRemoteTree } from './db-tree-creator';
+
+export class DbManager {
+  constructor(
+    private readonly dbConfigStore: DbConfigStore
+  ) {
+  }
+
+  public getDbItems(): DbItem[] {
+    const config = this.dbConfigStore.getConfig();
+
+    return [
+      createRemoteTree(config),
+      createLocalTree()
+    ];
+  }
+
+  public getConfigPath(): string {
+    return this.dbConfigStore.getConfigPath();
+  }
+}

extensions/ql-vscode/src/databases/db-module.ts

@@ -0,0 +1,45 @@
+import * as vscode from 'vscode';
+import { isCanary, isNewQueryRunExperienceEnabled } from '../config';
+import { logger } from '../logging';
+import { DisposableObject } from '../pure/disposable-object';
+import { DbConfigStore } from './db-config-store';
+import { DbManager } from './db-manager';
+import { DbPanel } from './ui/db-panel';
+
+export class DbModule extends DisposableObject {
+  public async initialize(
+    extensionContext: vscode.ExtensionContext
+  ): Promise<void> {
+    if (extensionContext.extensionMode !== vscode.ExtensionMode.Development ||
+      !isCanary() ||
+      !isNewQueryRunExperienceEnabled()) {
+      // Currently, we only want to expose the new database panel when we
+      // are in development and canary mode and the developer has enabled the 
+      // new query run experience.
+      return;
+    }
+
+    void logger.log('Initializing database module');
+
+    const storagePath = extensionContext.storageUri?.fsPath || extensionContext.globalStorageUri.fsPath;
+    const extensionPath = extensionContext.extensionPath;
+    const dbConfigStore = new DbConfigStore(storagePath, extensionPath);
+    await dbConfigStore.initialize();
+
+    const dbManager = new DbManager(dbConfigStore);
+    const dbPanel = new DbPanel(dbManager);
+    await dbPanel.initialize();
+    extensionContext.subscriptions.push(dbPanel);
+
+    this.push(dbPanel);
+    this.push(dbConfigStore);
+  }
+}
+
+export async function initializeDbModule(
+  extensionContext: vscode.ExtensionContext
+): Promise<DbModule> {
+  const dbModule = new DbModule();
+  await dbModule.initialize(extensionContext);
+  return dbModule;
+}

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -0,0 +1,70 @@
+import { DbConfig, RemoteRepositoryList } from './db-config';
+import {
+  DbItemKind,
+  RemoteOwnerDbItem,
+  RemoteRepoDbItem,
+  RemoteSystemDefinedListDbItem,
+  RemoteUserDefinedListDbItem,
+  RootLocalDbItem,
+  RootRemoteDbItem
+} from './db-item';
+
+export function createRemoteTree(dbConfig: DbConfig): RootRemoteDbItem {
+  const systemDefinedLists = [
+    createSystemDefinedList(10),
+    createSystemDefinedList(100),
+    createSystemDefinedList(1000)
+  ];
+
+  const userDefinedRepoLists = dbConfig.remote.repositoryLists.map(createUserDefinedList);
+  const owners = dbConfig.remote.owners.map(createOwnerItem);
+  const repos = dbConfig.remote.repositories.map(createRepoItem);
+
+  return {
+    kind: DbItemKind.RootRemote,
+    children: [
+      ...systemDefinedLists,
+      ...owners,
+      ...userDefinedRepoLists,
+      ...repos
+    ]
+  };
+}
+
+export function createLocalTree(): RootLocalDbItem {
+  // This will be fleshed out further in the future.
+  return {
+    kind: DbItemKind.RootLocal
+  };
+}
+
+function createSystemDefinedList(n: number): RemoteSystemDefinedListDbItem {
+  return {
+    kind: DbItemKind.RemoteSystemDefinedList,
+    listName: `top_${n}`,
+    listDisplayName: `Top ${n} repositories`,
+    listDescription: `Top ${n} repositories of a language`
+  };
+}
+
+function createUserDefinedList(list: RemoteRepositoryList): RemoteUserDefinedListDbItem {
+  return {
+    kind: DbItemKind.RemoteUserDefinedList,
+    listName: list.name,
+    repos: list.repositories.map((r) => createRepoItem(r))
+  };
+}
+
+function createOwnerItem(owner: string): RemoteOwnerDbItem {
+  return {
+    kind: DbItemKind.RemoteOwner,
+    ownerName: owner
+  };
+}
+
+function createRepoItem(repo: string): RemoteRepoDbItem {
+  return {
+    kind: DbItemKind.RemoteRepo,
+    repoFullName: repo
+  };
+}

extensions/ql-vscode/src/databases/ui/db-item-mapper.ts

@@ -0,0 +1,49 @@
+import { DbItem, DbItemKind } from '../db-item';
+import {
+  createDbTreeViewItemOwner,
+  createDbTreeViewItemRepo,
+  createDbTreeViewItemRoot,
+  createDbTreeViewItemSystemDefinedList,
+  createDbTreeViewItemUserDefinedList,
+  DbTreeViewItem
+} from './db-tree-view-item';
+
+export function mapDbItemToTreeViewItem(dbItem: DbItem): DbTreeViewItem {
+  switch (dbItem.kind) {
+    case DbItemKind.RootLocal:
+      return createDbTreeViewItemRoot(
+        dbItem,
+        'local',
+        'Local databases',
+        []);
+
+    case DbItemKind.RootRemote:
+      return createDbTreeViewItemRoot(
+        dbItem,
+        'remote',
+        'Remote databases',
+        dbItem.children.map(c => mapDbItemToTreeViewItem(c)));
+
+    case DbItemKind.RemoteSystemDefinedList:
+      return createDbTreeViewItemSystemDefinedList(
+        dbItem,
+        dbItem.listDisplayName,
+        dbItem.listDescription);
+
+    case DbItemKind.RemoteUserDefinedList:
+      return createDbTreeViewItemUserDefinedList(
+        dbItem,
+        dbItem.listName,
+        dbItem.repos.map(mapDbItemToTreeViewItem));
+
+    case DbItemKind.RemoteOwner:
+      return createDbTreeViewItemOwner(
+        dbItem,
+        dbItem.ownerName);
+
+    case DbItemKind.RemoteRepo:
+      return createDbTreeViewItemRepo(
+        dbItem,
+        dbItem.repoFullName);
+  }
+}

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -0,0 +1,39 @@
+import * as vscode from 'vscode';
+import { commandRunner } from '../../commandRunner';
+import { DisposableObject } from '../../pure/disposable-object';
+import { DbManager } from '../db-manager';
+import { DbTreeDataProvider } from './db-tree-data-provider';
+
+export class DbPanel extends DisposableObject {
+  private readonly dataProvider: DbTreeDataProvider;
+
+  public constructor(
+    private readonly dbManager: DbManager
+  ) {
+    super();
+
+    this.dataProvider = new DbTreeDataProvider(dbManager);
+
+    const treeView = vscode.window.createTreeView('codeQLDatabasesExperimental', {
+      treeDataProvider: this.dataProvider,
+      canSelectMany: false
+    });
+
+    this.push(treeView);
+  }
+
+  public async initialize(): Promise<void> {
+    this.push(
+      commandRunner(
+        'codeQLDatabasesExperimental.openConfigFile',
+        () => this.openConfigFile(),
+      )
+    );
+  }
+
+  private async openConfigFile(): Promise<void> {
+    const configPath = this.dbManager.getConfigPath();
+    const document = await vscode.workspace.openTextDocument(configPath);
+    await vscode.window.showTextDocument(document);
+  }
+}

extensions/ql-vscode/src/databases/ui/db-tree-data-provider.ts

@@ -0,0 +1,49 @@
+import { ProviderResult, TreeDataProvider, TreeItem } from 'vscode';
+import { createDbTreeViewItemWarning, DbTreeViewItem } from './db-tree-view-item';
+import { DbManager } from '../db-manager';
+import { mapDbItemToTreeViewItem } from './db-item-mapper';
+
+export class DbTreeDataProvider implements TreeDataProvider<DbTreeViewItem> {
+  private dbTreeItems: DbTreeViewItem[];
+
+  public constructor(
+    private readonly dbManager: DbManager
+  ) {
+    this.dbTreeItems = this.createTree();
+  }
+
+  /**
+   * Called when expanding a node (including the root node).
+   * @param node The node to expand.
+   * @returns The children of the node.
+   */
+  public getChildren(node?: DbTreeViewItem): ProviderResult<DbTreeViewItem[]> {
+    if (!node) {
+      // We're at the root.
+      return Promise.resolve(this.dbTreeItems);
+    } else {
+      return Promise.resolve(node.children);
+    }
+  }
+
+  /**
+   * Returns the UI presentation of the element that gets displayed in the view.
+   * @param node The node to represent.
+   * @returns The UI presentation of the node.
+   */
+  public getTreeItem(node: DbTreeViewItem): TreeItem | Thenable<TreeItem> {
+    return node;
+  }
+
+  private createTree(): DbTreeViewItem[] {
+    const dbItems = this.dbManager.getDbItems();
+
+    // Add a sample warning as a proof of concept.
+    const warningTreeViewItem = createDbTreeViewItemWarning(
+      'There was an error',
+      'Fix it'
+    );
+
+    return [...dbItems.map(mapDbItemToTreeViewItem), warningTreeViewItem];
+  }
+}

extensions/ql-vscode/src/databases/ui/db-tree-view-item.ts

@@ -0,0 +1,107 @@
+import * as vscode from 'vscode';
+import {
+  DbItem,
+  RemoteOwnerDbItem,
+  RemoteRepoDbItem,
+  RemoteSystemDefinedListDbItem,
+  RemoteUserDefinedListDbItem,
+  RootLocalDbItem,
+  RootRemoteDbItem
+} from '../db-item';
+
+/**
+ * Represents an item in the database tree view. This item could be 
+ * representing an actual database item or a warning.
+ */
+export class DbTreeViewItem extends vscode.TreeItem {
+  constructor(
+    public readonly dbItem: DbItem | undefined,
+    public readonly iconPath: vscode.ThemeIcon | undefined,
+    public readonly label: string,
+    public readonly tooltip: string | undefined,
+    public readonly collapsibleState: vscode.TreeItemCollapsibleState,
+    public readonly children: DbTreeViewItem[]
+  ) {
+    super(label, collapsibleState);
+  }
+}
+
+export function createDbTreeViewItemWarning(label: string, tooltip: string): DbTreeViewItem {
+  return new DbTreeViewItem(
+    undefined,
+    new vscode.ThemeIcon('warning', new vscode.ThemeColor('problemsWarningIcon.foreground')),
+    label,
+    tooltip,
+    vscode.TreeItemCollapsibleState.None,
+    []
+  );
+}
+
+export function createDbTreeViewItemRoot(
+  dbItem: RootLocalDbItem | RootRemoteDbItem,
+  label: string,
+  tooltip: string,
+  children: DbTreeViewItem[]
+): DbTreeViewItem {
+  return new DbTreeViewItem(
+    dbItem,
+    undefined,
+    label,
+    tooltip,
+    vscode.TreeItemCollapsibleState.Collapsed,
+    children);
+}
+
+export function createDbTreeViewItemSystemDefinedList(
+  dbItem: RemoteSystemDefinedListDbItem,
+  label: string,
+  tooltip: string
+): DbTreeViewItem {
+  return new DbTreeViewItem(
+    dbItem,
+    new vscode.ThemeIcon('github'),
+    label,
+    tooltip,
+    vscode.TreeItemCollapsibleState.None,
+    []);
+}
+
+export function createDbTreeViewItemUserDefinedList(
+  dbItem: RemoteUserDefinedListDbItem,
+  listName: string,
+  children: DbTreeViewItem[]
+): DbTreeViewItem {
+  return new DbTreeViewItem(
+    dbItem,
+    undefined,
+    listName,
+    undefined,
+    vscode.TreeItemCollapsibleState.Collapsed,
+    children);
+}
+
+export function createDbTreeViewItemOwner(
+  dbItem: RemoteOwnerDbItem,
+  ownerName: string,
+): DbTreeViewItem {
+  return new DbTreeViewItem(
+    dbItem,
+    new vscode.ThemeIcon('organization'),
+    ownerName,
+    undefined,
+    vscode.TreeItemCollapsibleState.None,
+    []);
+}
+
+export function createDbTreeViewItemRepo(
+  dbItem: RemoteRepoDbItem,
+  repoName: string,
+): DbTreeViewItem {
+  return new DbTreeViewItem(
+    dbItem,
+    new vscode.ThemeIcon('database'),
+    repoName,
+    undefined,
+    vscode.TreeItemCollapsibleState.None,
+    []);
+}

extensions/ql-vscode/src/eval-log-tree-builder.ts

@@ -0,0 +1,67 @@
+import { ChildEvalLogTreeItem, EvalLogTreeItem } from './eval-log-viewer';
+import { EvalLogData as EvalLogData } from './pure/log-summary-parser';
+
+/** Builds the tree data for the evaluator log viewer for a single query run. */
+export default class EvalLogTreeBuilder {
+  private queryName: string;
+  private evalLogDataItems: EvalLogData[];
+
+  constructor(queryName: string, evaluatorLogDataItems: EvalLogData[]) {
+    this.queryName = queryName;
+    this.evalLogDataItems = evaluatorLogDataItems;
+  }
+
+  async getRoots(): Promise<EvalLogTreeItem[]> {
+    return await this.parseRoots();
+  }
+
+  private async parseRoots(): Promise<EvalLogTreeItem[]> {
+    const roots: EvalLogTreeItem[] = [];
+
+    // Once the viewer can show logs for multiple queries, there will be more than 1 item at the root
+    // level. For now, there will always be one root (the one query being shown).
+    const queryItem: EvalLogTreeItem = {
+      label: this.queryName,
+      children: [] // Will assign predicate items as children shortly.
+    };
+
+    // Display descriptive message when no data exists 
+    if (this.evalLogDataItems.length === 0) {
+      const noResultsItem: ChildEvalLogTreeItem = {
+        label: 'No predicates evaluated in this query run.',
+        parent: queryItem,
+        children: [],
+      };
+      queryItem.children.push(noResultsItem);
+    }
+
+    // For each predicate, create a TreeItem object with appropriate parents/children 
+    this.evalLogDataItems.forEach(logDataItem => {
+      const predicateLabel = `${logDataItem.predicateName} (${logDataItem.resultSize} tuples, ${logDataItem.millis} ms)`;
+      const predicateItem: ChildEvalLogTreeItem = {
+        label: predicateLabel,
+        parent: queryItem,
+        children: [] // Will assign pipeline items as children shortly.
+      };
+      for (const [pipelineName, steps] of Object.entries(logDataItem.ra)) {
+        const pipelineLabel = `Pipeline: ${pipelineName}`;
+        const pipelineItem: ChildEvalLogTreeItem = {
+          label: pipelineLabel,
+          parent: predicateItem,
+          children: [] // Will assign step items as children shortly.
+        };
+        predicateItem.children.push(pipelineItem);
+
+        pipelineItem.children = steps.map((step: string) => ({
+          label: step,
+          parent: pipelineItem,
+          children: []
+        }));
+      }
+      queryItem.children.push(predicateItem);
+    });
+
+    roots.push(queryItem);
+    return roots;
+  }
+}

extensions/ql-vscode/src/eval-log-viewer.ts

@@ -0,0 +1,92 @@
+import { window, TreeDataProvider, TreeView, TreeItem, ProviderResult, Event, EventEmitter, TreeItemCollapsibleState } from 'vscode';
+import { commandRunner } from './commandRunner';
+import { DisposableObject } from './pure/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
+
+export interface EvalLogTreeItem {
+  label?: string;
+  children: ChildEvalLogTreeItem[];
+}
+
+export interface ChildEvalLogTreeItem extends EvalLogTreeItem {
+  parent: ChildEvalLogTreeItem | EvalLogTreeItem;
+}
+
+/** Provides data from parsed CodeQL evaluator logs to be rendered in a tree view. */
+class EvalLogDataProvider extends DisposableObject implements TreeDataProvider<EvalLogTreeItem> {
+  public roots: EvalLogTreeItem[] = [];
+
+  private _onDidChangeTreeData: EventEmitter<EvalLogTreeItem | undefined | null | void> = new EventEmitter<EvalLogTreeItem | undefined | null | void>();
+  readonly onDidChangeTreeData: Event<EvalLogTreeItem | undefined | null | void> = this._onDidChangeTreeData.event;
+
+  refresh(): void {
+    this._onDidChangeTreeData.fire();
+  }
+
+  getTreeItem(element: EvalLogTreeItem): TreeItem | Thenable<TreeItem> {
+    const state = element.children.length
+      ? TreeItemCollapsibleState.Collapsed
+      : TreeItemCollapsibleState.None;
+    const treeItem = new TreeItem(element.label || '', state);
+    treeItem.tooltip = `${treeItem.label} || ''}`;
+    return treeItem;
+  }
+
+  getChildren(element?: EvalLogTreeItem): ProviderResult<EvalLogTreeItem[]> {
+    // If no item is passed, return the root.
+    if (!element) {
+      return this.roots || [];
+    }
+    // Otherwise it is called with an existing item, to load its children.
+    return element.children;
+  }
+
+  getParent(element: ChildEvalLogTreeItem): ProviderResult<EvalLogTreeItem> {
+    return element.parent;
+  }
+}
+
+/** Manages a tree viewer of structured evaluator logs. */
+export class EvalLogViewer extends DisposableObject {
+  private treeView: TreeView<EvalLogTreeItem>;
+  private treeDataProvider: EvalLogDataProvider;
+
+  constructor() {
+    super();
+
+    this.treeDataProvider = new EvalLogDataProvider();
+    this.treeView = window.createTreeView('codeQLEvalLogViewer', {
+      treeDataProvider: this.treeDataProvider,
+      showCollapseAll: true
+    });
+
+    this.push(this.treeView);
+    this.push(this.treeDataProvider);
+    this.push(
+      commandRunner('codeQLEvalLogViewer.clear', async () => {
+        this.clear();
+      })
+    );
+  }
+
+  private clear(): void {
+    this.treeDataProvider.roots = [];
+    this.treeDataProvider.refresh();
+    this.treeView.message = undefined;
+  }
+
+  // Called when the Show Evaluator Log (UI) command is run on a new query.
+  updateRoots(roots: EvalLogTreeItem[]): void {
+    this.treeDataProvider.roots = roots;
+    this.treeDataProvider.refresh();
+
+    this.treeView.message = 'Viewer for query run:'; // Currently only one query supported at a time. 
+
+    // Handle error on reveal. This could happen if
+    // the tree view is disposed during the reveal.
+    this.treeView.reveal(roots[0], { focus: false })?.then(
+      () => { /**/ },
+      err => showAndLogErrorMessage(err)
+    );
+  }
+}

extensions/ql-vscode/src/extension.ts

@@ -1,5 +1,7 @@
+import 'source-map-support/register';
 import {
   CancellationToken,
+  CancellationTokenSource,
   commands,
   Disposable,
   ExtensionContext,
@@ -18,6 +20,7 @@ import {
 } from 'vscode';
 import { LanguageClient } from 'vscode-languageclient';
 import * as os from 'os';
+import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as tmp from 'tmp-promise';
 import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
@@ -30,6 +33,7 @@ import {
   CliConfigListener,
   DistributionConfigListener,
   isCanary,
+  joinOrderWarningThreshold,
   MAX_QUERIES,
   QueryHistoryConfigListener,
   QueryServerConfigListener
@@ -40,7 +44,8 @@ import { DatabaseUI } from './databases-ui';
 import {
   TemplateQueryDefinitionProvider,
   TemplateQueryReferenceProvider,
-  TemplatePrintAstProvider
+  TemplatePrintAstProvider,
+  TemplatePrintCfgProvider
 } from './contextual/templateProvider';
 import {
   DEFAULT_DISTRIBUTION_VERSION_RANGE,
@@ -52,20 +57,29 @@ import {
   GithubApiError,
   GithubRateLimitedError
 } from './distribution';
-import * as helpers from './helpers';
-import { assertNever } from './pure/helpers-pure';
+import {
+  findLanguage,
+  tmpDirDisposal,
+  showBinaryChoiceDialog,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  showAndLogInformationMessage,
+  showInformationMessageWithAction,
+  tmpDir
+} from './helpers';
+import { asError, assertNever, getErrorMessage } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
-import { InterfaceManager } from './interface';
+import { ResultsView } from './interface';
 import { WebviewReveal } from './interface-utils';
-import { ideServerLogger, logger, queryServerLogger } from './logging';
+import { ideServerLogger, logger, ProgressReporter, queryServerLogger } from './logging';
 import { QueryHistoryManager } from './query-history';
-import { CompletedQuery } from './query-results';
-import * as qsClient from './queryserver-client';
+import { CompletedLocalQueryInfo, LocalQueryInfo } from './query-results';
+import * as legacyQueryServer from './legacy-query-server/queryserver-client';
+import * as newQueryServer from './query-server/queryserver-client';
 import { displayQuickQuery } from './quick-query';
-import { compileAndRunQueryAgainstDatabase, tmpDirDisposal } from './run-queries';
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
-import { CompareInterfaceManager } from './compare/compare-interface';
+import { CompareView } from './compare/compare-view';
 import { gatherQlFiles } from './pure/files';
 import { initializeTelemetry } from './telemetry';
 import {
@@ -79,12 +93,28 @@ import { CodeQlStatusBarHandler } from './status-bar';
 
 import { Credentials } from './authentication';
 import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
-import { RemoteQuery } from './remote-queries/remote-query';
+import { RemoteQueryResult } from './remote-queries/remote-query-result';
 import { URLSearchParams } from 'url';
-import { RemoteQueriesInterfaceManager } from './remote-queries/remote-queries-interface';
-import { sampleRemoteQuery, sampleRemoteQueryResult } from './remote-queries/sample-data';
 import { handleDownloadPacks, handleInstallPackDependencies } from './packaging';
-import { AnalysesResultsManager } from './remote-queries/analyses-results-manager';
+import { HistoryItemLabelProvider } from './history-item-label-provider';
+import { exportRemoteQueryResults } from './remote-queries/export-results';
+import { RemoteQuery } from './remote-queries/remote-query';
+import { EvalLogViewer } from './eval-log-viewer';
+import { SummaryLanguageSupport } from './log-insights/summary-language-support';
+import { JoinOrderScannerProvider } from './log-insights/join-order';
+import { LogScannerService } from './log-insights/log-scanner-service';
+import { createInitialQueryInfo } from './run-queries-shared';
+import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
+import { NewQueryRunner } from './query-server/query-runner';
+import { QueryRunner } from './queryRunner';
+import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
+import { VariantAnalysisViewSerializer } from './remote-queries/variant-analysis-view-serializer';
+import { VariantAnalysis, VariantAnalysisScannedRepository } from './remote-queries/shared/variant-analysis';
+import { VariantAnalysisManager } from './remote-queries/variant-analysis-manager';
+import { createVariantAnalysisContentProvider } from './remote-queries/variant-analysis-content-provider';
+import { VSCodeMockGitHubApiServer } from './mocks/vscode-mock-gh-api-server';
+import { VariantAnalysisResultsManager } from './remote-queries/variant-analysis-results-manager';
+import { initializeDbModule } from './databases/db-module';
 
 /**
  * extension.ts
@@ -147,10 +177,11 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 export interface CodeQLExtensionInterface {
   readonly ctx: ExtensionContext;
   readonly cliServer: CodeQLCliServer;
-  readonly qs: qsClient.QueryServerClient;
+  readonly qs: QueryRunner;
   readonly distributionManager: DistributionManager;
   readonly databaseManager: DatabaseManager;
   readonly databaseUI: DatabaseUI;
+  readonly variantAnalysisManager: VariantAnalysisManager;
   readonly dispose: () => void;
 }
 
@@ -186,7 +217,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
   const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
   registerErrorStubs([checkForUpdatesCommand], command => (async () => {
-    void helpers.showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
+    void showAndLogErrorMessage(`Can't execute ${command}: waiting to finish loading CodeQL CLI.`);
   }));
 
   interface DistributionUpdateConfig {
@@ -198,7 +229,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
   async function installOrUpdateDistributionWithProgressTitle(progressTitle: string, config: DistributionUpdateConfig): Promise<void> {
     const minSecondsSinceLastUpdateCheck = config.isUserInitiated ? 0 : 86400;
     const noUpdatesLoggingFunc = config.shouldDisplayMessageWhenNoUpdates ?
-      helpers.showAndLogInformationMessage : async (message: string) => void logger.log(message);
+      showAndLogInformationMessage : async (message: string) => void logger.log(message);
     const result = await distributionManager.checkForUpdatesToExtensionManagedDistribution(minSecondsSinceLastUpdateCheck);
 
     // We do want to auto update if there is no distribution at all
@@ -220,7 +251,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
           const updateAvailableMessage = `Version "${result.updatedRelease.name}" of the CodeQL CLI is now available. ` +
             'Do you wish to upgrade?';
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, true);
-          if (await helpers.showInformationMessageWithAction(updateAvailableMessage, 'Restart and Upgrade')) {
+          if (await showInformationMessageWithAction(updateAvailableMessage, 'Restart and Upgrade')) {
             await commands.executeCommand('workbench.action.reloadWindow');
           }
         } else {
@@ -233,7 +264,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
             distributionManager.installExtensionManagedDistributionRelease(result.updatedRelease, progress));
 
           await ctx.globalState.update(shouldUpdateOnNextActivationKey, false);
-          void helpers.showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
+          void showAndLogInformationMessage(`CodeQL CLI updated to version "${result.updatedRelease.name}".`);
         }
         break;
       default:
@@ -260,7 +291,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
       // Don't rethrow the exception, because if the config is changed, we want to be able to retry installing
       // or updating the distribution.
       const alertFunction = (codeQlInstalled && !config.isUserInitiated) ?
-        helpers.showAndLogWarningMessage : helpers.showAndLogErrorMessage;
+        showAndLogWarningMessage : showAndLogErrorMessage;
       const taskDescription = (willUpdateCodeQl ? 'update' :
         codeQlInstalled ? 'check for updates to' : 'install') + ' CodeQL CLI';
 
@@ -295,20 +326,20 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
           }
         })();
 
-        void helpers.showAndLogWarningMessage(
+        void showAndLogWarningMessage(
           `The current version of the CodeQL CLI (${result.version.raw}) ` +
           `is incompatible with this extension. ${fixGuidanceMessage}`
         );
         break;
       }
       case FindDistributionResultKind.UnknownCompatibilityDistribution:
-        void helpers.showAndLogWarningMessage(
+        void showAndLogWarningMessage(
           'Compatibility with the configured CodeQL CLI could not be determined. ' +
           'You may experience problems using the extension.'
         );
         break;
       case FindDistributionResultKind.NoDistribution:
-        void helpers.showAndLogErrorMessage('The CodeQL CLI could not be found.');
+        void showAndLogErrorMessage('The CodeQL CLI could not be found.');
         break;
       default:
         assertNever(result);
@@ -335,7 +366,7 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = 'Install CodeQL CLI';
-        const chosenAction = await void helpers.showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
+        const chosenAction = await void showAndLogErrorMessage(`Can't execute ${command}: missing CodeQL CLI.`, {
           items: [installActionName]
         });
         if (chosenAction === installActionName) {
@@ -361,7 +392,10 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     allowAutoUpdating: true
   })));
 
-  return await installOrUpdateThenTryActivate({
+  const variantAnalysisViewSerializer = new VariantAnalysisViewSerializer(ctx);
+  Window.registerWebviewPanelSerializer(VariantAnalysisView.viewType, variantAnalysisViewSerializer);
+
+  const codeQlExtension = await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
 
@@ -369,8 +403,14 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     // otherwise, ask user to accept the update
     allowAutoUpdating: !!ctx.globalState.get(shouldUpdateOnNextActivationKey)
   });
+
+  variantAnalysisViewSerializer.onExtensionLoaded(codeQlExtension.variantAnalysisManager);
+
+  return codeQlExtension;
 }
 
+const PACK_GLOBS = ['**/codeql-pack.yml', '**/qlpack.yml', '**/queries.xml', '**/codeql-pack.lock.yml', '**/qlpack.lock.yml', '.codeqlmanifest.json', 'codeql-workspace.yml'];
+
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
   distributionManager: DistributionManager,
@@ -399,83 +439,119 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(statusBar);
 
   void logger.log('Initializing query server client.');
-  const qs = new qsClient.QueryServerClient(
-    qlConfigurationListener,
-    cliServer,
-    {
-      logger: queryServerLogger,
-      contextStoragePath: getContextStoragePath(ctx),
-    },
-    (task) =>
-      Window.withProgress(
-        { title: 'CodeQL query server', location: ProgressLocation.Window },
-        task
-      )
-  );
-  ctx.subscriptions.push(qs);
-  await qs.startQueryServer();
+  const qs = await createQueryServer(qlConfigurationListener, cliServer, ctx);
+
+
+  for (const glob of PACK_GLOBS) {
+    const fsWatcher = workspace.createFileSystemWatcher(glob);
+    ctx.subscriptions.push(fsWatcher);
+    fsWatcher.onDidChange(async (_uri) => {
+      await qs.clearPackCache();
+    });
+  }
 
   void logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
+
+  // Let this run async.
+  void dbm.loadPersistedState();
+
   ctx.subscriptions.push(dbm);
   void logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(
     dbm,
     qs,
     getContextStoragePath(ctx),
-    ctx.extensionPath
+    ctx.extensionPath,
+    () => Credentials.initialize(ctx),
   );
   databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
+  void logger.log('Initializing evaluator log viewer.');
+  const evalLogViewer = new EvalLogViewer();
+  ctx.subscriptions.push(evalLogViewer);
+
   void logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
   ctx.subscriptions.push(queryHistoryConfigurationListener);
-  const showResults = async (item: CompletedQuery) =>
+  const showResults = async (item: CompletedLocalQueryInfo) =>
     showResultsForCompletedQuery(item, WebviewReveal.Forced);
+  const queryStorageDir = path.join(ctx.globalStorageUri.fsPath, 'queries');
+  await fs.ensureDir(queryStorageDir);
+  const labelProvider = new HistoryItemLabelProvider(queryHistoryConfigurationListener);
 
+  void logger.log('Initializing results panel interface.');
+  const localQueryResultsView = new ResultsView(ctx, dbm, cliServer, queryServerLogger, labelProvider);
+  ctx.subscriptions.push(localQueryResultsView);
+
+  void logger.log('Initializing variant analysis manager.');
+  const variantAnalysisStorageDir = path.join(ctx.globalStorageUri.fsPath, 'variant-analyses');
+  await fs.ensureDir(variantAnalysisStorageDir);
+  const variantAnalysisResultsManager = new VariantAnalysisResultsManager(cliServer, logger);
+  const variantAnalysisManager = new VariantAnalysisManager(ctx, variantAnalysisStorageDir, variantAnalysisResultsManager);
+  ctx.subscriptions.push(variantAnalysisManager);
+  ctx.subscriptions.push(variantAnalysisResultsManager);
+  ctx.subscriptions.push(workspace.registerTextDocumentContentProvider('codeql-variant-analysis', createVariantAnalysisContentProvider(variantAnalysisManager)));
+
+  void logger.log('Initializing remote queries manager.');
+  const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger, variantAnalysisManager);
+  ctx.subscriptions.push(rqm);
+
+  void logger.log('Initializing query history.');
   const qhm = new QueryHistoryManager(
     qs,
-    ctx.extensionPath,
+    dbm,
+    localQueryResultsView,
+    rqm,
+    variantAnalysisManager,
+    evalLogViewer,
+    queryStorageDir,
+    ctx,
     queryHistoryConfigurationListener,
-    showResults,
-    async (from: CompletedQuery, to: CompletedQuery) =>
+    labelProvider,
+    async (from: CompletedLocalQueryInfo, to: CompletedLocalQueryInfo) =>
       showResultsForComparison(from, to),
   );
-  ctx.subscriptions.push(qhm);
-  void logger.log('Initializing results panel interface.');
-  const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
-  ctx.subscriptions.push(intm);
 
-  void logger.log('Initializing compare panel interface.');
-  const cmpm = new CompareInterfaceManager(
+
+  ctx.subscriptions.push(qhm);
+
+  void logger.log('Initializing evaluation log scanners.');
+  const logScannerService = new LogScannerService(qhm);
+  ctx.subscriptions.push(logScannerService);
+  ctx.subscriptions.push(logScannerService.scanners.registerLogScannerProvider(new JoinOrderScannerProvider(() => joinOrderWarningThreshold())));
+
+  void logger.log('Initializing compare view.');
+  const compareView = new CompareView(
     ctx,
     dbm,
     cliServer,
     queryServerLogger,
+    labelProvider,
     showResults
   );
-  ctx.subscriptions.push(cmpm);
+  ctx.subscriptions.push(compareView);
 
   void logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
 
   async function showResultsForComparison(
-    from: CompletedQuery,
-    to: CompletedQuery
+    from: CompletedLocalQueryInfo,
+    to: CompletedLocalQueryInfo
   ): Promise<void> {
     try {
-      await cmpm.showResults(from, to);
+      await compareView.showResults(from, to);
     } catch (e) {
-      void helpers.showAndLogErrorMessage(e.message);
+      void showAndLogErrorMessage(getErrorMessage(e));
     }
   }
 
   async function showResultsForCompletedQuery(
-    query: CompletedQuery,
+    query: CompletedLocalQueryInfo,
     forceReveal: WebviewReveal
   ): Promise<void> {
-    await intm.showResults(query, forceReveal, false);
+    await localQueryResultsView.showResults(query, forceReveal, false);
   }
 
   async function compileAndRunQuery(
@@ -492,22 +568,41 @@ async function activateWithInstalledDistribution(
       if (databaseItem === undefined) {
         throw new Error('Can\'t run query without a selected database');
       }
-      const info = await compileAndRunQueryAgainstDatabase(
-        cliServer,
-        qs,
-        databaseItem,
-        quickEval,
-        selectedQuery,
-        progress,
-        token,
-        undefined,
-        range
-      );
-      const item = qhm.buildCompletedQuery(info);
-      await showResultsForCompletedQuery(item, WebviewReveal.NotForced);
-      // Note we must update the query history view after showing results as the
-      // display and sorting might depend on the number of results
-      await qhm.addCompletedQuery(item);
+      const databaseInfo = {
+        name: databaseItem.name,
+        databaseUri: databaseItem.databaseUri.toString(),
+      };
+
+      // handle cancellation from the history view.
+      const source = new CancellationTokenSource();
+      token.onCancellationRequested(() => source.cancel());
+
+      const initialInfo = await createInitialQueryInfo(selectedQuery, databaseInfo, quickEval, range);
+      const item = new LocalQueryInfo(initialInfo, source);
+      qhm.addQuery(item);
+      try {
+        const completedQueryInfo = await qs.compileAndRunQueryAgainstDatabase(
+          databaseItem,
+          initialInfo,
+          queryStorageDir,
+          progress,
+          source.token,
+          undefined,
+          item,
+        );
+        qhm.completeQuery(item, completedQueryInfo);
+        await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.Forced);
+        // Note we must update the query history view after showing results as the
+        // display and sorting might depend on the number of results
+      } catch (e) {
+        const err = asError(e);
+        err.message = `Error running query: ${err.message}`;
+        item.failureReason = err.message;
+        throw e;
+      } finally {
+        await qhm.refreshTreeView();
+        source.dispose();
+      }
     }
   }
 
@@ -527,11 +622,11 @@ async function activateWithInstalledDistribution(
       try {
         await cliServer.generateQueryHelp(pathToQhelp, absolutePathToMd);
         await commands.executeCommand('markdown.showPreviewToSide', uri);
-      } catch (err) {
-        const errorMessage = err.message.includes('Generating qhelp in markdown') ? (
+      } catch (e) {
+        const errorMessage = getErrorMessage(e).includes('Generating qhelp in markdown') ? (
           `Could not generate markdown from ${pathToQhelp}: Bad formatting in .qhelp file.`
         ) : `Could not open a preview of the generated file (${absolutePathToMd}).`;
-        void helpers.showAndLogErrorMessage(errorMessage, { fullMessage: `${errorMessage}\n${err}` });
+        void showAndLogErrorMessage(errorMessage, { fullMessage: `${errorMessage}\n${e}` });
       }
     }
 
@@ -548,7 +643,7 @@ async function activateWithInstalledDistribution(
         const uri = Uri.file(resolved.resolvedPath);
         await window.showTextDocument(uri, { preview: false });
       } else {
-        void helpers.showAndLogErrorMessage(
+        void showAndLogErrorMessage(
           'Jumping from a .qlref file to the .ql file it references is not '
           + 'supported with the CLI version you are running.\n'
           + `Please upgrade your CLI to version ${CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF
@@ -602,7 +697,10 @@ async function activateWithInstalledDistribution(
       {
         title: 'Running query',
         cancellable: true
-      }
+      },
+
+      // Open the query server logger on error since that's usually where the interesting errors appear.
+      queryServerLogger
     )
   );
   interface DatabaseQuickPickItem extends QuickPickItem {
@@ -618,15 +716,15 @@ async function activateWithInstalledDistribution(
       ) => {
         let filteredDBs = dbm.databaseItems;
         if (filteredDBs.length === 0) {
-          void helpers.showAndLogErrorMessage('No databases found. Please add a suitable database to your workspace.');
+          void showAndLogErrorMessage('No databases found. Please add a suitable database to your workspace.');
           return;
         }
         // If possible, only show databases with the right language (otherwise show all databases).
-        const queryLanguage = await helpers.findLanguage(cliServer, uri);
+        const queryLanguage = await findLanguage(cliServer, uri);
         if (queryLanguage) {
           filteredDBs = dbm.databaseItems.filter(db => db.language === queryLanguage);
           if (filteredDBs.length === 0) {
-            void helpers.showAndLogErrorMessage(`No databases found for language ${queryLanguage}. Please add a suitable database to your workspace.`);
+            void showAndLogErrorMessage(`No databases found for language ${queryLanguage}. Please add a suitable database to your workspace.`);
             return;
           }
         }
@@ -651,19 +749,19 @@ async function activateWithInstalledDistribution(
           for (const item of quickpick) {
             try {
               await compileAndRunQuery(false, uri, progress, token, item.databaseItem);
-            } catch (error) {
+            } catch (e) {
               skippedDatabases.push(item.label);
-              errors.push(error.message);
+              errors.push(getErrorMessage(e));
             }
           }
           if (skippedDatabases.length > 0) {
             void logger.log(`Errors:\n${errors.join('\n')}`);
-            void helpers.showAndLogWarningMessage(
+            void showAndLogWarningMessage(
               `The following databases were skipped:\n${skippedDatabases.join('\n')}.\nFor details about the errors, see the logs.`
             );
           }
         } else {
-          void helpers.showAndLogErrorMessage('No databases selected.');
+          void showAndLogErrorMessage('No databases selected.');
         }
       },
       {
@@ -690,7 +788,7 @@ async function activateWithInstalledDistribution(
         // files may be hidden from the user.
         if (dirFound) {
           const fileString = files.map(file => path.basename(file)).join(', ');
-          const res = await helpers.showBinaryChoiceDialog(
+          const res = await showBinaryChoiceDialog(
             `You are about to run ${files.length} queries: ${fileString} Do you want to continue?`
           );
           if (!res) {
@@ -711,12 +809,13 @@ async function activateWithInstalledDistribution(
           });
         }
 
-        if (queryUris.length > 1) {
+        if (queryUris.length > 1 && !await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
           // Try to upgrade the current database before running any queries
           // so that the user isn't confronted with multiple upgrade
           // requests for each query to run.
           // Only do it if running multiple queries since this check is
           // performed on each query run anyway.
+          // Don't do this with non destructive upgrades as the user won't see anything anyway.
           await databaseUI.tryUpgradeCurrentDatabase(progress, token);
         }
 
@@ -734,7 +833,11 @@ async function activateWithInstalledDistribution(
       {
         title: 'Running queries',
         cancellable: true
-      })
+      },
+
+      // Open the query server logger on error since that's usually where the interesting errors appear.
+      queryServerLogger
+    )
   );
   ctx.subscriptions.push(
     commandRunnerWithProgress(
@@ -747,7 +850,10 @@ async function activateWithInstalledDistribution(
       {
         title: 'Running query',
         cancellable: true
-      })
+      },
+      // Open the query server logger on error since that's usually where the interesting errors appear.
+      queryServerLogger
+    )
   );
 
   ctx.subscriptions.push(
@@ -762,7 +868,11 @@ async function activateWithInstalledDistribution(
       {
         title: 'Running query',
         cancellable: true
-      })
+      },
+
+      // Open the query server logger on error since that's usually where the interesting errors appear.
+      queryServerLogger
+    )
   );
 
   ctx.subscriptions.push(
@@ -773,18 +883,19 @@ async function activateWithInstalledDistribution(
       displayQuickQuery(ctx, cliServer, databaseUI, progress, token),
       {
         title: 'Run Quick Query'
-      }
+      },
+
+      // Open the query server logger on error since that's usually where the interesting errors appear.
+      queryServerLogger
     )
   );
 
-  void logger.log('Initializing remote queries interface.');
-  const rqm = new RemoteQueriesManager(ctx, logger, cliServer);
 
   registerRemoteQueryTextProvider();
 
-  // The "runRemoteQuery" command is internal-only.
+  // The "runVariantAnalysis" command is internal-only.
   ctx.subscriptions.push(
-    commandRunnerWithProgress('codeQL.runRemoteQuery', async (
+    commandRunnerWithProgress('codeQL.runVariantAnalysis', async (
       progress: ProgressCallback,
       token: CancellationToken,
       uri: Uri | undefined
@@ -801,28 +912,87 @@ async function activateWithInstalledDistribution(
           token
         );
       } else {
-        throw new Error('Remote queries require the CodeQL Canary version to run.');
+        throw new Error('Variant analysis requires the CodeQL Canary version to run.');
       }
     }, {
-      title: 'Run Remote Query',
+      title: 'Run Variant Analysis',
       cancellable: true
     })
   );
 
   ctx.subscriptions.push(
     commandRunner('codeQL.monitorRemoteQuery', async (
+      queryId: string,
       query: RemoteQuery,
       token: CancellationToken) => {
-      await rqm.monitorRemoteQuery(query, token);
+      await rqm.monitorRemoteQuery(queryId, query, token);
     }));
 
   ctx.subscriptions.push(
-    commandRunner('codeQL.showFakeRemoteQueryResults', async () => {
-      const analysisResultsManager = new AnalysesResultsManager(ctx, logger);
-      const rqim = new RemoteQueriesInterfaceManager(ctx, logger, analysisResultsManager);
-      await rqim.showResults(sampleRemoteQuery, sampleRemoteQueryResult);
+    commandRunner('codeQL.copyRepoList', async (queryId: string) => {
+      await rqm.copyRemoteQueryRepoListToClipboard(queryId);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.monitorVariantAnalysis', async (
+      variantAnalysis: VariantAnalysis,
+      token: CancellationToken
+    ) => {
+      await variantAnalysisManager.monitorVariantAnalysis(variantAnalysis, token);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.autoDownloadVariantAnalysisResult', async (
+      scannedRepo: VariantAnalysisScannedRepository,
+      variantAnalysisSummary: VariantAnalysis,
+      token: CancellationToken
+    ) => {
+      await variantAnalysisManager.enqueueDownload(scannedRepo, variantAnalysisSummary, token);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.cancelVariantAnalysis', async (
+      variantAnalysisId: number,
+    ) => {
+      await variantAnalysisManager.cancelVariantAnalysis(variantAnalysisId);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openVariantAnalysis', async () => {
+      await variantAnalysisManager.promptOpenVariantAnalysis();
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.autoDownloadRemoteQueryResults', async (
+      queryResult: RemoteQueryResult,
+      token: CancellationToken) => {
+      await rqm.autoDownloadRemoteQueryResults(queryResult, token);
     }));
 
+  ctx.subscriptions.push(
+    commandRunner('codeQL.exportVariantAnalysisResults', async (queryId?: string) => {
+      await exportRemoteQueryResults(qhm, rqm, ctx, queryId);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.loadVariantAnalysisRepoResults', async (variantAnalysisId: number, repositoryFullName: string) => {
+      await variantAnalysisManager.loadResults(variantAnalysisId, repositoryFullName);
+    })
+  );
+
+  // The "openVariantAnalysisView" command is internal-only.
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openVariantAnalysisView', async (variantAnalysisId: number) => {
+      await variantAnalysisManager.showView(variantAnalysisId);
+    })
+  );
+
   ctx.subscriptions.push(
     commandRunner(
       'codeQL.openReferencedFile',
@@ -842,8 +1012,10 @@ async function activateWithInstalledDistribution(
       progress: ProgressCallback,
       token: CancellationToken
     ) => {
+      // We restart the CLI server too, to ensure they are the same version
+      cliServer.restartCliServer();
       await qs.restartQueryServer(progress, token);
-      void helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', {
+      void showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
       });
     }, {
@@ -869,6 +1041,18 @@ async function activateWithInstalledDistribution(
       title: 'Choose a Database from an Archive'
     })
   );
+  ctx.subscriptions.push(
+    commandRunnerWithProgress('codeQL.chooseDatabaseGithub', async (
+      progress: ProgressCallback,
+      token: CancellationToken
+    ) => {
+      const credentials = isCanary() ? await Credentials.initialize(ctx) : undefined;
+      await databaseUI.handleChooseDatabaseGithub(credentials, progress, token);
+    },
+      {
+        title: 'Adding database from GitHub',
+      })
+  );
   ctx.subscriptions.push(
     commandRunnerWithProgress('codeQL.chooseDatabaseLgtm', (
       progress: ProgressCallback,
@@ -899,7 +1083,7 @@ async function activateWithInstalledDistribution(
     commandRunner('codeQL.copyVersion', async () => {
       const text = `CodeQL extension version: ${extension?.packageJSON.version} \nCodeQL CLI version: ${await getCliVersion()} \nPlatform: ${os.platform()} ${os.arch()}`;
       await env.clipboard.writeText(text);
-      void helpers.showAndLogInformationMessage(text);
+      void showAndLogInformationMessage(text);
     }));
 
   const getCliVersion = async () => {
@@ -910,19 +1094,16 @@ async function activateWithInstalledDistribution(
     }
   };
 
-  // The "authenticateToGitHub" command is internal-only.
   ctx.subscriptions.push(
     commandRunner('codeQL.authenticateToGitHub', async () => {
-      if (isCanary()) {
-        /**
-         * Credentials for authenticating to GitHub.
-         * These are used when making API calls.
-         */
-        const credentials = await Credentials.initialize(ctx);
-        const octokit = await credentials.getOctokit();
-        const userInfo = await octokit.users.getAuthenticated();
-        void helpers.showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
-      }
+      /**
+       * Credentials for authenticating to GitHub.
+       * These are used when making API calls.
+       */
+      const credentials = await Credentials.initialize(ctx);
+      const octokit = await credentials.getOctokit();
+      const userInfo = await octokit.users.getAuthenticated();
+      void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
     }));
 
   ctx.subscriptions.push(
@@ -945,37 +1126,48 @@ async function activateWithInstalledDistribution(
       }
     ));
 
-  commands.registerCommand('codeQL.showLogs', () => {
-    logger.show();
-  });
+  ctx.subscriptions.push(
+    commandRunner('codeQL.showLogs', async () => {
+      logger.show();
+    })
+  );
+
+  ctx.subscriptions.push(new SummaryLanguageSupport());
 
   void logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
   // Jump-to-definition and find-references
   void logger.log('Registering jump-to-definition handlers.');
+
+  // Store contextual queries in a temporary folder so that they are removed
+  // when the application closes. There is no need for the user to interact with them.
+  const contextualQueryStorageDir = path.join(tmpDir.name, 'contextual-query-storage');
+  await fs.ensureDir(contextualQueryStorageDir);
   languages.registerDefinitionProvider(
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
-    new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
+    new TemplateQueryDefinitionProvider(cliServer, qs, dbm, contextualQueryStorageDir)
   );
 
   languages.registerReferenceProvider(
     { scheme: archiveFilesystemProvider.zipArchiveScheme },
-    new TemplateQueryReferenceProvider(cliServer, qs, dbm)
+    new TemplateQueryReferenceProvider(cliServer, qs, dbm, contextualQueryStorageDir)
   );
 
   const astViewer = new AstViewer();
-  const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm);
+  const printAstTemplateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm, contextualQueryStorageDir);
+  const cfgTemplateProvider = new TemplatePrintCfgProvider(cliServer, dbm);
 
   ctx.subscriptions.push(astViewer);
   ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
     progress: ProgressCallback,
-    token: CancellationToken
+    token: CancellationToken,
+    selectedFile: Uri
   ) => {
-    const ast = await templateProvider.provideAst(
+    const ast = await printAstTemplateProvider.provideAst(
       progress,
       token,
-      window.activeTextEditor?.document,
+      selectedFile ?? window.activeTextEditor?.document.uri,
     );
     if (ast) {
       astViewer.updateRoots(await ast.getRoots(), ast.db, ast.fileName);
@@ -985,8 +1177,66 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
+  ctx.subscriptions.push(
+    commandRunnerWithProgress(
+      'codeQL.viewCfg',
+      async (
+        progress: ProgressCallback,
+        token: CancellationToken
+      ) => {
+        const res = await cfgTemplateProvider.provideCfgUri(window.activeTextEditor?.document);
+        if (res) {
+          await compileAndRunQuery(false, res[0], progress, token, undefined);
+        }
+      },
+      {
+        title: 'Calculating Control Flow Graph',
+        cancellable: true
+      }
+    )
+  );
+
+  const mockServer = new VSCodeMockGitHubApiServer(ctx);
+  ctx.subscriptions.push(mockServer);
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.mockGitHubApiServer.startRecording',
+      async () => await mockServer.startRecording(),
+    )
+  );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.mockGitHubApiServer.saveScenario',
+      async () => await mockServer.saveScenario(),
+    )
+  );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.mockGitHubApiServer.cancelRecording',
+      async () => await mockServer.cancelRecording(),
+    )
+  );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.mockGitHubApiServer.loadScenario',
+      async () => await mockServer.loadScenario(),
+    )
+  );
+  ctx.subscriptions.push(
+    commandRunner(
+      'codeQL.mockGitHubApiServer.unloadScenario',
+      async () => await mockServer.unloadScenario(),
+    )
+  );
+
   await commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
 
+  void logger.log('Reading query history');
+  await qhm.readQueryHistory();
+
+  const dbModule = await initializeDbModule(ctx);
+  ctx.subscriptions.push(dbModule);
+
   void logger.log('Successfully finished extension initialization.');
 
   return {
@@ -996,20 +1246,51 @@ async function activateWithInstalledDistribution(
     distributionManager,
     databaseManager: dbm,
     databaseUI,
+    variantAnalysisManager,
     dispose: () => {
       ctx.subscriptions.forEach(d => d.dispose());
     }
   };
 }
 
+async function createQueryServer(qlConfigurationListener: QueryServerConfigListener, cliServer: CodeQLCliServer, ctx: ExtensionContext): Promise<QueryRunner> {
+  const qsOpts = {
+    logger: queryServerLogger,
+    contextStoragePath: getContextStoragePath(ctx),
+  };
+  const progressCallback = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Window.withProgress(
+    { title: 'CodeQL query server', location: ProgressLocation.Window },
+    task
+  );
+  if (await cliServer.cliConstraints.supportsNewQueryServer()) {
+    const qs = new newQueryServer.QueryServerClient(
+      qlConfigurationListener,
+      cliServer,
+      qsOpts,
+      progressCallback
+    );
+    ctx.subscriptions.push(qs);
+    await qs.startQueryServer();
+    return new NewQueryRunner(qs);
+
+  } else {
+    const qs = new legacyQueryServer.QueryServerClient(
+      qlConfigurationListener,
+      cliServer,
+      qsOpts,
+      progressCallback
+    );
+    ctx.subscriptions.push(qs);
+    await qs.startQueryServer();
+    return new LegacyQueryRunner(qs);
+  }
+}
+
 function getContextStoragePath(ctx: ExtensionContext) {
-  return ctx.storagePath || ctx.globalStoragePath;
+  return ctx.storageUri?.fsPath || ctx.globalStorageUri.fsPath;
 }
 
 async function initializeLogging(ctx: ExtensionContext): Promise<void> {
-  const storagePath = getContextStoragePath(ctx);
-  await logger.setLogStoragePath(storagePath, false);
-  await ideServerLogger.setLogStoragePath(storagePath, false);
   ctx.subscriptions.push(logger);
   ctx.subscriptions.push(queryServerLogger);
   ctx.subscriptions.push(ideServerLogger);
@@ -1019,7 +1300,7 @@ const checkForUpdatesCommand = 'codeQL.checkForUpdatesToCLI';
 
 /**
  * This text provider lets us open readonly files in the editor.
- * 
+ *
  * TODO: Consolidate this with the 'codeql' text provider in query-history.ts.
  */
 function registerRemoteQueryTextProvider() {

extensions/ql-vscode/src/helpers.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs-extra';
 import * as glob from 'glob-promise';
 import * as yaml from 'js-yaml';
 import * as path from 'path';
+import * as tmp from 'tmp-promise';
 import {
   ExtensionContext,
   Uri,
@@ -14,6 +15,21 @@ import { UserCancellationException } from './commandRunner';
 import { logger } from './logging';
 import { QueryMetadata } from './pure/interface-types';
 
+// Shared temporary folder for the extension.
+export const tmpDir = tmp.dirSync({ prefix: 'queries_', keep: false, unsafeCleanup: true });
+export const upgradesTmpDir = path.join(tmpDir.name, 'upgrades');
+fs.ensureDirSync(upgradesTmpDir);
+
+export const tmpDirDisposal = {
+  dispose: () => {
+    try {
+      tmpDir.removeCallback();
+    } catch (e) {
+      void logger.log(`Failed to remove temporary directory ${tmpDir.name}: ${e}`);
+    }
+  }
+};
+
 /**
  * Show an error message and log it to the console
  *
@@ -64,9 +80,10 @@ export async function showAndLogWarningMessage(message: string, {
  */
 export async function showAndLogInformationMessage(message: string, {
   outputLogger = logger,
-  items = [] as string[]
+  items = [] as string[],
+  fullMessage = ''
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage);
+  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage, fullMessage);
 }
 
 type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
@@ -276,7 +293,7 @@ interface QlPackWithPath {
 async function findDbschemePack(packs: QlPackWithPath[], dbschemePath: string): Promise<{ name: string; isLibraryPack: boolean; }> {
   for (const { packDir, packName } of packs) {
     if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
+      const qlpack = yaml.load(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
       if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
         return {
           name: packName,
@@ -457,9 +474,9 @@ export function getInitialQueryContents(language: string, dbscheme: string) {
 
 /**
  * Heuristically determines if the directory passed in corresponds
- * to a database root.
- *
- * @param maybeRoot
+ * to a database root. A database root is a directory that contains
+ * a codeql-database.yml or (historically) a .dbinfo file. It also
+ * contains a folder starting with `db-`.
  */
 export async function isLikelyDatabaseRoot(maybeRoot: string) {
   const [a, b, c] = (await Promise.all([
@@ -471,11 +488,14 @@ export async function isLikelyDatabaseRoot(maybeRoot: string) {
     glob('db-*/', { cwd: maybeRoot })
   ]));
 
-  return !!((a || b) && c);
+  return ((a || b) && c.length > 0);
 }
 
-export function isLikelyDbLanguageFolder(dbPath: string) {
-  return !!path.basename(dbPath).startsWith('db-');
+/**
+ * A language folder is any folder starting with `db-` that is itself not a database root.
+ */
+export async function isLikelyDbLanguageFolder(dbPath: string) {
+  return path.basename(dbPath).startsWith('db-') && !(await isLikelyDatabaseRoot(dbPath));
 }
 
 /**
@@ -533,3 +553,38 @@ export async function tryGetQueryMetadata(cliServer: CodeQLCliServer, queryPath:
     return;
   }
 }
+
+/**
+ * Creates a file in the query directory that indicates when this query was created.
+ * This is important for keeping track of when queries should be removed.
+ *
+ * @param queryPath The directory that will containt all files relevant to a query result.
+ * It does not need to exist.
+ */
+export async function createTimestampFile(storagePath: string) {
+  const timestampPath = path.join(storagePath, 'timestamp');
+  await fs.ensureDir(storagePath);
+  await fs.writeFile(timestampPath, Date.now().toString(), 'utf8');
+}
+
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(dir: string): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await fs.opendir(dir)) {
+    const entry = path.join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}

extensions/ql-vscode/src/history-item-label-provider.ts

@@ -0,0 +1,108 @@
+import { env } from 'vscode';
+import * as path from 'path';
+import { QueryHistoryConfig } from './config';
+import { LocalQueryInfo } from './query-results';
+import { buildRepoLabel, getRawQueryName, QueryHistoryInfo } from './query-history-info';
+import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
+import { VariantAnalysisHistoryItem } from './remote-queries/variant-analysis-history-item';
+import { assertNever } from './pure/helpers-pure';
+import { pluralize } from './pure/word';
+import { humanizeQueryStatus } from './query-status';
+
+interface InterpolateReplacements {
+  t: string; // Start time
+  q: string; // Query name
+  d: string; // Database/Controller repo name
+  r: string; // Result count/Empty
+  s: string; // Status
+  f: string; // Query file name
+  '%': '%'; // Percent sign
+}
+
+export class HistoryItemLabelProvider {
+  constructor(private config: QueryHistoryConfig) {
+    /**/
+  }
+
+  getLabel(item: QueryHistoryInfo) {
+    let replacements: InterpolateReplacements;
+    switch (item.t) {
+      case 'local':
+        replacements = this.getLocalInterpolateReplacements(item);
+        break;
+      case 'remote':
+        replacements = this.getRemoteInterpolateReplacements(item);
+        break;
+      case 'variant-analysis':
+        replacements = this.getVariantAnalysisInterpolateReplacements(item);
+        break;
+      default:
+        assertNever(item);
+    }
+
+    const rawLabel = item.userSpecifiedLabel ?? (this.config.format || '%q');
+
+    return this.interpolate(rawLabel, replacements);
+  }
+
+  /**
+   * If there is a user-specified label for this query, interpolate and use that.
+   * Otherwise, use the raw name of this query.
+   *
+   * @returns the name of the query, unless there is a custom label for this query.
+   */
+  getShortLabel(item: QueryHistoryInfo): string {
+    return item.userSpecifiedLabel
+      ? this.getLabel(item)
+      : getRawQueryName(item);
+  }
+
+
+  private interpolate(rawLabel: string, replacements: InterpolateReplacements): string {
+    const label = rawLabel.replace(/%(.)/g, (match, key: keyof InterpolateReplacements) => {
+      const replacement = replacements[key];
+      return replacement !== undefined ? replacement : match;
+    });
+
+    return label.replace(/\s+/g, ' ');
+  }
+
+  private getLocalInterpolateReplacements(item: LocalQueryInfo): InterpolateReplacements {
+    const { resultCount = 0, statusString = 'in progress' } = item.completedQuery || {};
+    return {
+      t: item.startTime,
+      q: item.getQueryName(),
+      d: item.initialInfo.databaseInfo.name,
+      r: `(${resultCount} results)`,
+      s: statusString,
+      f: item.getQueryFileName(),
+      '%': '%',
+    };
+  }
+
+  private getRemoteInterpolateReplacements(item: RemoteQueryHistoryItem): InterpolateReplacements {
+    const resultCount = item.resultCount ? `(${pluralize(item.resultCount, 'result', 'results')})` : '';
+    return {
+      t: new Date(item.remoteQuery.executionStartTime).toLocaleString(env.language),
+      q: `${item.remoteQuery.queryName} (${item.remoteQuery.language})`,
+      d: buildRepoLabel(item),
+      r: resultCount,
+      s: humanizeQueryStatus(item.status),
+      f: path.basename(item.remoteQuery.queryFilePath),
+      '%': '%'
+    };
+  }
+
+  private getVariantAnalysisInterpolateReplacements(item: VariantAnalysisHistoryItem): InterpolateReplacements {
+    const resultCount = item.resultCount ? `(${pluralize(item.resultCount, 'result', 'results')})` : '';
+    return {
+      t: new Date(item.variantAnalysis.executionStartTime).toLocaleString(env.language),
+      q: `${item.variantAnalysis.query.name} (${item.variantAnalysis.query.language})`,
+      d: buildRepoLabel(item),
+      r: resultCount,
+      s: humanizeQueryStatus(item.status),
+      f: path.basename(item.variantAnalysis.query.filePath),
+      '%': '%',
+    };
+  }
+}

extensions/ql-vscode/src/interface-utils.ts

@@ -4,6 +4,7 @@ import {
   Uri,
   Location,
   Range,
+  ExtensionContext,
   WebviewPanel,
   Webview,
   workspace,
@@ -111,15 +112,36 @@ export function tryResolveLocation(
   }
 }
 
+export type WebviewView = 'results' | 'compare' | 'remote-queries' | 'variant-analysis';
+
+export interface WebviewMessage {
+  t: string;
+}
+
 /**
  * Returns HTML to populate the given webview.
  * Uses a content security policy that only loads the given script.
  */
 export function getHtmlForWebview(
+  ctx: ExtensionContext,
   webview: Webview,
-  scriptUriOnDisk: Uri,
-  stylesheetUrisOnDisk: Uri[],
+  view: WebviewView,
+  {
+    allowInlineStyles,
+  }: {
+    allowInlineStyles?: boolean;
+  } = {
+      allowInlineStyles: false,
+    }
 ): string {
+  const scriptUriOnDisk = Uri.file(
+    ctx.asAbsolutePath('out/webview.js')
+  );
+
+  const stylesheetUrisOnDisk = [
+    Uri.file(ctx.asAbsolutePath('out/webview.css'))
+  ];
+
   // Convert the on-disk URIs into webview URIs.
   const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
   const stylesheetWebviewUris = stylesheetUrisOnDisk.map(stylesheetUriOnDisk =>
@@ -128,8 +150,15 @@ export function getHtmlForWebview(
   // Use a nonce in the content security policy to uniquely identify the above resources.
   const nonce = getNonce();
 
-  const stylesheetsHtmlLines = stylesheetWebviewUris.map(stylesheetWebviewUri =>
-    `<link nonce="${nonce}" rel="stylesheet" href="${stylesheetWebviewUri}">`);
+  const stylesheetsHtmlLines = allowInlineStyles
+    ? stylesheetWebviewUris.map(uri => createStylesLinkWithoutNonce(uri))
+    : stylesheetWebviewUris.map(uri => createStylesLinkWithNonce(nonce, uri));
+
+  const styleSrc = allowInlineStyles
+    ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
+    : `'nonce-${nonce}'`;
+
+  const fontSrc = webview.cspSource;
 
   /*
    * Content security policy:
@@ -143,11 +172,11 @@ export function getHtmlForWebview(
 <html>
   <head>
     <meta http-equiv="Content-Security-Policy"
-          content="default-src 'none'; script-src 'nonce-${nonce}'; style-src 'nonce-${nonce}'; connect-src ${webview.cspSource};">
+          content="default-src 'none'; script-src 'nonce-${nonce}'; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${webview.cspSource};">
         ${stylesheetsHtmlLines.join(`    ${os.EOL}`)}
   </head>
   <body>
-    <div id=root>
+    <div id=root data-view="${view}">
     </div>
       <script nonce="${nonce}" src="${scriptWebviewUri}">
     </script>
@@ -243,3 +272,11 @@ export async function jumpToLocation(
     }
   }
 }
+
+function createStylesLinkWithNonce(nonce: string, uri: Uri): string {
+  return `<link nonce="${nonce}" rel="stylesheet" href="${uri}">`;
+}
+
+function createStylesLinkWithoutNonce(uri: Uri): string {
+  return `<link rel="stylesheet" href="${uri}">`;
+}

extensions/ql-vscode/src/interface.ts

@@ -1,6 +1,4 @@
-import * as path from 'path';
 import * as Sarif from 'sarif';
-import { DisposableObject } from './pure/disposable-object';
 import * as vscode from 'vscode';
 import {
   Diagnostic,
@@ -9,13 +7,13 @@ import {
   languages,
   Uri,
   window as Window,
-  env
+  env, WebviewPanel
 } from 'vscode';
 import * as cli from './cli';
 import { CodeQLCliServer } from './cli';
 import { DatabaseEventKind, DatabaseItem, DatabaseManager } from './databases';
 import { showAndLogErrorMessage } from './helpers';
-import { assertNever } from './pure/helpers-pure';
+import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import {
   FromResultsViewMsg,
   Interpretation,
@@ -27,26 +25,29 @@ import {
   InterpretedResultsSortState,
   SortDirection,
   ALERTS_TABLE_NAME,
+  GRAPH_TABLE_NAME,
   RawResultsSortState,
+  NavigationDirection,
 } from './pure/interface-types';
 import { Logger } from './logging';
-import * as messages from './pure/messages';
 import { commandRunner } from './commandRunner';
-import { CompletedQuery, interpretResults } from './query-results';
-import { QueryInfo, tmpDir } from './run-queries';
+import { CompletedQueryInfo, interpretResultsSarif, interpretGraphResults } from './query-results';
+import { QueryEvaluationInfo } from './run-queries-shared';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
 import {
   WebviewReveal,
   fileUriToWebviewUri,
   tryResolveLocation,
-  getHtmlForWebview,
   shownLocationDecoration,
   shownLocationLineDecoration,
   jumpToLocation,
 } from './interface-utils';
 import { getDefaultResultSetName, ParsedResultSets } from './pure/interface-types';
 import { RawResultSet, transformBqrsResultSet, ResultSetSchema } from './pure/bqrs-cli-types';
+import { AbstractWebview, WebviewPanelConfig } from './abstract-webview';
 import { PAGE_SIZE } from './config';
+import { CompletedLocalQueryInfo } from './query-results';
+import { HistoryItemLabelProvider } from './history-item-label-provider';
 
 /**
  * interface.ts
@@ -87,20 +88,41 @@ function sortInterpretedResults(
   }
 }
 
-function numPagesOfResultSet(resultSet: RawResultSet): number {
-  return Math.ceil(resultSet.schema.rows / PAGE_SIZE.getValue<number>());
+function interpretedPageSize(interpretation: Interpretation | undefined): number {
+  if (interpretation?.data.t == 'GraphInterpretationData') {
+    // Graph views always have one result per page.
+    return 1;
+  }
+  return PAGE_SIZE.getValue<number>();
+}
+
+function numPagesOfResultSet(resultSet: RawResultSet, interpretation?: Interpretation): number {
+  const pageSize = interpretedPageSize(interpretation);
+
+  const n = interpretation?.data.t == 'GraphInterpretationData'
+    ? interpretation.data.dot.length
+    : resultSet.schema.rows;
+
+  return Math.ceil(n / pageSize);
 }
 
 function numInterpretedPages(interpretation: Interpretation | undefined): number {
-  return Math.ceil((interpretation?.sarif.runs[0].results?.length || 0) / PAGE_SIZE.getValue<number>());
+  if (!interpretation) {
+    return 0;
+  }
+
+  const pageSize = interpretedPageSize(interpretation);
+
+  const n = interpretation.data.t == 'GraphInterpretationData'
+    ? interpretation.data.dot.length
+    : interpretation.data.runs[0].results?.length || 0;
+
+  return Math.ceil(n / pageSize);
 }
 
-export class InterfaceManager extends DisposableObject {
-  private _displayedQuery?: CompletedQuery;
+export class ResultsView extends AbstractWebview<IntoResultsViewMsg, FromResultsViewMsg> {
+  private _displayedQuery?: CompletedLocalQueryInfo;
   private _interpretation?: Interpretation;
-  private _panel: vscode.WebviewPanel | undefined;
-  private _panelLoaded = false;
-  private _panelLoadedCallBacks: (() => void)[] = [];
 
   private readonly _diagnosticCollection = languages.createDiagnosticCollection(
     'codeql-query-results'
@@ -110,34 +132,40 @@ export class InterfaceManager extends DisposableObject {
     public ctx: vscode.ExtensionContext,
     private databaseManager: DatabaseManager,
     public cliServer: CodeQLCliServer,
-    public logger: Logger
+    public logger: Logger,
+    private labelProvider: HistoryItemLabelProvider
   ) {
-    super();
+    super(ctx);
     this.push(this._diagnosticCollection);
     this.push(
       vscode.window.onDidChangeTextEditorSelection(
         this.handleSelectionChange.bind(this)
       )
     );
-    void logger.log('Registering path-step navigation commands.');
-    this.push(
-      commandRunner(
-        'codeQLQueryResults.nextPathStep',
-        this.navigatePathStep.bind(this, 1)
-      )
-    );
-    this.push(
-      commandRunner(
-        'codeQLQueryResults.previousPathStep',
-        this.navigatePathStep.bind(this, -1)
-      )
-    );
+    const navigationCommands = {
+      'codeQLQueryResults.up': NavigationDirection.up,
+      'codeQLQueryResults.down': NavigationDirection.down,
+      'codeQLQueryResults.left': NavigationDirection.left,
+      'codeQLQueryResults.right': NavigationDirection.right,
+      // For backwards compatibility with keybindings set using an earlier version of the extension.
+      'codeQLQueryResults.nextPathStep': NavigationDirection.down,
+      'codeQLQueryResults.previousPathStep': NavigationDirection.up,
+    };
+    void logger.log('Registering result view navigation commands.');
+    for (const [commandId, direction] of Object.entries(navigationCommands)) {
+      this.push(
+        commandRunner(
+          commandId,
+          this.navigateResultView.bind(this, direction)
+        )
+      );
+    }
 
     this.push(
       this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
         if (kind === DatabaseEventKind.Remove) {
           this._diagnosticCollection.clear();
-          if (this.isShowingPanel()) {
+          if (this.isShowingPanel) {
             void this.postMessage({
               t: 'untoggleShowProblems'
             });
@@ -147,61 +175,90 @@ export class InterfaceManager extends DisposableObject {
     );
   }
 
-  async navigatePathStep(direction: number): Promise<void> {
-    await this.postMessage({ t: 'navigatePath', direction });
-  }
-
-  private isShowingPanel() {
-    return !!this._panel;
-  }
-
-  // Returns the webview panel, creating it if it doesn't already
-  // exist.
-  getPanel(): vscode.WebviewPanel {
-    if (this._panel == undefined) {
-      const { ctx } = this;
-      const webViewColumn = this.chooseColumnForWebview();
-      const panel = (this._panel = Window.createWebviewPanel(
-        'resultsView', // internal name
-        'CodeQL Query Results', // user-visible name
-        { viewColumn: webViewColumn, preserveFocus: true },
-        {
-          enableScripts: true,
-          enableFindWidget: true,
-          retainContextWhenHidden: true,
-          localResourceRoots: [
-            vscode.Uri.file(tmpDir.name),
-            vscode.Uri.file(path.join(this.ctx.extensionPath, 'out'))
-          ]
-        }
-      ));
-
-      this._panel.onDidDispose(
-        () => {
-          this._panel = undefined;
-          this._displayedQuery = undefined;
-        },
-        null,
-        ctx.subscriptions
-      );
-      const scriptPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath('out/resultsView.js')
-      );
-      const stylesheetPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath('out/view/resultsView.css')
-      );
-      panel.webview.html = getHtmlForWebview(
-        panel.webview,
-        scriptPathOnDisk,
-        [stylesheetPathOnDisk]
-      );
-      panel.webview.onDidReceiveMessage(
-        async (e) => this.handleMsgFromView(e),
-        undefined,
-        ctx.subscriptions
-      );
+  async navigateResultView(direction: NavigationDirection): Promise<void> {
+    if (!this.panel?.visible) {
+      return;
+    }
+    // Reveal the panel now as the subsequent call to 'Window.showTextEditor' in 'showLocation' may destroy the webview otherwise.
+    this.panel.reveal();
+    await this.postMessage({ t: 'navigate', direction });
+  }
+
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: 'resultsView',
+      title: 'CodeQL Query Results',
+      viewColumn: this.chooseColumnForWebview(),
+      preserveFocus: true,
+      view: 'results',
+    };
+  }
+
+  protected onPanelDispose(): void {
+    this._displayedQuery = undefined;
+  }
+
+  protected async onMessage(msg: FromResultsViewMsg): Promise<void> {
+    try {
+      switch (msg.t) {
+        case 'viewLoaded':
+          this.onWebViewLoaded();
+          break;
+        case 'viewSourceFile': {
+          await jumpToLocation(msg, this.databaseManager, this.logger);
+          break;
+        }
+        case 'toggleDiagnostics': {
+          if (msg.visible) {
+            const databaseItem = this.databaseManager.findDatabaseItem(
+              Uri.parse(msg.databaseUri)
+            );
+            if (databaseItem !== undefined) {
+              await this.showResultsAsDiagnostics(
+                msg.origResultsPaths,
+                msg.metadata,
+                databaseItem
+              );
+            }
+          } else {
+            // TODO: Only clear diagnostics on the same database.
+            this._diagnosticCollection.clear();
+          }
+          break;
+        }
+        case 'changeSort':
+          await this.changeRawSortState(msg.resultSetName, msg.sortState);
+          break;
+        case 'changeInterpretedSort':
+          await this.changeInterpretedSortState(msg.sortState);
+          break;
+        case 'changePage':
+          if (msg.selectedTable === ALERTS_TABLE_NAME || msg.selectedTable === GRAPH_TABLE_NAME) {
+            await this.showPageOfInterpretedResults(msg.pageNumber);
+          }
+          else {
+            await this.showPageOfRawResults(
+              msg.selectedTable,
+              msg.pageNumber,
+              // When we are in an unsorted state, we guarantee that
+              // sortedResultsInfo doesn't have an entry for the current
+              // result set. Use this to determine whether or not we use
+              // the sorted bqrs file.
+              !!this._displayedQuery?.completedQuery.sortedResultsInfo[msg.selectedTable]
+            );
+          }
+          break;
+        case 'openFile':
+          await this.openFile(msg.filePath);
+          break;
+        default:
+          assertNever(msg);
+      }
+    } catch (e) {
+      void showAndLogErrorMessage(getErrorMessage(e), {
+        fullMessage: getErrorStack(e)
+      });
     }
-    return this._panel;
   }
 
   /**
@@ -238,7 +295,7 @@ export class InterfaceManager extends DisposableObject {
     }
     // Notify the webview that it should expect new results.
     await this.postMessage({ t: 'resultsUpdating' });
-    await this._displayedQuery.updateInterpretedSortState(sortState);
+    await this._displayedQuery.completedQuery.updateInterpretedSortState(sortState);
     await this.showResults(this._displayedQuery, WebviewReveal.NotForced, true);
   }
 
@@ -254,7 +311,7 @@ export class InterfaceManager extends DisposableObject {
     }
     // Notify the webview that it should expect new results.
     await this.postMessage({ t: 'resultsUpdating' });
-    await this._displayedQuery.updateSortState(
+    await this._displayedQuery.completedQuery.updateSortState(
       this.cliServer,
       resultSetName,
       sortState
@@ -266,88 +323,9 @@ export class InterfaceManager extends DisposableObject {
     await this.showPageOfRawResults(resultSetName, 0, true);
   }
 
-  private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
-    try {
-      switch (msg.t) {
-        case 'viewSourceFile': {
-          await jumpToLocation(msg, this.databaseManager, this.logger);
-          break;
-        }
-        case 'toggleDiagnostics': {
-          if (msg.visible) {
-            const databaseItem = this.databaseManager.findDatabaseItem(
-              Uri.parse(msg.databaseUri)
-            );
-            if (databaseItem !== undefined) {
-              await this.showResultsAsDiagnostics(
-                msg.origResultsPaths,
-                msg.metadata,
-                databaseItem
-              );
-            }
-          } else {
-            // TODO: Only clear diagnostics on the same database.
-            this._diagnosticCollection.clear();
-          }
-          break;
-        }
-        case 'resultViewLoaded':
-          this._panelLoaded = true;
-          this._panelLoadedCallBacks.forEach((cb) => cb());
-          this._panelLoadedCallBacks = [];
-          break;
-        case 'changeSort':
-          await this.changeRawSortState(msg.resultSetName, msg.sortState);
-          break;
-        case 'changeInterpretedSort':
-          await this.changeInterpretedSortState(msg.sortState);
-          break;
-        case 'changePage':
-          if (msg.selectedTable === ALERTS_TABLE_NAME) {
-            await this.showPageOfInterpretedResults(msg.pageNumber);
-          }
-          else {
-            await this.showPageOfRawResults(
-              msg.selectedTable,
-              msg.pageNumber,
-              // When we are in an unsorted state, we guarantee that
-              // sortedResultsInfo doesn't have an entry for the current
-              // result set. Use this to determine whether or not we use
-              // the sorted bqrs file.
-              this._displayedQuery?.sortedResultsInfo.has(msg.selectedTable) || false
-            );
-          }
-          break;
-        case 'openFile':
-          await this.openFile(msg.filePath);
-          break;
-        default:
-          assertNever(msg);
-      }
-    } catch (e) {
-      void showAndLogErrorMessage(e.message, {
-        fullMessage: e.stack
-      });
-    }
-  }
-
-  postMessage(msg: IntoResultsViewMsg): Thenable<boolean> {
-    return this.getPanel().webview.postMessage(msg);
-  }
-
-  private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve) => {
-      if (this._panelLoaded) {
-        resolve();
-      } else {
-        this._panelLoadedCallBacks.push(resolve);
-      }
-    });
-  }
-
   /**
    * Show query results in webview panel.
-   * @param results Evaluation info for the executed query.
+   * @param fullQuery Evaluation info for the executed query.
    * @param shouldKeepOldResultsWhileRendering Should keep old results while rendering.
    * @param forceReveal Force the webview panel to be visible and
    * Appropriate when the user has just performed an explicit
@@ -355,29 +333,30 @@ export class InterfaceManager extends DisposableObject {
    * history entry.
    */
   public async showResults(
-    results: CompletedQuery,
+    fullQuery: CompletedLocalQueryInfo,
     forceReveal: WebviewReveal,
     shouldKeepOldResultsWhileRendering = false
   ): Promise<void> {
-    if (results.result.resultType !== messages.QueryResultType.SUCCESS) {
+    if (!fullQuery.completedQuery.successful) {
       return;
     }
 
+    const panel = await this.getPanel();
+
     this._interpretation = undefined;
     const interpretationPage = await this.interpretResultsInfo(
-      results.query,
-      results.interpretedResultsSortState
+      fullQuery.completedQuery.query,
+      fullQuery.completedQuery.interpretedResultsSortState
     );
 
     const sortedResultsMap: SortedResultsMap = {};
-    results.sortedResultsInfo.forEach(
-      (v, k) =>
-        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v))
+    Object.entries(fullQuery.completedQuery.sortedResultsInfo).forEach(
+      ([k, v]) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(panel, v))
     );
 
-    this._displayedQuery = results;
+    this._displayedQuery = fullQuery;
 
-    const panel = this.getPanel();
     await this.waitForPanelLoaded();
     if (!panel.visible) {
       if (forceReveal === WebviewReveal.Forced) {
@@ -388,7 +367,7 @@ export class InterfaceManager extends DisposableObject {
         // more asynchronous message to not so abruptly interrupt
         // user's workflow by immediately revealing the panel.
         const showButton = 'View Results';
-        const queryName = results.queryName;
+        const queryName = this.labelProvider.getShortLabel(fullQuery);
         const resultPromise = vscode.window.showInformationMessage(
           `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ''
           }.`,
@@ -407,7 +386,7 @@ export class InterfaceManager extends DisposableObject {
     // Note that the resultSetSchemas will return offsets for the default (unsorted) page,
     // which may not be correct. However, in this case, it doesn't matter since we only
     // need the first offset, which will be the same no matter which sorting we use.
-    const resultSetSchemas = await this.getResultSetSchemas(results);
+    const resultSetSchemas = await this.getResultSetSchemas(fullQuery.completedQuery);
     const resultSetNames = resultSetSchemas.map(schema => schema.name);
 
     const selectedTable = getDefaultResultSetName(resultSetNames);
@@ -417,7 +396,7 @@ export class InterfaceManager extends DisposableObject {
 
     // Use sorted results path if it exists. This may happen if we are
     // reloading the results view after it has been sorted in the past.
-    const resultsPath = results.getResultsPath(selectedTable);
+    const resultsPath = fullQuery.completedQuery.getResultsPath(selectedTable);
     const pageSize = PAGE_SIZE.getValue<number>();
     const chunk = await this.cliServer.bqrsDecode(
       resultsPath,
@@ -432,11 +411,11 @@ export class InterfaceManager extends DisposableObject {
       }
     );
     const resultSet = transformBqrsResultSet(schema, chunk);
-    results.setResultCount(interpretationPage?.numTotalResults || resultSet.schema.rows);
+    fullQuery.completedQuery.setResultCount(interpretationPage?.numTotalResults || resultSet.schema.rows);
     const parsedResultSets: ParsedResultSets = {
       pageNumber: 0,
       pageSize,
-      numPages: numPagesOfResultSet(resultSet),
+      numPages: numPagesOfResultSet(resultSet, this._interpretation),
       numInterpretedPages: numInterpretedPages(this._interpretation),
       resultSet: { ...resultSet, t: 'RawResultSet' },
       selectedTable: undefined,
@@ -446,17 +425,18 @@ export class InterfaceManager extends DisposableObject {
     await this.postMessage({
       t: 'setState',
       interpretation: interpretationPage,
-      origResultsPaths: results.query.resultsPaths,
+      origResultsPaths: fullQuery.completedQuery.query.resultsPaths,
       resultsPath: this.convertPathToWebviewUri(
-        results.query.resultsPaths.resultsPath
+        panel,
+        fullQuery.completedQuery.query.resultsPaths.resultsPath
       ),
       parsedResultSets,
       sortedResultsMap,
-      database: results.database,
+      database: fullQuery.initialInfo.databaseInfo,
       shouldKeepOldResultsWhileRendering,
-      metadata: results.query.metadata,
-      queryName: results.toString(),
-      queryPath: results.query.program.queryPath
+      metadata: fullQuery.completedQuery.query.metadata,
+      queryName: this.labelProvider.getLabel(fullQuery),
+      queryPath: fullQuery.initialInfo.queryPath
     });
   }
 
@@ -472,29 +452,29 @@ export class InterfaceManager extends DisposableObject {
     if (this._interpretation === undefined) {
       throw new Error('Trying to show interpreted results but interpretation was undefined');
     }
-    if (this._interpretation.sarif.runs[0].results === undefined) {
+    if (this._interpretation.data.t === 'SarifInterpretationData' && this._interpretation.data.runs[0].results === undefined) {
       throw new Error('Trying to show interpreted results but results were undefined');
     }
 
-    const resultSetSchemas = await this.getResultSetSchemas(this._displayedQuery);
+    const resultSetSchemas = await this.getResultSetSchemas(this._displayedQuery.completedQuery);
     const resultSetNames = resultSetSchemas.map(schema => schema.name);
 
     await this.postMessage({
       t: 'showInterpretedPage',
       interpretation: this.getPageOfInterpretedResults(pageNumber),
-      database: this._displayedQuery.database,
-      metadata: this._displayedQuery.query.metadata,
+      database: this._displayedQuery.initialInfo.databaseInfo,
+      metadata: this._displayedQuery.completedQuery.query.metadata,
       pageNumber,
       resultSetNames,
-      pageSize: PAGE_SIZE.getValue(),
+      pageSize: interpretedPageSize(this._interpretation),
       numPages: numInterpretedPages(this._interpretation),
-      queryName: this._displayedQuery.toString(),
-      queryPath: this._displayedQuery.query.program.queryPath
+      queryName: this.labelProvider.getLabel(this._displayedQuery),
+      queryPath: this._displayedQuery.initialInfo.queryPath
     });
   }
 
-  private async getResultSetSchemas(results: CompletedQuery, selectedTable = ''): Promise<ResultSetSchema[]> {
-    const resultsPath = results.getResultsPath(selectedTable);
+  private async getResultSetSchemas(completedQuery: CompletedQueryInfo, selectedTable = ''): Promise<ResultSetSchema[]> {
+    const resultsPath = completedQuery.getResultsPath(selectedTable);
     const schemas = await this.cliServer.bqrsInfo(
       resultsPath,
       PAGE_SIZE.getValue()
@@ -520,18 +500,20 @@ export class InterfaceManager extends DisposableObject {
       throw new Error('trying to view a page of a query that is not loaded');
     }
 
+    const panel = await this.getPanel();
+
     const sortedResultsMap: SortedResultsMap = {};
-    results.sortedResultsInfo.forEach(
-      (v, k) =>
-        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(v))
+    Object.entries(results.completedQuery.sortedResultsInfo).forEach(
+      ([k, v]) =>
+        (sortedResultsMap[k] = this.convertPathPropertiesToWebviewUris(panel, v))
     );
 
-    const resultSetSchemas = await this.getResultSetSchemas(results, sorted ? selectedTable : '');
+    const resultSetSchemas = await this.getResultSetSchemas(results.completedQuery, sorted ? selectedTable : '');
 
     // If there is a specific sorted table selected, a different bqrs file is loaded that doesn't have all the result set names.
     // Make sure that we load all result set names here.
     // See https://github.com/github/vscode-codeql/issues/1005
-    const allResultSetSchemas = sorted ? await this.getResultSetSchemas(results, '') : resultSetSchemas;
+    const allResultSetSchemas = sorted ? await this.getResultSetSchemas(results.completedQuery, '') : resultSetSchemas;
     const resultSetNames = allResultSetSchemas.map(schema => schema.name);
 
     const schema = resultSetSchemas.find(
@@ -542,7 +524,7 @@ export class InterfaceManager extends DisposableObject {
 
     const pageSize = PAGE_SIZE.getValue<number>();
     const chunk = await this.cliServer.bqrsDecode(
-      results.getResultsPath(selectedTable, sorted),
+      results.completedQuery.getResultsPath(selectedTable, sorted),
       schema.name,
       {
         offset: schema.pagination?.offsets[pageNumber],
@@ -564,17 +546,18 @@ export class InterfaceManager extends DisposableObject {
     await this.postMessage({
       t: 'setState',
       interpretation: this._interpretation,
-      origResultsPaths: results.query.resultsPaths,
+      origResultsPaths: results.completedQuery.query.resultsPaths,
       resultsPath: this.convertPathToWebviewUri(
-        results.query.resultsPaths.resultsPath
+        panel,
+        results.completedQuery.query.resultsPaths.resultsPath
       ),
       parsedResultSets,
       sortedResultsMap,
-      database: results.database,
+      database: results.initialInfo.databaseInfo,
       shouldKeepOldResultsWhileRendering: false,
-      metadata: results.query.metadata,
-      queryName: results.toString(),
-      queryPath: results.query.program.queryPath
+      metadata: results.completedQuery.query.metadata,
+      queryName: this.labelProvider.getLabel(results),
+      queryPath: results.initialInfo.queryPath
     });
   }
 
@@ -589,28 +572,45 @@ export class InterfaceManager extends DisposableObject {
       void this.logger.log('No results path. Cannot display interpreted results.');
       return undefined;
     }
+    let data;
+    let numTotalResults;
+    if (metadata?.kind === GRAPH_TABLE_NAME) {
+      data = await interpretGraphResults(
+        this.cliServer,
+        metadata,
+        resultsPaths,
+        sourceInfo
+      );
+      numTotalResults = data.dot.length;
+    } else {
+      const sarif = await interpretResultsSarif(
+        this.cliServer,
+        metadata,
+        resultsPaths,
+        sourceInfo
+      );
 
-    const sarif = await interpretResults(
-      this.cliServer,
-      metadata,
-      resultsPaths,
-      sourceInfo
-    );
+      sarif.runs.forEach(run => {
+        if (run.results) {
+          sortInterpretedResults(run.results, sortState);
+        }
+      });
 
-    sarif.runs.forEach(run => {
-      if (run.results !== undefined) {
-        sortInterpretedResults(run.results, sortState);
-      }
-    });
+      sarif.sortState = sortState;
+      data = sarif;
 
-    const numTotalResults = sarif.runs[0]?.results?.length || 0;
+      numTotalResults = (() => {
+        return sarif.runs?.[0]?.results
+          ? sarif.runs[0].results.length
+          : 0;
+      })();
+    }
 
     const interpretation: Interpretation = {
-      sarif,
+      data,
       sourceLocationPrefix,
       numTruncatedResults: 0,
-      numTotalResults,
-      sortState,
+      numTotalResults
     };
     this._interpretation = interpretation;
     return interpretation;
@@ -619,7 +619,6 @@ export class InterfaceManager extends DisposableObject {
   private getPageOfInterpretedResults(
     pageNumber: number
   ): Interpretation {
-
     function getPageOfRun(run: Sarif.Run): Sarif.Run {
       return {
         ...run, results: run.results?.slice(
@@ -629,32 +628,44 @@ export class InterfaceManager extends DisposableObject {
       };
     }
 
-    if (this._interpretation === undefined) {
+    const interp = this._interpretation;
+    if (interp === undefined) {
       throw new Error('Tried to get interpreted results before interpretation finished');
     }
-    if (this._interpretation.sarif.runs.length !== 1) {
-      void this.logger.log(`Warning: SARIF file had ${this._interpretation.sarif.runs.length} runs, expected 1`);
+
+    if (interp.data.t !== 'SarifInterpretationData')
+      return interp;
+
+    if (interp.data.runs.length !== 1) {
+      void this.logger.log(`Warning: SARIF file had ${interp.data.runs.length} runs, expected 1`);
     }
-    const interp = this._interpretation;
+
     return {
       ...interp,
-      sarif: { ...interp.sarif, runs: [getPageOfRun(interp.sarif.runs[0])] },
+      data: {
+        ...interp.data,
+        runs: [getPageOfRun(interp.data.runs[0])]
+      }
     };
   }
 
   private async interpretResultsInfo(
-    query: QueryInfo,
+    query: QueryEvaluationInfo,
     sortState: InterpretedResultsSortState | undefined
   ): Promise<Interpretation | undefined> {
     if (
-      (await query.canHaveInterpretedResults()) &&
+      query.canHaveInterpretedResults() &&
       query.quickEvalPosition === undefined // never do results interpretation if quickEval
     ) {
       try {
-        const sourceLocationPrefix = await query.dbItem.getSourceLocationPrefix(
+        const dbItem = this.databaseManager.findDatabaseItem(Uri.file(query.dbItemPath));
+        if (!dbItem) {
+          throw new Error(`Could not find database item for ${query.dbItemPath}`);
+        }
+        const sourceLocationPrefix = await dbItem.getSourceLocationPrefix(
           this.cliServer
         );
-        const sourceArchiveUri = query.dbItem.sourceArchive;
+        const sourceArchiveUri = dbItem.sourceArchive;
         const sourceInfo =
           sourceArchiveUri === undefined
             ? undefined
@@ -673,7 +684,7 @@ export class InterfaceManager extends DisposableObject {
         // If interpretation fails, accept the error and continue
         // trying to render uninterpreted results anyway.
         void showAndLogErrorMessage(
-          `Showing raw results instead of interpreted ones due to an error. ${e.message}`
+          `Showing raw results instead of interpreted ones due to an error. ${getErrorMessage(e)}`
         );
       }
     }
@@ -712,9 +723,8 @@ export class InterfaceManager extends DisposableObject {
     try {
       await this.showProblemResultsAsDiagnostics(interpretation, database);
     } catch (e) {
-      const msg = e instanceof Error ? e.message : e.toString();
       void this.logger.log(
-        `Exception while computing problem results as diagnostics: ${msg}`
+        `Exception while computing problem results as diagnostics: ${getErrorMessage(e)}`
       );
       this._diagnosticCollection.clear();
     }
@@ -724,9 +734,12 @@ export class InterfaceManager extends DisposableObject {
     interpretation: Interpretation,
     databaseItem: DatabaseItem
   ): Promise<void> {
-    const { sarif, sourceLocationPrefix } = interpretation;
+    const { data, sourceLocationPrefix } = interpretation;
 
-    if (!sarif.runs || !sarif.runs[0].results) {
+    if (data.t !== 'SarifInterpretationData')
+      return;
+
+    if (!data.runs || !data.runs[0].results) {
       void this.logger.log(
         'Didn\'t find a run in the sarif results. Error processing sarif?'
       );
@@ -735,7 +748,7 @@ export class InterfaceManager extends DisposableObject {
 
     const diagnostics: [Uri, ReadonlyArray<Diagnostic>][] = [];
 
-    for (const result of sarif.runs[0].results) {
+    for (const result of data.runs[0].results) {
       const message = result.message.text;
       if (message === undefined) {
         void this.logger.log('Sarif had result without plaintext message');
@@ -804,15 +817,16 @@ export class InterfaceManager extends DisposableObject {
     this._diagnosticCollection.set(diagnostics);
   }
 
-  private convertPathToWebviewUri(path: string): string {
-    return fileUriToWebviewUri(this.getPanel(), Uri.file(path));
+  private convertPathToWebviewUri(panel: WebviewPanel, path: string): string {
+    return fileUriToWebviewUri(panel, Uri.file(path));
   }
 
   private convertPathPropertiesToWebviewUris(
+    panel: WebviewPanel,
     info: SortedResultSetInfo
   ): SortedResultSetInfo {
     return {
-      resultsPath: this.convertPathToWebviewUri(info.resultsPath),
+      resultsPath: this.convertPathToWebviewUri(panel, info.resultsPath),
       sortState: info.sortState,
     };
   }

extensions/ql-vscode/src/json-rpc-server.ts

@@ -0,0 +1,30 @@
+import { Logger } from './logging';
+import * as cp from 'child_process';
+import { Disposable } from 'vscode';
+import { MessageConnection } from 'vscode-jsonrpc';
+
+
+/** A running query server process and its associated message connection. */
+export class ServerProcess implements Disposable {
+  child: cp.ChildProcess;
+  connection: MessageConnection;
+  logger: Logger;
+
+  constructor(child: cp.ChildProcess, connection: MessageConnection, private name: string, logger: Logger) {
+    this.child = child;
+    this.connection = connection;
+    this.logger = logger;
+  }
+
+  dispose(): void {
+    void this.logger.log(`Stopping ${this.name}...`);
+    this.connection.dispose();
+    this.child.stdin!.end();
+    this.child.stderr!.destroy();
+    // TODO kill the process if it doesn't terminate after a certain time limit.
+
+    // On Windows, we usually have to terminate the process before closing its stdout.
+    this.child.stdout!.destroy();
+    void this.logger.log(`Stopped ${this.name}.`);
+  }
+}

extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts

@@ -0,0 +1,65 @@
+import { CancellationToken } from 'vscode';
+import { ProgressCallback } from '../commandRunner';
+import { DatabaseItem } from '../databases';
+import { Dataset, deregisterDatabases, registerDatabases } from '../pure/legacy-messages';
+import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
+import { QueryRunner } from '../queryRunner';
+import { QueryWithResults } from '../run-queries-shared';
+import { QueryServerClient } from './queryserver-client';
+import { clearCacheInDatabase, compileAndRunQueryAgainstDatabase } from './run-queries';
+import { upgradeDatabaseExplicit } from './upgrades';
+
+export class LegacyQueryRunner extends QueryRunner {
+
+
+  constructor(public readonly qs: QueryServerClient) {
+    super();
+  }
+
+  get cliServer() {
+    return this.qs.cliServer;
+  }
+
+  async restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await this.qs.restartQueryServer(progress, token);
+  }
+
+  onStart(callBack: (progress: ProgressCallback, token: CancellationToken) => Promise<void>) {
+    this.qs.onDidStartQueryServer(callBack);
+  }
+  async clearCacheInDatabase(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await clearCacheInDatabase(this.qs, dbItem, progress, token);
+  }
+  async compileAndRunQueryAgainstDatabase(dbItem: DatabaseItem, initialInfo: InitialQueryInfo, queryStorageDir: string, progress: ProgressCallback, token: CancellationToken, templates?: Record<string, string>, queryInfo?: LocalQueryInfo): Promise<QueryWithResults> {
+    return await compileAndRunQueryAgainstDatabase(this.qs.cliServer, this.qs, dbItem, initialInfo, queryStorageDir, progress, token, templates, queryInfo);
+  }
+
+  async deregisterDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
+    if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
+    }
+  }
+  async registerDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
+    if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
+    }
+  }
+
+  async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await upgradeDatabaseExplicit(this.qs, dbItem, progress, token);
+  }
+
+  async clearPackCache(): Promise<void> {
+    /**
+     * Nothing needs to be done
+     */
+  }
+}

extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts

@@ -1,49 +1,25 @@
-import * as cp from 'child_process';
 import * as path from 'path';
-import { DisposableObject } from './pure/disposable-object';
-import { Disposable, CancellationToken, commands } from 'vscode';
-import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
-import * as cli from './cli';
-import { QueryServerConfig } from './config';
-import { Logger, ProgressReporter } from './logging';
-import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
-import * as messages from './pure/messages';
-import { ProgressCallback, ProgressTask } from './commandRunner';
 import * as fs from 'fs-extra';
-import * as helpers from './helpers';
+
+import { DisposableObject } from '../pure/disposable-object';
+import { CancellationToken, commands } from 'vscode';
+import { createMessageConnection, RequestType } from 'vscode-jsonrpc';
+import * as cli from '../cli';
+import { QueryServerConfig } from '../config';
+import { Logger, ProgressReporter } from '../logging';
+import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from '../pure/legacy-messages';
+import * as messages from '../pure/legacy-messages';
+import { ProgressCallback, ProgressTask } from '../commandRunner';
+import { findQueryLogFile } from '../run-queries-shared';
+import { ServerProcess } from '../json-rpc-server';
+
+type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
 
 type ServerOpts = {
   logger: Logger;
   contextStoragePath: string;
 }
 
-/** A running query server process and its associated message connection. */
-class ServerProcess implements Disposable {
-  child: cp.ChildProcess;
-  connection: MessageConnection;
-  logger: Logger;
-
-  constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) {
-    this.child = child;
-    this.connection = connection;
-    this.logger = logger;
-  }
-
-  dispose(): void {
-    void this.logger.log('Stopping query server...');
-    this.connection.dispose();
-    this.child.stdin!.end();
-    this.child.stderr!.destroy();
-    // TODO kill the process if it doesn't terminate after a certain time limit.
-
-    // On Windows, we usually have to terminate the process before closing its stdout.
-    this.child.stdout!.destroy();
-    void this.logger.log('Stopped query server.');
-  }
-}
-
-type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
-
 /**
  * Client that manages a query server process.
  * The server process is started upon initialization and tracked during its lifetime.
@@ -68,7 +44,7 @@ export class QueryServerClient extends DisposableObject {
     this.queryServerStartListeners.push(e);
   }
 
-  public activeQueryName: string | undefined;
+  public activeQueryLogFile: string | undefined;
 
   constructor(
     readonly config: QueryServerConfig,
@@ -89,26 +65,6 @@ export class QueryServerClient extends DisposableObject {
     this.evaluationResultCallbacks = {};
   }
 
-  async initLogger() {
-    let storagePath = this.opts.contextStoragePath;
-    let isCustomLogDirectory = false;
-    if (this.config.customLogDirectory) {
-      try {
-        if (!(await fs.pathExists(this.config.customLogDirectory))) {
-          await fs.mkdir(this.config.customLogDirectory);
-        }
-        void this.logger.log(`Saving query server logs to user-specified directory: ${this.config.customLogDirectory}.`);
-        storagePath = this.config.customLogDirectory;
-        isCustomLogDirectory = true;
-      } catch (e) {
-        void helpers.showAndLogErrorMessage(`${this.config.customLogDirectory} is not a valid directory. Logs will be stored in a temporary workspace directory instead.`);
-      }
-    }
-
-    await this.logger.setLogStoragePath(storagePath, isCustomLogDirectory);
-
-  }
-
   get logger(): Logger {
     return this.opts.logger;
   }
@@ -150,7 +106,6 @@ export class QueryServerClient extends DisposableObject {
 
   /** Starts a new query server process, sending progress messages to the given reporter. */
   private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
-    await this.initLogger();
     const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
     const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
 
@@ -167,16 +122,29 @@ export class QueryServerClient extends DisposableObject {
       args.push('--require-db-registration');
     }
 
-    if (await this.cliServer.cliConstraints.supportsOldEvalStats()) {
+    if (await this.cliServer.cliConstraints.supportsOldEvalStats() && !(await this.cliServer.cliConstraints.supportsPerQueryEvalLog())) {
       args.push('--old-eval-stats');
     }
 
+    if (await this.cliServer.cliConstraints.supportsStructuredEvalLog()) {
+      const structuredLogFile = `${this.opts.contextStoragePath}/structured-evaluator-log.json`;
+      await fs.ensureFile(structuredLogFile);
+
+      args.push('--evaluator-log');
+      args.push(structuredLogFile);
+
+      // We hard-code the verbosity level to 5 and minify to false.
+      // This will be the behavior of the per-query structured logging in the CLI after 2.8.3.
+      args.push('--evaluator-log-level');
+      args.push('5');
+    }
+
     if (this.config.debug) {
       args.push('--debug', '--tuple-counting');
     }
 
     if (cli.shouldDebugQueryServer()) {
-      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9010,server=y,suspend=n,quiet=y');
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9010,server=n,suspend=y,quiet=y');
     }
 
     const child = cli.spawnServer(
@@ -187,7 +155,7 @@ export class QueryServerClient extends DisposableObject {
       this.logger,
       data => this.logger.log(data.toString(), {
         trailingNewline: false,
-        additionalLogLocation: this.activeQueryName
+        additionalLogLocation: this.activeQueryLogFile
       }),
       undefined, // no listener for stdout
       progressReporter
@@ -198,10 +166,6 @@ export class QueryServerClient extends DisposableObject {
       if (!(res.runId in this.evaluationResultCallbacks)) {
         void this.logger.log(`No callback associated with run id ${res.runId}, continuing without executing any callback`);
       } else {
-        const baseLocation = this.logger.getBaseLocation();
-        if (baseLocation && this.activeQueryName) {
-          res.logFileLocation = path.join(baseLocation, this.activeQueryName);
-        }
         this.evaluationResultCallbacks[res.runId](res);
       }
       return {};
@@ -212,7 +176,7 @@ export class QueryServerClient extends DisposableObject {
         callback(res);
       }
     });
-    this.serverProcess = new ServerProcess(child, connection, this.logger);
+    this.serverProcess = new ServerProcess(child, connection, 'Query server', this.logger);
     // Ensure the server process is disposed together with this client.
     this.track(this.serverProcess);
     connection.listen();
@@ -234,7 +198,7 @@ export class QueryServerClient extends DisposableObject {
   }
 
   get serverProcessPid(): number {
-    return this.serverProcess!.child.pid;
+    return this.serverProcess!.child.pid || 0;
   }
 
   async sendRequest<P, R, E, RO>(type: RequestType<WithProgressId<P>, R, E, RO>, parameter: P, token?: CancellationToken, progress?: (res: ProgressMessage) => void): Promise<R> {
@@ -262,8 +226,7 @@ export class QueryServerClient extends DisposableObject {
    */
   private updateActiveQuery(method: string, parameter: any): void {
     if (method === messages.compileQuery.method) {
-      const queryPath = parameter?.queryToCheck?.queryPath || 'unknown';
-      this.activeQueryName = `query-${path.basename(queryPath)}-${this.nextProgress}.log`;
+      this.activeQueryLogFile = findQueryLogFile(path.dirname(parameter.resultPath));
     }
   }
 }

extensions/ql-vscode/src/legacy-query-server/run-queries.ts

@@ -0,0 +1,526 @@
+import * as crypto from 'crypto';
+import * as fs from 'fs-extra';
+import * as tmp from 'tmp-promise';
+import * as path from 'path';
+import {
+  CancellationToken,
+  Uri,
+} from 'vscode';
+import { ErrorCodes, ResponseError } from 'vscode-languageclient';
+
+import * as cli from '../cli';
+import { DatabaseItem, } from '../databases';
+import {
+  getOnDiskWorkspaceFolders,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  tryGetQueryMetadata,
+  upgradesTmpDir
+} from '../helpers';
+import { ProgressCallback } from '../commandRunner';
+import { QueryMetadata } from '../pure/interface-types';
+import { logger } from '../logging';
+import * as messages from '../pure/legacy-messages';
+import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
+import * as qsClient from './queryserver-client';
+import { getErrorMessage } from '../pure/helpers-pure';
+import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades';
+import { QueryEvaluationInfo, QueryWithResults } from '../run-queries-shared';
+
+/**
+ * A collection of evaluation-time information about a query,
+ * including the query itself, and where we have decided to put
+ * temporary files associated with it, such as the compiled query
+ * output and results.
+ */
+export class QueryInProgress {
+
+  public queryEvalInfo: QueryEvaluationInfo;
+  /**
+   * Note that in the {@link slurpQueryHistory} method, we create a QueryEvaluationInfo instance
+   * by explicitly setting the prototype in order to avoid calling this constructor.
+   */
+  constructor(
+    readonly querySaveDir: string,
+    readonly dbItemPath: string,
+    databaseHasMetadataFile: boolean,
+    readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
+    readonly quickEvalPosition?: messages.Position,
+    readonly metadata?: QueryMetadata,
+    readonly templates?: Record<string, string>,
+  ) {
+    this.queryEvalInfo = new QueryEvaluationInfo(querySaveDir, dbItemPath, databaseHasMetadataFile, quickEvalPosition, metadata);
+    /**/
+  }
+
+  get compiledQueryPath() {
+    return this.queryEvalInfo.compileQueryPath;
+  }
+
+
+  async run(
+    qs: qsClient.QueryServerClient,
+    upgradeQlo: string | undefined,
+    availableMlModels: cli.MlModelInfo[],
+    dbItem: DatabaseItem,
+    progress: ProgressCallback,
+    token: CancellationToken,
+    queryInfo?: LocalQueryInfo,
+  ): Promise<messages.EvaluationResult> {
+    if (!dbItem.contents || dbItem.error) {
+      throw new Error('Can\'t run query on invalid database.');
+    }
+
+    let result: messages.EvaluationResult | null = null;
+
+    const callbackId = qs.registerCallback(res => {
+      result = {
+        ...res,
+        logFileLocation: this.queryEvalInfo.logPath
+      };
+    });
+
+    const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
+
+    const queryToRun: messages.QueryToRun = {
+      resultsPath: this.queryEvalInfo.resultsPaths.resultsPath,
+      qlo: Uri.file(this.compiledQueryPath).toString(),
+      compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
+      allowUnknownTemplates: true,
+      templateValues: createSimpleTemplates(this.templates),
+      availableMlModels: availableMlModelUris,
+      id: callbackId,
+      timeoutSecs: qs.config.timeoutSecs,
+    };
+
+    const dataset: messages.Dataset = {
+      dbDir: dbItem.contents.datasetUri.fsPath,
+      workingSet: 'default'
+    };
+    if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
+      await qs.sendRequest(messages.startLog, {
+        db: dataset,
+        logPath: this.queryEvalInfo.evalLogPath,
+      });
+
+    }
+    const params: messages.EvaluateQueriesParams = {
+      db: dataset,
+      evaluateId: callbackId,
+      queries: [queryToRun],
+      stopOnError: false,
+      useSequenceHint: false
+    };
+    try {
+      await qs.sendRequest(messages.runQueries, params, token, progress);
+      if (qs.config.customLogDirectory) {
+        void showAndLogWarningMessage(
+          `Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.queryEvalInfo.logPath}.`
+        );
+      }
+    } finally {
+      qs.unRegisterCallback(callbackId);
+      if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
+        await qs.sendRequest(messages.endLog, {
+          db: dataset,
+          logPath: this.queryEvalInfo.evalLogPath,
+        });
+        if (await this.queryEvalInfo.hasEvalLog()) {
+          await this.queryEvalInfo.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
+        } else {
+          void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.queryEvalInfo.evalLogPath}.`);
+        }
+      }
+    }
+    return result || {
+      evaluationTime: 0,
+      message: 'No result from server',
+      queryId: -1,
+      runId: callbackId,
+      resultType: messages.QueryResultType.OTHER_ERROR
+    };
+  }
+
+  async compile(
+    qs: qsClient.QueryServerClient,
+    program: messages.QlProgram,
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<messages.CompilationMessage[]> {
+    let compiled: messages.CheckQueryResult | undefined;
+    try {
+      const target = this.quickEvalPosition ? {
+        quickEval: { quickEvalPos: this.quickEvalPosition }
+      } : { query: {} };
+      const params: messages.CompileQueryParams = {
+        compilationOptions: {
+          computeNoLocationUrls: true,
+          failOnWarnings: false,
+          fastCompilation: false,
+          includeDilInQlo: true,
+          localChecking: false,
+          noComputeGetUrl: false,
+          noComputeToString: false,
+          computeDefaultStrings: true,
+          emitDebugInfo: true
+        },
+        extraOptions: {
+          timeoutSecs: qs.config.timeoutSecs
+        },
+        queryToCheck: program,
+        resultPath: this.compiledQueryPath,
+        target,
+      };
+
+      compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
+    } finally {
+      void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.queryEvalInfo.logPath });
+    }
+    return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
+  }
+}
+
+export async function clearCacheInDatabase(
+  qs: qsClient.QueryServerClient,
+  dbItem: DatabaseItem,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<messages.ClearCacheResult> {
+  if (dbItem.contents === undefined) {
+    throw new Error('Can\'t clear the cache in an invalid database.');
+  }
+
+  const db: messages.Dataset = {
+    dbDir: dbItem.contents.datasetUri.fsPath,
+    workingSet: 'default',
+  };
+
+  const params: messages.ClearCacheParams = {
+    dryRun: false,
+    db,
+  };
+
+  return qs.sendRequest(messages.clearCache, params, token, progress);
+}
+
+
+/**
+ * Compare the dbscheme implied by the query `query` and that of the current database.
+ * - If they are compatible, do nothing.
+ * - If they are incompatible but the database can be upgraded, suggest that upgrade.
+ * - If they are incompatible and the database cannot be upgraded, throw an error.
+ */
+async function checkDbschemeCompatibility(
+  cliServer: cli.CodeQLCliServer,
+  qs: qsClient.QueryServerClient,
+  query: QueryInProgress,
+  qlProgram: messages.QlProgram,
+  dbItem: DatabaseItem,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<void> {
+  const searchPath = getOnDiskWorkspaceFolders();
+
+  if (dbItem.contents?.dbSchemeUri !== undefined) {
+    const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
+    const hash = async function(filename: string): Promise<string> {
+      return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
+    };
+
+    // At this point, we have learned about three dbschemes:
+
+    // the dbscheme of the actual database we're querying.
+    const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath);
+
+    // the dbscheme of the query we're running, including the library we've resolved it to use.
+    const dbschemeOfLib = await hash(query.queryDbscheme);
+
+    // the database we're able to upgrade to
+    const upgradableTo = await hash(finalDbscheme);
+
+    if (upgradableTo != dbschemeOfLib) {
+      reportNoUpgradePath(qlProgram, query);
+    }
+
+    if (upgradableTo == dbschemeOfLib &&
+      dbschemeOfDb != dbschemeOfLib) {
+      // Try to upgrade the database
+      await upgradeDatabaseExplicit(
+        qs,
+        dbItem,
+        progress,
+        token
+      );
+    }
+  }
+}
+
+function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryInProgress): void {
+  throw new Error(
+    `Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`
+  );
+}
+
+/**
+ * Compile a non-destructive upgrade.
+ */
+async function compileNonDestructiveUpgrade(
+  qs: qsClient.QueryServerClient,
+  upgradeTemp: tmp.DirectoryResult,
+  query: QueryInProgress,
+  qlProgram: messages.QlProgram,
+  dbItem: DatabaseItem,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<string> {
+
+  if (!dbItem?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
+  }
+
+  // When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
+  // When packaging is not used, all dependencies are in the workspace.
+  const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
+    ? qlProgram.libraryPath
+    : getOnDiskWorkspaceFolders();
+
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
+    dbItem.contents.dbSchemeUri.fsPath,
+    upgradesPath,
+    true,
+    query.queryDbscheme
+  );
+
+  if (!matchesTarget) {
+    reportNoUpgradePath(qlProgram, query);
+  }
+  const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token);
+  if (result.compiledUpgrade === undefined) {
+    const error = result.error || '[no error message available]';
+    throw new Error(error);
+  }
+  // We can upgrade to the actual target
+  qlProgram.dbschemePath = query.queryDbscheme;
+  // We are new enough that we will always support single file upgrades.
+  return result.compiledUpgrade;
+}
+
+
+
+export async function compileAndRunQueryAgainstDatabase(
+  cliServer: cli.CodeQLCliServer,
+  qs: qsClient.QueryServerClient,
+  dbItem: DatabaseItem,
+  initialInfo: InitialQueryInfo,
+  queryStorageDir: string,
+  progress: ProgressCallback,
+  token: CancellationToken,
+  templates?: Record<string, string>,
+  queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
+): Promise<QueryWithResults> {
+  if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
+    throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
+  }
+
+  // Get the workspace folder paths.
+  const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
+  // Figure out the library path for the query.
+  const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath);
+
+  if (!packConfig.dbscheme) {
+    throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
+  }
+
+  // Check whether the query has an entirely different schema from the
+  // database. (Queries that merely need the database to be upgraded
+  // won't trigger this check)
+  // This test will produce confusing results if we ever change the name of the database schema files.
+  const querySchemaName = path.basename(packConfig.dbscheme);
+  const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath);
+  if (querySchemaName != dbSchemaName) {
+    void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
+    throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`);
+  }
+
+  const qlProgram: messages.QlProgram = {
+    // The project of the current document determines which library path
+    // we use. The `libraryPath` field in this server message is relative
+    // to the workspace root, not to the project root.
+    libraryPath: packConfig.libraryPath,
+    // Since we are compiling and running a query against a database,
+    // we use the database's DB scheme here instead of the DB scheme
+    // from the current document's project.
+    dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
+    queryPath: initialInfo.queryPath
+  };
+
+  // Read the query metadata if possible, to use in the UI.
+  const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
+
+  let availableMlModels: cli.MlModelInfo[] = [];
+  if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
+    void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
+  } else {
+    try {
+      availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
+      if (availableMlModels.length) {
+        void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
+      } else {
+        void logger.log('Did not find any available ML models.');
+      }
+    } catch (e) {
+      const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
+        `query without any ML models: ${e}.`;
+      void showAndLogErrorMessage(message);
+    }
+  }
+
+  const hasMetadataFile = (await dbItem.hasMetadataFile());
+  const query = new QueryInProgress(
+    path.join(queryStorageDir, initialInfo.id),
+    dbItem.databaseUri.fsPath,
+    hasMetadataFile,
+    packConfig.dbscheme,
+    initialInfo.quickEvalPosition,
+    metadata,
+    templates
+  );
+  await query.queryEvalInfo.createTimestampFile();
+
+  let upgradeDir: tmp.DirectoryResult | undefined;
+  try {
+    let upgradeQlo;
+    if (await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
+      upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
+      upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
+    } else {
+      await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
+    }
+    let errors;
+    try {
+      errors = await query.compile(qs, qlProgram, progress, token);
+    } catch (e) {
+      if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+        return createSyntheticResult(query, 'Query cancelled');
+      } else {
+        throw e;
+      }
+    }
+
+    if (errors.length === 0) {
+      const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
+      if (result.resultType !== messages.QueryResultType.SUCCESS) {
+        const message = result.message || 'Failed to run query';
+        void logger.log(message);
+        void showAndLogErrorMessage(message);
+      }
+      const message = formatLegacyMessage(result);
+
+      return {
+        query: query.queryEvalInfo,
+        message,
+        result,
+        successful: result.resultType == messages.QueryResultType.SUCCESS,
+        logFileLocation: result.logFileLocation,
+        dispose: () => {
+          qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+        }
+      };
+    } else {
+      // Error dialogs are limited in size and scrollability,
+      // so we include a general description of the problem,
+      // and direct the user to the output window for the detailed compilation messages.
+      // However we don't show quick eval errors there so we need to display them anyway.
+      void qs.logger.log(
+        `Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
+        { additionalLogLocation: query.queryEvalInfo.logPath }
+      );
+
+      const formattedMessages: string[] = [];
+
+      for (const error of errors) {
+        const message = error.message || '[no error message available]';
+        const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
+        formattedMessages.push(formatted);
+        void qs.logger.log(formatted, { additionalLogLocation: query.queryEvalInfo.logPath });
+      }
+      if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
+        // If there are more than 2 error messages, they will not be displayed well in a popup
+        // and will be trimmed by the function displaying the error popup. Accordingly, we only
+        // try to show the errors if there are 2 or less, otherwise we direct the user to the log.
+        void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
+      } else {
+        void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
+      }
+      return createSyntheticResult(query, 'Query had compilation errors');
+    }
+  } finally {
+    try {
+      await upgradeDir?.cleanup();
+    } catch (e) {
+      void qs.logger.log(
+        `Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
+        { additionalLogLocation: query.queryEvalInfo.logPath }
+      );
+    }
+  }
+}
+
+
+const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
+  ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
+  ' and choose CodeQL Query Server from the dropdown.';
+
+export function formatLegacyMessage(result: messages.EvaluationResult) {
+  switch (result.resultType) {
+    case messages.QueryResultType.CANCELLATION:
+      return `cancelled after ${Math.round(result.evaluationTime / 1000)} seconds`;
+    case messages.QueryResultType.OOM:
+      return 'out of memory';
+    case messages.QueryResultType.SUCCESS:
+      return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
+    case messages.QueryResultType.TIMEOUT:
+      return `timed out after ${Math.round(result.evaluationTime / 1000)} seconds`;
+    case messages.QueryResultType.OTHER_ERROR:
+    default:
+      return result.message ? `failed: ${result.message}` : 'failed';
+  }
+}
+
+/**
+ * Create a synthetic result for a query that failed to compile.
+ */
+function createSyntheticResult(
+  query: QueryInProgress,
+  message: string,
+): QueryWithResults {
+  return {
+    query: query.queryEvalInfo,
+    message,
+    result: {
+      evaluationTime: 0,
+      queryId: 0,
+      resultType: messages.QueryResultType.OTHER_ERROR,
+      message,
+      runId: 0,
+    },
+    successful: false,
+    dispose: () => { /**/ },
+  };
+}
+
+
+function createSimpleTemplates(templates: Record<string, string> | undefined): messages.TemplateDefinitions | undefined {
+  if (!templates) {
+    return undefined;
+  }
+  const result: messages.TemplateDefinitions = {};
+  for (const key of Object.keys(templates)) {
+    result[key] = {
+      values: {
+        tuples: [[{ stringValue: templates[key] }]]
+      }
+    };
+  }
+  return result;
+}

extensions/ql-vscode/src/legacy-query-server/upgrades.ts

@@ -1,14 +1,12 @@
 import * as vscode from 'vscode';
-import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from './helpers';
-import { ProgressCallback, UserCancellationException } from './commandRunner';
-import { logger } from './logging';
-import * as messages from './pure/messages';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from '../helpers';
+import { ProgressCallback, UserCancellationException } from '../commandRunner';
+import { logger } from '../logging';
+import * as messages from '../pure/legacy-messages';
 import * as qsClient from './queryserver-client';
-import { upgradesTmpDir } from './run-queries';
 import * as tmp from 'tmp-promise';
 import * as path from 'path';
-import * as semver from 'semver';
-import { DatabaseItem } from './databases';
+import { DatabaseItem } from '../databases';
 
 /**
  * Maximum number of lines to include from database upgrade message,
@@ -17,17 +15,6 @@ import { DatabaseItem } from './databases';
  */
 const MAX_UPGRADE_MESSAGE_LINES = 10;
 
-/**
- * Check that we support non-destructive upgrades.
- *
- * This requires 3 features. The ability to compile an upgrade sequence; The ability to
- * run a non-destructive upgrades as a query; the ability to specify a target when
- * resolving upgrades. We check for a version of codeql that has all three features.
- */
-export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QueryServerClient): Promise<boolean> {
-  return semver.gte(await qs.cliServer.getVersion(), '2.4.2');
-}
-
 
 /**
  * Compile a database upgrade sequence.
@@ -35,16 +22,16 @@ export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QuerySer
  */
 export async function compileDatabaseUpgradeSequence(
   qs: qsClient.QueryServerClient,
-  db: DatabaseItem,
+  dbItem: DatabaseItem,
   resolvedSequence: string[],
   currentUpgradeTmp: tmp.DirectoryResult,
   progress: ProgressCallback,
   token: vscode.CancellationToken
 ): Promise<messages.CompileUpgradeSequenceResult> {
-  if (db.contents === undefined || db.contents.dbSchemeUri === undefined) {
+  if (dbItem.contents === undefined || dbItem.contents.dbSchemeUri === undefined) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
-  if (!await hasNondestructiveUpgradeCapabilities(qs)) {
+  if (!await qs.cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
     throw new Error('The version of codeql is too old to run non-destructive upgrades.');
   }
   // If possible just compile the upgrade sequence
@@ -56,14 +43,14 @@ export async function compileDatabaseUpgradeSequence(
 
 async function compileDatabaseUpgrade(
   qs: qsClient.QueryServerClient,
-  db: DatabaseItem,
+  dbItem: DatabaseItem,
   targetDbScheme: string,
   resolvedSequence: string[],
   currentUpgradeTmp: tmp.DirectoryResult,
   progress: ProgressCallback,
   token: vscode.CancellationToken
 ): Promise<messages.CompileUpgradeResult> {
-  if (!db.contents?.dbSchemeUri) {
+  if (!dbItem.contents?.dbSchemeUri) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
   // We have the upgrades we want but compileUpgrade
@@ -78,7 +65,7 @@ async function compileDatabaseUpgrade(
   });
   return qs.sendRequest(messages.compileUpgrade, {
     upgrade: {
-      fromDbscheme: db.contents.dbSchemeUri.fsPath,
+      fromDbscheme: dbItem.contents.dbSchemeUri.fsPath,
       toDbscheme: targetDbScheme,
       additionalUpgrades: Array.from(uniqueParentDirs)
     },
@@ -159,18 +146,18 @@ function getUpgradeDescriptions(compiled: messages.CompiledUpgrades): messages.U
  */
 export async function upgradeDatabaseExplicit(
   qs: qsClient.QueryServerClient,
-  db: DatabaseItem,
+  dbItem: DatabaseItem,
   progress: ProgressCallback,
   token: vscode.CancellationToken,
 ): Promise<messages.RunUpgradeResult | undefined> {
 
   const searchPath: string[] = getOnDiskWorkspaceFolders();
 
-  if (!db?.contents?.dbSchemeUri) {
+  if (!dbItem?.contents?.dbSchemeUri) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
   const upgradeInfo = await qs.cliServer.resolveUpgrades(
-    db.contents.dbSchemeUri.fsPath,
+    dbItem.contents.dbSchemeUri.fsPath,
     searchPath,
     false
   );
@@ -180,11 +167,11 @@ export async function upgradeDatabaseExplicit(
   if (finalDbscheme === undefined) {
     throw new Error('Could not determine target dbscheme to upgrade to.');
   }
-  const currentUpgradeTmp = await tmp.dir({ dir: upgradesTmpDir.name, prefix: 'upgrade_', keep: false, unsafeCleanup: true });
+  const currentUpgradeTmp = await tmp.dir({ dir: tmpDir.name, prefix: 'upgrade_', keep: false, unsafeCleanup: true });
   try {
     let compileUpgradeResult: messages.CompileUpgradeResult;
     try {
-      compileUpgradeResult = await compileDatabaseUpgrade(qs, db, finalDbscheme, scripts, currentUpgradeTmp, progress, token);
+      compileUpgradeResult = await compileDatabaseUpgrade(qs, dbItem, finalDbscheme, scripts, currentUpgradeTmp, progress, token);
     }
     catch (e) {
       void showAndLogErrorMessage(`Compilation of database upgrades failed: ${e}`);
@@ -200,13 +187,20 @@ export async function upgradeDatabaseExplicit(
       return;
     }
 
-    await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, db, qs.cliServer.quiet);
+    await checkAndConfirmDatabaseUpgrade(compileUpgradeResult.compiledUpgrades, dbItem, qs.cliServer.quiet);
 
     try {
       void qs.logger.log('Running the following database upgrade:');
 
       getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n');
-      return await runDatabaseUpgrade(qs, db, compileUpgradeResult.compiledUpgrades, progress, token);
+      const result = await runDatabaseUpgrade(qs, dbItem, compileUpgradeResult.compiledUpgrades, progress, token);
+
+      // TODO Can remove the next lines when https://github.com/github/codeql-team/issues/1241 is fixed
+      // restart the query server to avoid a bug in the CLI where the upgrade is applied, but the old dbscheme
+      // is still cached in memory.
+
+      await qs.restartQueryServer(progress, token);
+      return result;
     }
     catch (e) {
       void showAndLogErrorMessage(`Database upgrade failed: ${e}`);

extensions/ql-vscode/src/log-insights/join-order.ts

@@ -0,0 +1,462 @@
+import * as I from 'immutable';
+import { EvaluationLogProblemReporter, EvaluationLogScanner, EvaluationLogScannerProvider } from './log-scanner';
+import { InLayer, ComputeRecursive, SummaryEvent, PipelineRun, ComputeSimple } from './log-summary';
+
+/**
+ * Like `max`, but returns 0 if no meaningful maximum can be computed.
+ */
+function safeMax(it?: Iterable<number>) {
+  const m = Math.max(...(it || []));
+  return Number.isFinite(m) ? m : 0;
+}
+
+/**
+ * Compute a key for the maps that that is sent to report generation.
+ * Should only be used on events that are known to define queryCausingWork.
+ */
+function makeKey(
+  queryCausingWork: string | undefined,
+  predicate: string,
+  suffix = ''
+): string {
+  if (queryCausingWork === undefined) {
+    throw new Error(
+      'queryCausingWork was not defined on an event we expected it to be defined for!'
+    );
+  }
+  return `${queryCausingWork}:${predicate}${suffix ? ' ' + suffix : ''}`;
+}
+
+const DEPENDENT_PREDICATES_REGEXP = (() => {
+  const regexps = [
+    // SCAN id
+    String.raw`SCAN\s+([0-9a-zA-Z:#_]+)\s`,
+    // JOIN id WITH id
+    String.raw`JOIN\s+([0-9a-zA-Z:#_]+)\s+WITH\s+([0-9a-zA-Z:#_]+)\s`,
+    // AGGREGATE id, id
+    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+)\s*,\s+([0-9a-zA-Z:#_]+)`,
+    // id AND NOT id
+    String.raw`([0-9a-zA-Z:#_]+)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+)`,
+    // INVOKE HIGHER-ORDER RELATION rel ON <id, ..., id>
+    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+)((?:,[0-9a-zA-Z:#_<>]+)*)>`,
+    // SELECT id
+    String.raw`SELECT\s+([0-9a-zA-Z:#_]+)`
+  ];
+  return new RegExp(
+    `${String.raw`\{[0-9]+\}\s+[0-9a-zA-Z]+\s=\s(?:` + regexps.join('|')})`
+  );
+})();
+
+function getDependentPredicates(operations: string[]): I.List<string> {
+  return I.List(operations).flatMap(operation => {
+    const matches = DEPENDENT_PREDICATES_REGEXP.exec(operation.trim());
+    if (matches !== null) {
+      return I.List(matches)
+        .rest() // Skip the first group as it's just the entire string
+        .filter(x => !!x && !x.match('r[0-9]+|PRIMITIVE')) // Only keep the references to predicates.
+        .flatMap(x => x.split(',')) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
+        .filter(x => !!x); // Remove empty strings
+    } else {
+      return I.List();
+    }
+  });
+}
+
+function getMainHash(event: InLayer | ComputeRecursive): string {
+  switch (event.evaluationStrategy) {
+    case 'IN_LAYER':
+      return event.mainHash;
+    case 'COMPUTE_RECURSIVE':
+      return event.raHash;
+  }
+}
+
+/**
+ * Sum arrays a and b element-wise. The shorter array is padded with 0s if the arrays are not the same length.
+ */
+function pointwiseSum(a: Int32Array, b: Int32Array, problemReporter: EvaluationLogProblemReporter): Int32Array {
+  function reportIfInconsistent(ai: number, bi: number) {
+    if (ai === -1 && bi !== -1) {
+      problemReporter.log(
+        `Operation was not evaluated in the first pipeline, but it was evaluated in the accumulated pipeline (with tuple count ${bi}).`
+      );
+    }
+    if (ai !== -1 && bi === -1) {
+      problemReporter.log(
+        `Operation was evaluated in the first pipeline (with tuple count ${ai}), but it was not evaluated in the accumulated pipeline.`
+      );
+    }
+  }
+
+  const length = Math.max(a.length, b.length);
+  const result = new Int32Array(length);
+  for (let i = 0; i < length; i++) {
+    const ai = a[i] || 0;
+    const bi = b[i] || 0;
+    // -1 is used to represent the absence of a tuple count for a line in the pretty-printed RA (e.g. an empty line), so we ignore those.
+    if (i < a.length && i < b.length && (ai === -1 || bi === -1)) {
+      result[i] = -1;
+      reportIfInconsistent(ai, bi);
+    } else {
+      result[i] = ai + bi;
+    }
+  }
+  return result;
+}
+
+function pushValue<K, V>(m: Map<K, V[]>, k: K, v: V) {
+  if (!m.has(k)) {
+    m.set(k, []);
+  }
+  m.get(k)!.push(v);
+  return m;
+}
+
+function computeJoinOrderBadness(
+  maxTupleCount: number,
+  maxDependentPredicateSize: number,
+  resultSize: number
+): number {
+  return maxTupleCount / Math.max(maxDependentPredicateSize, resultSize);
+}
+
+/**
+ * A bucket contains the pointwise sum of the tuple counts, result sizes and dependent predicate sizes
+ * For each (predicate, order) in an SCC, we will compute a bucket.
+ */
+interface Bucket {
+  tupleCounts: Int32Array;
+  resultSize: number;
+  dependentPredicateSizes: I.Map<string, number>;
+}
+
+class JoinOrderScanner implements EvaluationLogScanner {
+  // Map a predicate hash to its result size
+  private readonly predicateSizes = new Map<string, number>();
+  private readonly layerEvents = new Map<string, (ComputeRecursive | InLayer)[]>();
+  // Map a key of the form 'query-with-demand : predicate name' to its badness input.
+  private readonly maxTupleCountMap = new Map<string, number[]>();
+  private readonly resultSizeMap = new Map<string, number[]>();
+  private readonly maxDependentPredicateSizeMap = new Map<string, number[]>();
+  private readonly joinOrderMetricMap = new Map<string, number>();
+
+  constructor(
+    private readonly problemReporter: EvaluationLogProblemReporter,
+    private readonly warningThreshold: number) {
+  }
+
+  public onEvent(event: SummaryEvent): void {
+    if (
+      event.completionType !== undefined &&
+      event.completionType !== 'SUCCESS'
+    ) {
+      return; // Skip any evaluation that wasn't successful
+    }
+
+    this.recordPredicateSizes(event);
+    this.computeBadnessMetric(event);
+  }
+
+  public onDone(): void {
+    void this;
+  }
+
+  private recordPredicateSizes(event: SummaryEvent): void {
+    switch (event.evaluationStrategy) {
+      case 'EXTENSIONAL':
+      case 'COMPUTED_EXTENSIONAL':
+      case 'COMPUTE_SIMPLE':
+      case 'CACHACA':
+      case 'CACHE_HIT': {
+        this.predicateSizes.set(event.raHash, event.resultSize);
+        break;
+      }
+      case 'SENTINEL_EMPTY': {
+        this.predicateSizes.set(event.raHash, 0);
+        break;
+      }
+      case 'COMPUTE_RECURSIVE':
+      case 'IN_LAYER': {
+        this.predicateSizes.set(event.raHash, event.resultSize);
+        // layerEvents are indexed by the mainHash.
+        const hash = getMainHash(event);
+        if (!this.layerEvents.has(hash)) {
+          this.layerEvents.set(hash, []);
+        }
+        this.layerEvents.get(hash)!.push(event);
+        break;
+      }
+    }
+  }
+
+  private reportProblemIfNecessary(event: SummaryEvent, iteration: number, metric: number): void {
+    if (metric >= this.warningThreshold) {
+      this.problemReporter.reportProblem(event.predicateName, event.raHash, iteration,
+        `Relation '${event.predicateName}' has an inefficient join order. Its join order metric is ${metric.toFixed(2)}, which is larger than the threshold of ${this.warningThreshold.toFixed(2)}.`);
+    }
+  }
+
+  private computeBadnessMetric(event: SummaryEvent): void {
+    if (
+      event.completionType !== undefined &&
+      event.completionType !== 'SUCCESS'
+    ) {
+      return; // Skip any evaluation that wasn't successful
+    }
+    switch (event.evaluationStrategy) {
+      case 'COMPUTE_SIMPLE': {
+        if (!event.pipelineRuns) {
+          // skip if the optional pipelineRuns field is not present.
+          break;
+        }
+        // Compute the badness metric for a non-recursive predicate. The metric in this case is defined as:
+        // badness = (max tuple count in the pipeline) / (largest predicate this pipeline depends on)
+        const key = makeKey(event.queryCausingWork, event.predicateName);
+        const resultSize = event.resultSize;
+
+        // There is only one entry in `pipelineRuns` if it's a non-recursive predicate.
+        const { maxTupleCount, maxDependentPredicateSize } =
+          this.badnessInputsForNonRecursiveDelta(event.pipelineRuns[0], event);
+
+        if (maxDependentPredicateSize > 0) {
+          pushValue(this.maxTupleCountMap, key, maxTupleCount);
+          pushValue(this.resultSizeMap, key, resultSize);
+          pushValue(
+            this.maxDependentPredicateSizeMap,
+            key,
+            maxDependentPredicateSize
+          );
+          const metric = computeJoinOrderBadness(maxTupleCount, maxDependentPredicateSize, resultSize!);
+          this.joinOrderMetricMap.set(key, metric);
+          this.reportProblemIfNecessary(event, 0, metric);
+        }
+        break;
+      }
+
+      case 'COMPUTE_RECURSIVE': {
+        // Compute the badness metric for a recursive predicate for each ordering.
+        const sccMetricInput = this.badnessInputsForRecursiveDelta(event);
+        // Loop through each predicate in the SCC
+        sccMetricInput.forEach((buckets, predicate) => {
+          // Loop through each ordering of the predicate
+          buckets.forEach((bucket, raReference) => {
+            // Format the key as demanding-query:name (ordering)
+            const key = makeKey(
+              event.queryCausingWork,
+              predicate,
+              `(${raReference})`
+            );
+            const maxTupleCount = Math.max(...bucket.tupleCounts);
+            const resultSize = bucket.resultSize;
+            const maxDependentPredicateSize = Math.max(
+              ...bucket.dependentPredicateSizes.values()
+            );
+
+            if (maxDependentPredicateSize > 0) {
+              pushValue(this.maxTupleCountMap, key, maxTupleCount);
+              pushValue(this.resultSizeMap, key, resultSize);
+              pushValue(
+                this.maxDependentPredicateSizeMap,
+                key,
+                maxDependentPredicateSize
+              );
+              const metric = computeJoinOrderBadness(maxTupleCount, maxDependentPredicateSize, resultSize);
+              const oldMetric = this.joinOrderMetricMap.get(key);
+              if ((oldMetric === undefined) || (metric > oldMetric)) {
+                this.joinOrderMetricMap.set(key, metric);
+              }
+            }
+          });
+        });
+        break;
+      }
+    }
+  }
+
+  /**
+   * Iterate through an SCC with main node `event`.
+   */
+  private iterateSCC(
+    event: ComputeRecursive,
+    func: (
+      inLayerEvent: ComputeRecursive | InLayer,
+      run: PipelineRun,
+      iteration: number
+    ) => void
+  ): void {
+    const sccEvents = this.layerEvents.get(event.raHash)!;
+    const nextPipeline: number[] = new Array(sccEvents.length).fill(0);
+
+    const maxIteration = Math.max(
+      ...sccEvents.map(e => e.predicateIterationMillis.length)
+    );
+
+    for (let iteration = 0; iteration < maxIteration; ++iteration) {
+      // Loop through each predicate in this iteration
+      for (let predicate = 0; predicate < sccEvents.length; ++predicate) {
+        const inLayerEvent = sccEvents[predicate];
+        const iterationTime =
+          inLayerEvent.predicateIterationMillis.length <= iteration
+            ? -1
+            : inLayerEvent.predicateIterationMillis[iteration];
+        if (iterationTime != -1) {
+          const run: PipelineRun =
+            inLayerEvent.pipelineRuns[nextPipeline[predicate]++];
+          func(inLayerEvent, run, iteration);
+        }
+      }
+    }
+  }
+
+  /**
+   * Compute the maximum tuple count and maximum dependent predicate size for a non-recursive pipeline
+   */
+  private badnessInputsForNonRecursiveDelta(
+    pipelineRun: PipelineRun,
+    event: ComputeSimple
+  ): { maxTupleCount: number; maxDependentPredicateSize: number } {
+    const dependentPredicateSizes = Object.values(event.dependencies).map(hash =>
+      this.predicateSizes.get(hash) ?? 0  // Should always be present, but zero is a safe default.
+    );
+    const maxDependentPredicateSize = safeMax(dependentPredicateSizes);
+    return {
+      maxTupleCount: safeMax(pipelineRun.counts),
+      maxDependentPredicateSize: maxDependentPredicateSize
+    };
+  }
+
+  private prevDeltaSizes(event: ComputeRecursive, predicate: string, i: number) {
+    // If an iteration isn't present in the map it means it was skipped because the optimizer
+    // inferred that it was empty. So its size is 0.
+    return this.curDeltaSizes(event, predicate, i - 1);
+  }
+
+  private curDeltaSizes(event: ComputeRecursive, predicate: string, i: number) {
+    // If an iteration isn't present in the map it means it was skipped because the optimizer
+    // inferred that it was empty. So its size is 0.
+    return (
+      this.layerEvents.get(event.raHash)?.find(x => x.predicateName === predicate)?.deltaSizes[i] ?? 0
+    );
+  }
+
+  /**
+   * Compute the metric dependent predicate sizes and the result size for a predicate in an SCC.
+   */
+  private badnessInputsForLayer(
+    event: ComputeRecursive,
+    inLayerEvent: InLayer | ComputeRecursive,
+    raReference: string,
+    iteration: number
+  ) {
+    const dependentPredicates = getDependentPredicates(
+      inLayerEvent.ra[raReference]
+    );
+    let dependentPredicateSizes: I.Map<string, number>;
+    // We treat the base case as a non-recursive pipeline. In that case, the dependent predicates are
+    // the dependencies of the base case and the cur_deltas.
+    if (raReference === 'base') {
+      dependentPredicateSizes = I.Map(
+        dependentPredicates.map((pred): [string, number] => {
+          // A base case cannot contain a `prev_delta`, but it can contain a `cur_delta`.
+          let size = 0;
+          if (pred.endsWith('#cur_delta')) {
+            size = this.curDeltaSizes(
+              event,
+              pred.slice(0, -'#cur_delta'.length),
+              iteration
+            );
+          } else {
+            const hash = event.dependencies[pred];
+            size = this.predicateSizes.get(hash)!;
+          }
+          return [pred, size];
+        })
+      );
+    } else {
+      // It's a non-base case in a recursive pipeline. In that case, the dependent predicates are
+      // only the prev_deltas.
+      dependentPredicateSizes = I.Map(
+        dependentPredicates
+          .flatMap(pred => {
+            // If it's actually a prev_delta
+            if (pred.endsWith('#prev_delta')) {
+              // Return the predicate without the #prev_delta suffix.
+              return [pred.slice(0, -'#prev_delta'.length)];
+            } else {
+              // Not a recursive delta. Skip it.
+              return [];
+            }
+          })
+          .map((prev): [string, number] => {
+            const size = this.prevDeltaSizes(event, prev, iteration);
+            return [prev, size];
+          })
+      );
+    }
+
+    const deltaSize = inLayerEvent.deltaSizes[iteration];
+    return { dependentPredicateSizes, deltaSize };
+  }
+
+  /**
+   * Compute the metric input for all the events in a SCC that starts with main node `event`
+   */
+  private badnessInputsForRecursiveDelta(event: ComputeRecursive): Map<string, Map<string, Bucket>> {
+    // nameToOrderToBucket : predicate name -> ordering (i.e., standard, order_500000, etc.) -> bucket
+    const nameToOrderToBucket = new Map<string, Map<string, Bucket>>();
+
+    // Iterate through the SCC and compute the metric inputs
+    this.iterateSCC(event, (inLayerEvent, run, iteration) => {
+      const raReference = run.raReference;
+      const predicateName = inLayerEvent.predicateName;
+      if (!nameToOrderToBucket.has(predicateName)) {
+        nameToOrderToBucket.set(predicateName, new Map());
+      }
+      const orderTobucket = nameToOrderToBucket.get(predicateName)!;
+      if (!orderTobucket.has(raReference)) {
+        orderTobucket.set(raReference, {
+          tupleCounts: new Int32Array(0),
+          resultSize: 0,
+          dependentPredicateSizes: I.Map()
+        });
+      }
+
+      const { dependentPredicateSizes, deltaSize } = this.badnessInputsForLayer(
+        event,
+        inLayerEvent,
+        raReference,
+        iteration
+      );
+
+      const bucket = orderTobucket.get(raReference)!;
+      // Pointwise sum the tuple counts
+      const newTupleCounts = pointwiseSum(
+        bucket.tupleCounts,
+        new Int32Array(run.counts),
+        this.problemReporter
+      );
+      const resultSize = bucket.resultSize + deltaSize;
+      // Pointwise sum the deltas.
+      const newDependentPredicateSizes = bucket.dependentPredicateSizes.mergeWith(
+        (oldSize, newSize) => oldSize + newSize,
+        dependentPredicateSizes
+      );
+      orderTobucket.set(raReference, {
+        tupleCounts: newTupleCounts,
+        resultSize: resultSize,
+        dependentPredicateSizes: newDependentPredicateSizes
+      });
+    });
+    return nameToOrderToBucket;
+  }
+}
+
+export class JoinOrderScannerProvider implements EvaluationLogScannerProvider {
+  constructor(private readonly getThreshdold: () => number) {
+  }
+
+  public createScanner(problemReporter: EvaluationLogProblemReporter): EvaluationLogScanner {
+    const threshold = this.getThreshdold();
+    return new JoinOrderScanner(problemReporter, threshold);
+  }
+}

extensions/ql-vscode/src/log-insights/jsonl-reader.ts

@@ -0,0 +1,23 @@
+import * as fs from 'fs-extra';
+
+/**
+ * Read a file consisting of multiple JSON objects. Each object is separated from the previous one
+ * by a double newline sequence. This is basically a more human-readable form of JSONL.
+ *
+ * The current implementation reads the entire text of the document into memory, but in the future
+ * it will stream the document to improve the performance with large documents.
+ *
+ * @param path The path to the file.
+ * @param handler Callback to be invoked for each top-level JSON object in order.
+ */
+export async function readJsonlFile(path: string, handler: (value: any) => Promise<void>): Promise<void> {
+  const logSummary = await fs.readFile(path, 'utf-8');
+
+  // Remove newline delimiters because summary is in .jsonl format.
+  const jsonSummaryObjects: string[] = logSummary.split(/\r?\n\r?\n/g);
+
+  for (const obj of jsonSummaryObjects) {
+    const jsonObj = JSON.parse(obj);
+    await handler(jsonObj);
+  }
+}

extensions/ql-vscode/src/log-insights/log-scanner-service.ts

@@ -0,0 +1,109 @@
+import { Diagnostic, DiagnosticSeverity, languages, Range, Uri } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+import { QueryHistoryManager } from '../query-history';
+import { QueryHistoryInfo } from '../query-history-info';
+import { EvaluationLogProblemReporter, EvaluationLogScannerSet } from './log-scanner';
+import { PipelineInfo, SummarySymbols } from './summary-parser';
+import * as fs from 'fs-extra';
+import { logger } from '../logging';
+
+/**
+ * Compute the key used to find a predicate in the summary symbols.
+ * @param name The name of the predicate.
+ * @param raHash The RA hash of the predicate.
+ * @returns The key of the predicate, consisting of `name@shortHash`, where `shortHash` is the first
+ * eight characters of `raHash`.
+ */
+function predicateSymbolKey(name: string, raHash: string): string {
+  return `${name}@${raHash.substring(0, 8)}`;
+}
+
+/**
+ * Implementation of `EvaluationLogProblemReporter` that generates `Diagnostic` objects to display
+ * in the VS Code "Problems" view.
+ */
+class ProblemReporter implements EvaluationLogProblemReporter {
+  public readonly diagnostics: Diagnostic[] = [];
+
+  constructor(private readonly symbols: SummarySymbols | undefined) {
+  }
+
+  public reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void {
+    const nameWithHash = predicateSymbolKey(predicateName, raHash);
+    const predicateSymbol = this.symbols?.predicates[nameWithHash];
+    let predicateInfo: PipelineInfo | undefined = undefined;
+    if (predicateSymbol !== undefined) {
+      predicateInfo = predicateSymbol.iterations[iteration];
+    }
+    if (predicateInfo !== undefined) {
+      const range = new Range(predicateInfo.raStartLine, 0, predicateInfo.raEndLine + 1, 0);
+      this.diagnostics.push(new Diagnostic(range, message, DiagnosticSeverity.Error));
+    }
+  }
+
+  public log(message: string): void {
+    void logger.log(message);
+  }
+}
+
+export class LogScannerService extends DisposableObject {
+  public readonly scanners = new EvaluationLogScannerSet();
+  private readonly diagnosticCollection = this.push(languages.createDiagnosticCollection('ql-eval-log'));
+  private currentItem: QueryHistoryInfo | undefined = undefined;
+
+  constructor(qhm: QueryHistoryManager) {
+    super();
+
+    this.push(qhm.onDidChangeCurrentQueryItem(async (item) => {
+      if (item !== this.currentItem) {
+        this.currentItem = item;
+        await this.scanEvalLog(item);
+      }
+    }));
+
+    this.push(qhm.onDidCompleteQuery(async (item) => {
+      if (item === this.currentItem) {
+        await this.scanEvalLog(item);
+      }
+    }));
+  }
+
+  /**
+   * Scan the evaluation log for a query, and report any diagnostics.
+   *
+   * @param query The query whose log is to be scanned.
+   */
+  public async scanEvalLog(
+    query: QueryHistoryInfo | undefined
+  ): Promise<void> {
+    this.diagnosticCollection.clear();
+
+    if ((query?.t !== 'local')
+      || (query.evalLogSummaryLocation === undefined)
+      || (query.jsonEvalLogSummaryLocation === undefined)) {
+      return;
+    }
+
+    const diagnostics = await this.scanLog(query.jsonEvalLogSummaryLocation, query.evalLogSummarySymbolsLocation);
+    const uri = Uri.file(query.evalLogSummaryLocation);
+    this.diagnosticCollection.set(uri, diagnostics);
+  }
+
+  /**
+   * Scan the evaluator summary log for problems, using the scanners for all registered providers.
+   * @param jsonSummaryLocation The file path of the JSON summary log.
+   * @param symbolsLocation The file path of the symbols file for the human-readable log summary.
+   * @returns An array of `Diagnostic`s representing the problems found by scanners.
+   */
+  private async scanLog(jsonSummaryLocation: string, symbolsLocation: string | undefined): Promise<Diagnostic[]> {
+    let symbols: SummarySymbols | undefined = undefined;
+    if (symbolsLocation !== undefined) {
+      symbols = JSON.parse(await fs.readFile(symbolsLocation, { encoding: 'utf-8' }));
+    }
+    const problemReporter = new ProblemReporter(symbols);
+
+    await this.scanners.scanLog(jsonSummaryLocation, problemReporter);
+
+    return problemReporter.diagnostics;
+  }
+}

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -0,0 +1,103 @@
+import { SummaryEvent } from './log-summary';
+import { readJsonlFile } from './jsonl-reader';
+
+/**
+ * Callback interface used to report diagnostics from a log scanner.
+ */
+export interface EvaluationLogProblemReporter {
+  /**
+   * Report a potential problem detected in the evaluation log.
+   *
+   * @param predicateName The mangled name of the predicate with the problem.
+   * @param raHash The RA hash of the predicate with the problem.
+   * @param iteration The iteration number with the problem. For a non-recursive predicate, this
+   * must be zero.
+   * @param message The problem message.
+   */
+  reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void;
+
+  /**
+   * Log a message about a problem in the implementation of the scanner. These will typically be
+   * displayed separate from any problems reported via `reportProblem()`.
+   */
+  log(message: string): void;
+}
+
+/**
+ * Interface implemented by a log scanner. Instances are created via
+ * `EvaluationLogScannerProvider.createScanner()`.
+ */
+export interface EvaluationLogScanner {
+  /**
+   * Called for each event in the log summary, in order. The implementation can report problems via
+   * the `EvaluationLogProblemReporter` interface that was supplied to `createScanner()`.
+   * @param event The log summary event.
+   */
+  onEvent(event: SummaryEvent): void;
+  /**
+   * Called after all events in the log summary have been processed. The implementation can report
+   * problems via the `EvaluationLogProblemReporter` interface that was supplied to
+   * `createScanner()`.
+   */
+  onDone(): void;
+}
+
+/**
+ * A factory for log scanners. When a log is to be scanned, all registered
+ * `EvaluationLogScannerProviders` will be asked to create a new instance of `EvaluationLogScanner`
+ * to do the scanning.
+ */
+export interface EvaluationLogScannerProvider {
+  /**
+   * Create a new instance of `EvaluationLogScanner` to scan a single summary log.
+   * @param problemReporter Callback interface for reporting any problems discovered.
+   */
+  createScanner(problemReporter: EvaluationLogProblemReporter): EvaluationLogScanner;
+}
+
+/**
+ * Same as VSCode's `Disposable`, but avoids a dependency on VS Code.
+ */
+export interface Disposable {
+  dispose(): void;
+}
+
+export class EvaluationLogScannerSet {
+  private readonly scannerProviders = new Map<number, EvaluationLogScannerProvider>();
+  private nextScannerProviderId = 0;
+
+  /**
+   * Register a provider that can create instances of `EvaluationLogScanner` to scan evaluation logs
+   * for problems.
+   * @param provider The provider.
+   * @returns A `Disposable` that, when disposed, will unregister the provider.
+   */
+  public registerLogScannerProvider(provider: EvaluationLogScannerProvider): Disposable {
+    const id = this.nextScannerProviderId;
+    this.nextScannerProviderId++;
+
+    this.scannerProviders.set(id, provider);
+    return {
+      dispose: () => {
+        this.scannerProviders.delete(id);
+      }
+    };
+  }
+
+  /**
+   * Scan the evaluator summary log for problems, using the scanners for all registered providers.
+   * @param jsonSummaryLocation The file path of the JSON summary log.
+   * @param problemReporter Callback interface for reporting any problems discovered.
+   */
+  public async scanLog(jsonSummaryLocation: string, problemReporter: EvaluationLogProblemReporter): Promise<void> {
+    const scanners = [...this.scannerProviders.values()].map(p => p.createScanner(problemReporter));
+
+    await readJsonlFile(jsonSummaryLocation, async obj => {
+      scanners.forEach(scanner => {
+        scanner.onEvent(obj);
+      });
+    });
+
+    scanners.forEach(scanner => scanner.onDone());
+  }
+}

extensions/ql-vscode/src/log-insights/log-summary.ts

@@ -0,0 +1,93 @@
+export interface PipelineRun {
+  raReference: string;
+  counts: number[];
+  duplicationPercentages: number[];
+}
+
+export interface Ra {
+  [key: string]: string[];
+}
+
+export type EvaluationStrategy =
+  'COMPUTE_SIMPLE' |
+  'COMPUTE_RECURSIVE' |
+  'IN_LAYER' |
+  'COMPUTED_EXTENSIONAL' |
+  'EXTENSIONAL' |
+  'SENTINEL_EMPTY' |
+  'CACHACA' |
+  'CACHE_HIT';
+
+interface SummaryEventBase {
+  evaluationStrategy: EvaluationStrategy;
+  predicateName: string;
+  raHash: string;
+  appearsAs: { [key: string]: { [key: string]: number[] } };
+  completionType?: string;
+}
+
+interface ResultEventBase extends SummaryEventBase {
+  resultSize: number;
+}
+
+export interface ComputeSimple extends ResultEventBase {
+  evaluationStrategy: 'COMPUTE_SIMPLE';
+  ra: Ra;
+  pipelineRuns?: [PipelineRun];
+  queryCausingWork?: string;
+  dependencies: { [key: string]: string };
+}
+
+export interface ComputeRecursive extends ResultEventBase {
+  evaluationStrategy: 'COMPUTE_RECURSIVE';
+  deltaSizes: number[];
+  ra: Ra;
+  pipelineRuns: PipelineRun[];
+  queryCausingWork?: string;
+  dependencies: { [key: string]: string };
+  predicateIterationMillis: number[];
+}
+
+export interface InLayer extends ResultEventBase {
+  evaluationStrategy: 'IN_LAYER';
+  deltaSizes: number[];
+  ra: Ra;
+  pipelineRuns: PipelineRun[];
+  queryCausingWork?: string;
+  mainHash: string;
+  predicateIterationMillis: number[];
+}
+
+export interface ComputedExtensional extends ResultEventBase {
+  evaluationStrategy: 'COMPUTED_EXTENSIONAL';
+  queryCausingWork?: string;
+}
+
+export interface NonComputedExtensional extends ResultEventBase {
+  evaluationStrategy: 'EXTENSIONAL';
+  queryCausingWork?: string;
+}
+
+export interface SentinelEmpty extends SummaryEventBase {
+  evaluationStrategy: 'SENTINEL_EMPTY';
+  sentinelRaHash: string;
+}
+
+export interface Cachaca extends ResultEventBase {
+  evaluationStrategy: 'CACHACA';
+}
+
+export interface CacheHit extends ResultEventBase {
+  evaluationStrategy: 'CACHE_HIT';
+}
+
+export type Extensional = ComputedExtensional | NonComputedExtensional;
+
+export type SummaryEvent =
+  | ComputeSimple
+  | ComputeRecursive
+  | InLayer
+  | Extensional
+  | SentinelEmpty
+  | Cachaca
+  | CacheHit;

extensions/ql-vscode/src/log-insights/summary-language-support.ts

@@ -0,0 +1,154 @@
+import * as fs from 'fs-extra';
+import { RawSourceMap, SourceMapConsumer } from 'source-map';
+import { commands, Position, Selection, TextDocument, TextEditor, TextEditorRevealType, TextEditorSelectionChangeEvent, ViewColumn, window, workspace } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+import { commandRunner } from '../commandRunner';
+import { logger } from '../logging';
+import { getErrorMessage } from '../pure/helpers-pure';
+
+/** A `Position` within a specified file on disk. */
+interface PositionInFile {
+  filePath: string;
+  position: Position;
+}
+
+/**
+ * Opens the specified source location in a text editor.
+ * @param position The position (including file path) to show.
+ */
+async function showSourceLocation(position: PositionInFile): Promise<void> {
+  const document = await workspace.openTextDocument(position.filePath);
+  const editor = await window.showTextDocument(document, ViewColumn.Active);
+  editor.selection = new Selection(position.position, position.position);
+  editor.revealRange(editor.selection, TextEditorRevealType.InCenterIfOutsideViewport);
+}
+
+/**
+ * Simple language support for human-readable evaluator log summaries.
+ *
+ * This class implements the `codeQL.gotoQL` command, which jumps from RA code to the corresponding
+ * QL code that generated it. It also tracks the current selection and active editor to enable and
+ * disable that command based on whether there is a QL mapping for the current selection.
+ */
+export class SummaryLanguageSupport extends DisposableObject {
+  /**
+   * The last `TextDocument` (with language `ql-summary`) for which we tried to find a sourcemap, or
+   * `undefined` if we have not seen such a document yet.
+   */
+  private lastDocument: TextDocument | undefined = undefined;
+  /**
+   * The sourcemap for `lastDocument`, or `undefined` if there was no such sourcemap or document.
+   */
+  private sourceMap: SourceMapConsumer | undefined = undefined;
+
+  constructor() {
+    super();
+
+    this.push(window.onDidChangeActiveTextEditor(this.handleDidChangeActiveTextEditor));
+    this.push(window.onDidChangeTextEditorSelection(this.handleDidChangeTextEditorSelection));
+    this.push(workspace.onDidCloseTextDocument(this.handleDidCloseTextDocument));
+
+    this.push(commandRunner('codeQL.gotoQL', this.handleGotoQL));
+  }
+
+  /**
+   * Gets the location of the QL code that generated the RA at the current selection in the active
+   * editor, or `undefined` if there is no mapping.
+   */
+  private async getQLSourceLocation(): Promise<PositionInFile | undefined> {
+    const editor = window.activeTextEditor;
+    if (editor === undefined) {
+      return undefined;
+    }
+
+    const document = editor.document;
+    if (document.languageId !== 'ql-summary') {
+      return undefined;
+    }
+
+    if (document.uri.scheme !== 'file') {
+      return undefined;
+    }
+
+    if (this.lastDocument !== document) {
+      this.clearCache();
+
+      const mapPath = document.uri.fsPath + '.map';
+
+      try {
+        const sourceMapText = await fs.readFile(mapPath, 'utf-8');
+        const rawMap: RawSourceMap = JSON.parse(sourceMapText);
+        this.sourceMap = await new SourceMapConsumer(rawMap);
+      } catch (e: unknown) {
+        // Error reading sourcemap. Pretend there was no sourcemap.
+        void logger.log(`Error reading sourcemap file '${mapPath}': ${getErrorMessage(e)}`);
+        this.sourceMap = undefined;
+      }
+      this.lastDocument = document;
+    }
+
+    if (this.sourceMap === undefined) {
+      return undefined;
+    }
+
+    const qlPosition = this.sourceMap.originalPositionFor({
+      line: editor.selection.start.line + 1,
+      column: editor.selection.start.character,
+      bias: SourceMapConsumer.GREATEST_LOWER_BOUND
+    });
+
+    if ((qlPosition.source === null) || (qlPosition.line === null)) {
+      // No position found.
+      return undefined;
+    }
+    const line = qlPosition.line - 1;  // In `source-map`, lines are 1-based...
+    const column = qlPosition.column ?? 0;  // ...but columns are 0-based :(
+
+    return {
+      filePath: qlPosition.source,
+      position: new Position(line, column)
+    };
+  }
+
+  /**
+   * Clears the cached sourcemap and its corresponding `TextDocument`.
+   */
+  private clearCache(): void {
+    if (this.sourceMap !== undefined) {
+      this.sourceMap.destroy();
+      this.sourceMap = undefined;
+      this.lastDocument = undefined;
+    }
+  }
+
+  /**
+   * Updates the `codeql.hasQLSource` context variable based on the current selection. This variable
+   * controls whether or not the `codeQL.gotoQL` command is enabled.
+   */
+  private async updateContext(): Promise<void> {
+    const position = await this.getQLSourceLocation();
+
+    await commands.executeCommand('setContext', 'codeql.hasQLSource', position !== undefined);
+  }
+
+  handleDidChangeActiveTextEditor = async (_editor: TextEditor | undefined): Promise<void> => {
+    await this.updateContext();
+  }
+
+  handleDidChangeTextEditorSelection = async (_e: TextEditorSelectionChangeEvent): Promise<void> => {
+    await this.updateContext();
+  }
+
+  handleDidCloseTextDocument = (document: TextDocument): void => {
+    if (this.lastDocument === document) {
+      this.clearCache();
+    }
+  }
+
+  handleGotoQL = async (): Promise<void> => {
+    const position = await this.getQLSourceLocation();
+    if (position !== undefined) {
+      await showSourceLocation(position);
+    }
+  };
+}

extensions/ql-vscode/src/log-insights/summary-parser.ts

@@ -0,0 +1,113 @@
+import * as fs from 'fs-extra';
+
+/**
+ * Location information for a single pipeline invocation in the RA.
+ */
+export interface PipelineInfo {
+  startLine: number;
+  raStartLine: number;
+  raEndLine: number;
+}
+
+/**
+ * Location information for a single predicate in the RA.
+ */
+export interface PredicateSymbol {
+  /**
+   * `PipelineInfo` for each iteration. A non-recursive predicate will have a single iteration `0`.
+   */
+  iterations: Record<number, PipelineInfo>;
+}
+
+/**
+ * Location information for the RA from an evaluation log. Line numbers point into the
+ * human-readable log summary.
+ */
+export interface SummarySymbols {
+  predicates: Record<string, PredicateSymbol>;
+}
+
+// Tuple counts for Expr::Expr::getParent#dispred#f0820431#ff@76d6745o:
+const NON_RECURSIVE_TUPLE_COUNT_REGEXP = /^Evaluated relational algebra for predicate (?<predicateName>\S+) with tuple counts:$/;
+// Tuple counts for Expr::Expr::getEnclosingStmt#f0820431#bf@923ddwj9 on iteration 0 running pipeline base:
+const RECURSIVE_TUPLE_COUNT_REGEXP = /^Evaluated relational algebra for predicate (?<predicateName>\S+) on iteration (?<iteration>\d+) running pipeline (?<pipeline>\S+) with tuple counts:$/;
+const RETURN_REGEXP = /^\s*return /;
+
+/**
+ * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline
+ * run.
+ *
+ * TODO: Once we're more certain about the symbol format, we should have the CLI generate this as it
+ * generates the human-readabe summary to avoid having to rely on regular expression matching of the
+ * human-readable text.
+ *
+ * @param summaryPath The path to the summary file.
+ * @param symbolsPath The path to the symbols file to generate.
+ */
+export async function generateSummarySymbolsFile(summaryPath: string, symbolsPath: string): Promise<void> {
+  const symbols = await generateSummarySymbols(summaryPath);
+  await fs.writeFile(symbolsPath, JSON.stringify(symbols));
+}
+
+/**
+ * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline
+ * run.
+ *
+ * @param fileLocation The path to the summary file.
+ * @returns Symbol information for the summary file.
+ */
+async function generateSummarySymbols(summaryPath: string): Promise<SummarySymbols> {
+  const summary = await fs.promises.readFile(summaryPath, { encoding: 'utf-8' });
+  const symbols: SummarySymbols = {
+    predicates: {}
+  };
+
+  const lines = summary.split(/\r?\n/);
+  let lineNumber = 0;
+  while (lineNumber < lines.length) {
+    const startLineNumber = lineNumber;
+    lineNumber++;
+    const startLine = lines[startLineNumber];
+    const nonRecursiveMatch = startLine.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
+    let predicateName: string | undefined = undefined;
+    let iteration = 0;
+    if (nonRecursiveMatch) {
+      predicateName = nonRecursiveMatch.groups!.predicateName;
+    } else {
+      const recursiveMatch = startLine.match(RECURSIVE_TUPLE_COUNT_REGEXP);
+      if (recursiveMatch?.groups) {
+        predicateName = recursiveMatch.groups.predicateName;
+        iteration = parseInt(recursiveMatch.groups.iteration);
+      }
+    }
+
+    if (predicateName !== undefined) {
+      const raStartLine = lineNumber;
+      let raEndLine: number | undefined = undefined;
+      while ((lineNumber < lines.length) && (raEndLine === undefined)) {
+        const raLine = lines[lineNumber];
+        const returnMatch = raLine.match(RETURN_REGEXP);
+        if (returnMatch) {
+          raEndLine = lineNumber;
+        }
+        lineNumber++;
+      }
+      if (raEndLine !== undefined) {
+        let symbol = symbols.predicates[predicateName];
+        if (symbol === undefined) {
+          symbol = {
+            iterations: {}
+          };
+          symbols.predicates[predicateName] = symbol;
+        }
+        symbol.iterations[iteration] = {
+          startLine: lineNumber,
+          raStartLine: raStartLine,
+          raEndLine: raEndLine
+        };
+      }
+    }
+  }
+
+  return symbols;
+}