Merge pull request #1591 from github/v1.7.1

Release prep v1.7.1

.gitattributes

@@ -18,4 +18,8 @@ yarn.lock                    merge=binary
 # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/gitattributes.html
 # suggests that this might interleave lines arbitrarily, but empirically
 # it keeps added chunks contiguous
-CHANGELOG.md                merge=union
+CHANGELOG.md                merge=union
+
+# Mark some JSON files containing test data as generated so they are not included
+# as part of diffs or language statistics.
+extensions/ql-vscode/src/stories/remote-queries/data/*.json     linguist-generated

.github/workflows/dependency-review.yml

@@ -1,11 +1,10 @@
 name: 'Dependency Review'
-on: 
+on:
   - pull_request
   - workflow_dispatch
 
 permissions:
-  actions: read
-  pull-requests: read
+  contents: read
 
 jobs:
   dependency-review:
@@ -14,4 +13,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@v3
       - name: 'Dependency Review'
-        uses: dsp-testing/dependency-review-action@main
+        uses: actions/dependency-review-action@v1

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '16.13.0'
+          node-version: '16.14.2'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -82,7 +82,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '16.13.0'
+          node-version: '16.14.0'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -118,6 +118,8 @@ jobs:
       - name: Run integration tests (Linux)
         if: matrix.os == 'ubuntu-latest'
         working-directory: extensions/ql-vscode
+        env:
+          VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
           sudo apt-get install xvfb
           /usr/bin/xvfb-run npm run integration
@@ -125,6 +127,8 @@ jobs:
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
         working-directory: extensions/ql-vscode
+        env:
+          VSCODE_CODEQL_GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
           npm run integration
 
@@ -135,7 +139,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.6', 'v2.8.4', 'nightly']
+        version: ['v2.6.3', 'v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.1', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
       NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
@@ -147,7 +151,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '16.13.0'
+          node-version: '16.14.0'
 
       - name: Install dependencies
         working-directory: extensions/ql-vscode
@@ -168,9 +172,6 @@ jobs:
           if [[ "${{ matrix.version }}" == "nightly" ]]
           then
             REF="codeql-cli/latest"
-          elif [[ "${{ matrix.version }}" == "v2.2.6" || "${{ matrix.version }}" == "v2.3.3" ]]
-          then
-            REF="codeql-cli/v2.4.5"
           else
             REF="codeql-cli/${{ matrix.version }}"
           fi

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '16.13.0'
+          node-version: '16.14.2'
 
       - name: Install dependencies
         run: |
@@ -147,13 +147,11 @@ jobs:
                   If this was an authentication problem, please make sure the \
                   auth token hasn't expired."
 
-  # TODO This job is currently broken and is blocked on https://github.com/github/vscode-codeql/issues/1085
   open-vsx-publish:
     name: Publish to Open VSX Registry
     needs: build
     environment: publish-open-vsx
     runs-on: ubuntu-latest
-    if: 1 == 0
     env:
       OPEN_VSX_TOKEN: ${{ secrets.OPEN_VSX_TOKEN }}
     steps:

.vscode/launch.json

@@ -12,7 +12,6 @@
         // Add a reference to a workspace to open. Eg-
         // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -36,6 +35,9 @@
       "runtimeArgs": [
         "--inspect=9229"
       ],
+      "env": {
+        "LANG": "en-US"
+      },
       "args": [
         "--exit",
         "-u",
@@ -44,9 +46,22 @@
         "--diff",
         "-r",
         "ts-node/register",
+        "-r",
+        "test/mocha.setup.js",
         "test/pure-tests/**/*.ts"
       ],
-      "port": 9229,
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
+    },
+    {
+      "name": "Launch Unit Tests - React (vscode-codeql)",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/extensions/ql-vscode/node_modules/jest/bin/jest.js",
+      "showAsyncStacks": true,
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
       "stopOnEntry": false,
       "sourceMaps": true,
       "console": "integratedTerminal",
@@ -60,10 +75,10 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
+        "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -77,11 +92,11 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
+        "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu",
         "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
@@ -95,6 +110,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "--disable-workspace-trust",
         "--disable-gpu",
         "--disable-extension",
         "eamodio.gitlens",
@@ -121,11 +137,18 @@
         // This option overrides the CLI_VERSION option.
         // "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
       },
-      "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
+    },
+    {
+      "name": "Launch Storybook",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/extensions/ql-vscode",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["run-script", "storybook"]
     }
   ]
 }

.vscode/settings.json

@@ -30,12 +30,11 @@
     "typescript",
     "typescriptreact"
   ],
-  "eslint.options": {
-    // This is necessary so that eslint can properly resolve its plugins
-    "resolvePluginsRelativeTo": "./extensions/ql-vscode"
-  },
+  // This is necessary to ensure that ESLint can find the correct configuration files and plugins.
+  "eslint.workingDirectories": ["./extensions/ql-vscode"],
   "editor.formatOnSave": false,
   "typescript.preferences.quoteStyle": "single",
   "javascript.preferences.quoteStyle": "single",
-  "editor.wordWrapColumn": 100
+  "editor.wordWrapColumn": 100,
+  "jest.rootPath": "./extensions/ql-vscode"
 }

CODEOWNERS

@@ -1,2 +1,3 @@
 **/* @github/codeql-vscode-reviewers
-/extensions/ql-vscode/src/remote-queries/ @github/code-scanning-secexp-reviewers
+**/remote-queries/ @github/code-scanning-secexp-reviewers
+**/variant-analysis/ @github/code-scanning-secexp-reviewers

CONTRIBUTING.md

@@ -29,7 +29,9 @@ Here are a few things you can do that will increase the likelihood of your pull
 
 ## Setting up a local build
 
-Make sure you have installed recent versions of vscode (>= v1.52), node (>=12.16), and npm (>= 7.5.2). Earlier versions will probably work, but we no longer test against them.
+Make sure you have installed recent versions of vscode, node, and npm. Check the `engines` block in [`package.json`](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/package.json) file for compatible versions. Earlier versions may work, but we no longer test against them.
+
+To automatically switch to the correct version of node, we recommend using [nvm](https://github.com/nvm-sh/nvm), which will pick-up the node version from `.nvmrc`.
 
 ### Installing all packages
 
@@ -56,8 +58,6 @@ We recommend that you keep `npm run watch` running in the backgound and you only
 
 1. on first checkout
 2. whenever any of the non-TypeScript resources have changed
-3. on any change to files included in one of the webviews
-   - **Important**: This is easy to forget. You must explicitly run `npm run build` whenever one of the files in the webview is changed. These are the files in the `src/view` and `src/compare/view` folders.
 
 ### Installing the extension
 
@@ -77,6 +77,20 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
+### Storybook
+
+You can use [Storybook](https://storybook.js.org/) to preview React components outside VSCode. Inside the `extensions/ql-vscode` directory, run:
+
+```shell
+npm run storybook
+```
+
+Your browser should automatically open to the Storybook UI. Stories live in the `src/stories` directory.
+
+Alternatively, you can start Storybook inside of VSCode. There is a VSCode launch configuration for starting Storybook. It can be found in the debug view.
+
+More information about Storybook can be found inside the **Overview** page once you have launched Storybook.
+
 ### Running the unit tests and integration tests that do not require a CLI instance
 
 Unit tests and many integration tests do not require a copy of the CodeQL CLI.
@@ -95,15 +109,21 @@ Running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point
 
 ### Running the integration tests
 
-The _Launch Integration Tests - With CLI_ tests require a CLI instance in order to run. There are several environment variables you can use to configure this.
+You will need to run CLI tests using a task from inside of VS Code called _Launch Integration Tests - With CLI_.
 
-From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ uncomment and change the environment variables appropriate for your purpose.
+The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
+
+From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ task, uncomment the `"${workspaceRoot}/../codeql"` line. If necessary, replace value with a path to your checkout, and then run the task.
 
 ## Releasing (write access required)
 
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
+1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
+    * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
+    * `.github/workflows/main.yml` - all the "node-version: <version>" settings
+    * `.github/workflows/release.yml` - the "node-version: <version>" setting
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
 1. Create a PR for this release:
     * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
@@ -111,19 +131,40 @@ From inside of VSCode, open the `launch.json` file and in the _Launch Integratio
     * Create a new commit with a message the same as the branch name.
     * Create a PR for this branch.
     * Wait for the PR to be merged into `main`
-1. Trigger a release build on Actions by adding a new tag on branch `main` named after the release, as above. Note that when you push to upstream, you will need to fully qualify the ref. A command like this will work:
+1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+    ```bash
+    git checkout main
+    git tag v1.3.6
+    ```
+
+   If you've accidentally created a badly named tag, you can delete it via 
+    ```bash
+    git tag -d badly-named-tag
+    ```
+1. Push the new tag up:
+
+   a. If you're using a fork of the repo:
 
     ```bash
     git push upstream refs/tags/v1.3.6
     ```
+   
+   b. If you're working straight in this repo:
+
+    ```bash
+    git push origin refs/tags/v1.3.6
+    ``` 
+   
+   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
 
     * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
     * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
-
 1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
+    * DO NOT approve the "publish" stages of the workflow yet.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
+1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
 1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
     - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
 1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.

extensions/ql-vscode/.eslintrc.js

@@ -10,7 +10,7 @@ module.exports = {
     node: true,
     es6: true,
   },
-  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended", "plugin:jest-dom/recommended"],
   rules: {
     "@typescript-eslint/no-use-before-define": 0,
     "@typescript-eslint/no-unused-vars": [

extensions/ql-vscode/.npmrc

@@ -0,0 +1,2 @@
+# Storybook requires this option to be set. See https://github.com/storybookjs/storybook/issues/18298
+legacy-peer-deps=true

extensions/ql-vscode/.nvmrc

@@ -0,0 +1 @@
+v16.14.2

extensions/ql-vscode/.storybook/main.ts

@@ -0,0 +1,19 @@
+import type { StorybookConfig } from '@storybook/core-common';
+
+const config: StorybookConfig = {
+  stories: [
+    '../src/**/*.stories.mdx',
+    '../src/**/*.stories.@(js|jsx|ts|tsx)'
+  ],
+  addons: [
+    '@storybook/addon-links',
+    '@storybook/addon-essentials',
+    '@storybook/addon-interactions'
+  ],
+  framework: '@storybook/react',
+  core: {
+    builder: '@storybook/builder-webpack5'
+  }
+};
+
+module.exports = config;

extensions/ql-vscode/.storybook/manager.ts

@@ -0,0 +1,7 @@
+import { addons } from '@storybook/addons';
+import { themes } from '@storybook/theming';
+
+addons.setConfig({
+  theme: themes.dark,
+  enableShortcuts: false,
+});

extensions/ql-vscode/.storybook/preview.ts

@@ -0,0 +1,38 @@
+import { themes } from '@storybook/theming';
+import { action } from '@storybook/addon-actions';
+
+// Allow all stories/components to use Codicons
+import '@vscode/codicons/dist/codicon.css';
+
+import '../src/stories/vscode-theme.css';
+
+// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
+export const parameters = {
+  // All props starting with `on` will automatically receive an action as a prop
+  actions: { argTypesRegex: "^on[A-Z].*" },
+  // All props matching these names will automatically get the correct control
+  controls: {
+    matchers: {
+      color: /(background|color)$/i,
+      date: /Date$/,
+    },
+  },
+  // Use a dark theme to be aligned with VSCode
+  docs: {
+    theme: themes.dark,
+  },
+  backgrounds: {
+    default: 'dark',
+    values: [
+      {
+        name: 'dark',
+        value: '#1e1e1e',
+      },
+    ],
+  }
+};
+
+(window as any).acquireVsCodeApi = () => ({
+  postMessage: action('post-vscode-message'),
+  setState: action('set-vscode-state'),
+});

extensions/ql-vscode/CHANGELOG.md

@@ -1,6 +1,58 @@
 # CodeQL for Visual Studio Code: Changelog
 
-## 1.6.2 - 4 April 2022
+## 1.7.1 - 12 October 2022
+
+- Fix a bug where it was not possible to add a database folder if the folder name starts with `db-`. [#1565](https://github.com/github/vscode-codeql/pull/1565)
+- Ensure the results view opens in an editor column beside the currently active editor. [#1557](https://github.com/github/vscode-codeql/pull/1557)
+
+## 1.7.0 - 20 September 2022
+
+- Remove ability to download databases from LGTM. [#1467](https://github.com/github/vscode-codeql/pull/1467)
+- Remove the ability to manually upgrade databases from the context menu on databases. Databases are non-destructively upgraded automatically so for most users this was not needed. For advanced users this is still available in the Command Palette. [#1501](https://github.com/github/vscode-codeql/pull/1501)
+- Always restart the query server after a manual database upgrade. This avoids a bug in the query server where an invalid dbscheme was being retained in memory after an upgrade. [#1519](https://github.com/github/vscode-codeql/pull/1519)
+
+## 1.6.12 - 1 September 2022
+
+- Add ability for users to download databases directly from GitHub. [#1485](https://github.com/github/vscode-codeql/pull/1485)
+- Fix a race condition that could cause a failure to open the evaluator log when running a query. [#1490](https://github.com/github/vscode-codeql/pull/1490)
+- Fix an error when running a query with an older version of the CodeQL CLI. [#1490](https://github.com/github/vscode-codeql/pull/1490)
+
+## 1.6.11 - 25 August 2022
+
+No user facing changes.
+
+## 1.6.10 - 9 August 2022
+
+No user facing changes.
+
+## 1.6.9 - 20 July 2022
+
+No user facing changes.
+
+## 1.6.8 - 29 June 2022
+
+- Fix a bug where quick queries cannot be compiled if the core libraries are not in the workspace. [#1411](https://github.com/github/vscode-codeql/pull/1411)
+- Fix a bug where quick evaluation of library files would display an error message when using CodeQL CLI v2.10.0. [#1412](https://github.com/github/vscode-codeql/pull/1412)
+
+## 1.6.7 - 15 June 2022
+
+- Prints end-of-query evaluator log summaries to the Query Log. [#1349](https://github.com/github/vscode-codeql/pull/1349)
+- Be consistent about casing in Query History menu. [#1369](https://github.com/github/vscode-codeql/pull/1369)
+- Fix quoting string columns in exported CSV results. [#1379](https://github.com/github/vscode-codeql/pull/1379)
+
+## 1.6.6 - 17 May 2022
+
+No user facing changes.
+
+## 1.6.5 - 25 April 2022
+
+- Re-enable publishing to open-vsx. [#1285](https://github.com/github/vscode-codeql/pull/1285)
+
+## 1.6.4 - 6 April 2022
+
+No user facing changes.
+
+## 1.6.3 - 4 April 2022
 
 - Fix a bug where the AST viewer was not synchronizing its selected node when the editor selection changes. [#1230](https://github.com/github/vscode-codeql/pull/1230)
 - Avoid synchronizing the `codeQL.cli.executablePath` setting. [#1252](https://github.com/github/vscode-codeql/pull/1252)

extensions/ql-vscode/README.md

@@ -22,7 +22,7 @@ For information about other configurations, see the separate [CodeQL help](https
 
 ### Quick start: Using CodeQL
 
-1. [Import a database from LGTM](#importing-a-database-from-lgtm).
+1. [Import a database from GitHub](#importing-a-database-from-github).
 1. [Run a query](#running-a-query).
 
 ---
@@ -73,18 +73,19 @@ If you're using your own clone of the CodeQL standard libraries, you can do a `g
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.
 
-### Importing a database from LGTM
+### Importing a database from GitHub
 
-While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from LGTM.com.
+While you can use the [CodeQL CLI to create your own databases](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/), the simplest way to start is by downloading a database from GitHub.com.
 
-1. Open [LGTM.com](https://lgtm.com/#explore) in your browser.
-1. Search for a project you're interested in, for example [Apache Kafka](https://lgtm.com/projects/g/apache/kafka).
-1. Copy the link to that project, for example `https://lgtm.com/projects/g/apache/kafka`.
-1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from LGTM** command.
+1. Find a project that you're interested in on GitHub.com, for example [Apache Kafka](https://github.com/apache/kafka).
+1. Copy the link to that project, for example `https://github.com/apache/kafka`.
+1. In VS Code, open the Command Palette and choose the **CodeQL: Download Database from GitHub** command.
 1. Paste the link you copied earlier.
 1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
 1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
+For more information, see [Choosing a database](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#choosing-a-database) on codeql.github.com.
+
 ### Running a query
 
 The instructions below assume that you're using the CodeQL starter workspace, or that you've added the CodeQL libraries and queries repository to your workspace.

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -1,5 +1,4 @@
 import * as fs from 'fs-extra';
-import * as jsonc from 'jsonc-parser';
 import * as path from 'path';
 
 export interface DeployedPackage {
@@ -28,7 +27,7 @@ async function copyPackage(sourcePath: string, destPath: string): Promise<void>
 
 export async function deployPackage(packageJsonPath: string): Promise<DeployedPackage> {
   try {
-    const packageJson: any = jsonc.parse(await fs.readFile(packageJsonPath, 'utf8'));
+    const packageJson: any = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
 
     // Default to development build; use flag --release to indicate release build.
     const isDevBuild = !process.argv.includes('--release');

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -1,5 +1,5 @@
 import * as gulp from 'gulp';
-import { compileTypeScript, watchTypeScript, copyViewCss, cleanOutput, watchCss } from './typescript';
+import { compileTypeScript, watchTypeScript, cleanOutput } from './typescript';
 import { compileTextMateGrammar } from './textmate';
 import { copyTestData } from './tests';
 import { compileView, watchView } from './webpack';
@@ -10,7 +10,7 @@ export const buildWithoutPackage =
   gulp.series(
     cleanOutput,
     gulp.parallel(
-      compileTypeScript, compileTextMateGrammar, compileView, copyTestData, copyViewCss
+      compileTypeScript, compileTextMateGrammar, compileView, copyTestData
     )
   );
 
@@ -23,6 +23,5 @@ export {
   copyTestData,
   injectAppInsightsKey,
   compileView,
-  watchCss
 };
 export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -219,14 +219,14 @@ function transformFile(yaml: any) {
 }
 
 export function transpileTextMateGrammar() {
-  return through.obj((file: Vinyl, _encoding: string, callback: Function): void => {
+  return through.obj((file: Vinyl, _encoding: string, callback: (err: string | null, file: Vinyl | PluginError) => void): void => {
     if (file.isNull()) {
       callback(null, file);
     }
     else if (file.isBuffer()) {
       const buf: Buffer = file.contents;
       const yamlText: string = buf.toString('utf8');
-      const jsonData: any = jsYaml.safeLoad(yamlText);
+      const jsonData: any = jsYaml.load(yamlText);
       transformFile(jsonData);
 
       file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), 'utf8');

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -39,13 +39,3 @@ export function compileTypeScript() {
 export function watchTypeScript() {
   gulp.watch('src/**/*.ts', compileTypeScript);
 }
-
-export function watchCss() {
-  gulp.watch('src/**/*.css', copyViewCss);
-}
-
-/** Copy CSS files for the results view into the output directory. */
-export function copyViewCss() {
-  return gulp.src('src/**/view/*.css')
-    .pipe(gulp.dest('out'));
-}

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -1,12 +1,11 @@
 import * as path from 'path';
 import * as webpack from 'webpack';
+import * as MiniCssExtractPlugin from 'mini-css-extract-plugin';
 
 export const config: webpack.Configuration = {
   mode: 'development',
   entry: {
-    resultsView: './src/view/results.tsx',
-    compareView: './src/compare/view/Compare.tsx',
-    remoteQueriesView: './src/remote-queries/view/RemoteQueries.tsx',
+    webview: './src/view/webview.tsx'
   },
   output: {
     path: path.resolve(__dirname, '..', 'out'),
@@ -31,9 +30,7 @@ export const config: webpack.Configuration = {
       {
         test: /\.less$/,
         use: [
-          {
-            loader: 'style-loader'
-          },
+          MiniCssExtractPlugin.loader,
           {
             loader: 'css-loader',
             options: {
@@ -53,17 +50,31 @@ export const config: webpack.Configuration = {
       {
         test: /\.css$/,
         use: [
-          {
-            loader: 'style-loader'
-          },
+          MiniCssExtractPlugin.loader,
           {
             loader: 'css-loader'
           }
         ]
+      },
+      {
+        test: /\.(woff(2)?|ttf|eot)$/,
+        use: [
+          {
+            loader: 'file-loader',
+            options: {
+              name: '[name].[ext]',
+              outputPath: 'fonts/',
+              // We need this to make Webpack use the correct path for the fonts.
+              // Without this, the CSS file will use `url([object Module])`
+              esModule: false
+            }
+          },
+        ],
       }
     ]
   },
   performance: {
     hints: false
-  }
+  },
+  plugins: [new MiniCssExtractPlugin()],
 };

extensions/ql-vscode/jest.config.js

@@ -0,0 +1,214 @@
+/*
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+module.exports = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/6m/1394pht172qgd7dmw1fwjk100000gn/T/jest_dx",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  // clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  // coverageDirectory: undefined,
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: 'v8',
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  moduleFileExtensions: [
+    'js',
+    'mjs',
+    'cjs',
+    'jsx',
+    'ts',
+    'tsx',
+    'json'
+  ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  'moduleNameMapper': {
+    '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': '<rootDir>/test/__mocks__/fileMock.ts',
+    '\\.(css|less)$': '<rootDir>/test/__mocks__/styleMock.ts'
+  },
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  preset: 'ts-jest',
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  setupFilesAfterEnv: ['<rootDir>/test/jest.setup.ts'],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  testEnvironment: 'jsdom',
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  testMatch: [
+    '**/__tests__/**/*.[jt]s?(x)'
+  ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        tsconfig: 'src/view/tsconfig.spec.json',
+      },
+    ],
+    'node_modules': [
+      'babel-jest',
+      {
+        presets: [
+          '@babel/preset-env'
+        ],
+        plugins: [
+          '@babel/plugin-transform-modules-commonjs',
+        ]
+      }
+    ]
+  },
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  'transformIgnorePatterns': [
+    // These use ES modules, so need to be transformed
+    'node_modules/(?!(?:@vscode/webview-ui-toolkit|@microsoft/.+|exenv-es6)/.*)'
+  ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.6.2",
+  "version": "1.7.1",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -14,15 +14,14 @@
   },
   "engines": {
     "vscode": "^1.59.0",
-    "node": ">=14.17.1",
+    "node": "^16.13.0",
     "npm": ">=7.20.6"
   },
   "categories": [
     "Programming Languages"
   ],
   "extensionDependencies": [
-    "hbenl.vscode-test-explorer",
-    "ms-vscode.test-adapter-converter"
+    "hbenl.vscode-test-explorer"
   ],
   "capabilities": {
     "untrustedWorkspaces": {
@@ -36,9 +35,11 @@
   },
   "activationEvents": [
     "onLanguage:ql",
+    "onLanguage:ql-summary",
     "onView:codeQLDatabases",
     "onView:codeQLQueryHistory",
     "onView:codeQLAstViewer",
+    "onView:codeQLEvalLogViewer",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
     "onCommand:codeQL.authenticateToGitHub",
@@ -62,6 +63,7 @@
     "onCommand:codeQL.quickQuery",
     "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
+    "onWebviewPanel:codeQL.variantAnalysis",
     "onFileSystem:codeql-zip-archive"
   ],
   "main": "./out/extension",
@@ -111,6 +113,12 @@
         "extensions": [
           ".qhelp"
         ]
+      },
+      {
+        "id": "ql-summary",
+        "filenames": [
+          "evaluator-log.summary"
+        ]
       }
     ],
     "grammars": [
@@ -225,7 +233,7 @@
         },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "%q on %d - %s, %r result count [%t]",
+          "default": "%q on %d - %s %r [%t]",
           "markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
         },
         "codeQL.queryHistory.ttl": {
@@ -281,7 +289,7 @@
           "default": "",
           "pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
           "patternErrorMessage": "Please enter a valid GitHub repository",
-          "markdownDescription": "[For internal use only] The name of the GitHub repository where you can view the progress and results of the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
+          "markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
         }
       }
     },
@@ -302,6 +310,14 @@
         "command": "codeQL.runVariantAnalysis",
         "title": "CodeQL: Run Variant Analysis"
       },
+      {
+        "command": "codeQL.exportVariantAnalysisResults",
+        "title": "CodeQL: Export Variant Analysis Results"
+      },
+      {
+        "command": "codeQL.openVariantAnalysis",
+        "title": "CodeQL: Open Variant Analysis"
+      },
       {
         "command": "codeQL.runQueries",
         "title": "CodeQL: Run Queries in Selected Files"
@@ -468,7 +484,7 @@
       },
       {
         "command": "codeQLQueryHistory.openQuery",
-        "title": "Open the query that produced these results",
+        "title": "Open the Query that Produced these Results",
         "icon": {
           "light": "media/light/edit.svg",
           "dark": "media/dark/edit.svg"
@@ -520,15 +536,19 @@
       },
       {
         "command": "codeQLQueryHistory.openQueryDirectory",
-        "title": "Open query directory"
+        "title": "Open Query Directory"
       },
       {
         "command": "codeQLQueryHistory.showEvalLog",
-        "title": "Show Evaluator Log (Raw)"
+        "title": "Show Evaluator Log (Raw JSON)"
       },
       {
         "command": "codeQLQueryHistory.showEvalLogSummary",
-        "title": "Show Evaluator Log (Summary)"
+        "title": "Show Evaluator Log (Summary Text)"
+      },
+      {
+        "command": "codeQLQueryHistory.showEvalLogViewer",
+        "title": "Show Evaluator Log (UI)"
       },
       {
         "command": "codeQLQueryHistory.cancel",
@@ -538,6 +558,10 @@
         "command": "codeQLQueryHistory.showQueryText",
         "title": "Show Query Text"
       },
+      {
+        "command": "codeQLQueryHistory.exportResults",
+        "title": "Export Results"
+      },
       {
         "command": "codeQLQueryHistory.viewCsvResults",
         "title": "View Results (CSV)"
@@ -566,6 +590,10 @@
         "command": "codeQLQueryHistory.openOnGithub",
         "title": "Open Variant Analysis on GitHub"
       },
+      {
+        "command": "codeQLQueryHistory.copyRepoList",
+        "title": "Copy Repository List"
+      },
       {
         "command": "codeQLQueryResults.nextPathStep",
         "title": "CodeQL: Show Next Step on Path"
@@ -597,6 +625,19 @@
           "light": "media/light/clear-all.svg",
           "dark": "media/dark/clear-all.svg"
         }
+      },
+      {
+        "command": "codeQLEvalLogViewer.clear",
+        "title": "Clear Viewer",
+        "icon": {
+          "light": "media/light/clear-all.svg",
+          "dark": "media/dark/clear-all.svg"
+        }
+      },
+      {
+        "command": "codeQL.gotoQL",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
       }
     ],
     "menus": {
@@ -628,12 +669,12 @@
         },
         {
           "command": "codeQLDatabases.chooseDatabaseGithub",
-          "when": "config.codeQL.canary && view == codeQLDatabases",
+          "when": "view == codeQLDatabases",
           "group": "navigation"
         },
         {
           "command": "codeQLDatabases.chooseDatabaseLgtm",
-          "when": "view == codeQLDatabases",
+          "when": "config.codeQL.canary && view == codeQLDatabases",
           "group": "navigation"
         },
         {
@@ -670,6 +711,11 @@
           "command": "codeQLAstViewer.clear",
           "when": "view == codeQLAstViewer",
           "group": "navigation"
+        },
+        {
+          "command": "codeQLEvalLogViewer.clear",
+          "when": "view == codeQLEvalLogViewer",
+          "group": "navigation"
         }
       ],
       "view/item/context": [
@@ -683,11 +729,6 @@
           "group": "9_qlCommands",
           "when": "view == codeQLDatabases"
         },
-        {
-          "command": "codeQLDatabases.upgradeDatabase",
-          "group": "9_qlCommands",
-          "when": "view == codeQLDatabases"
-        },
         {
           "command": "codeQLDatabases.renameDatabase",
           "group": "9_qlCommands",
@@ -711,7 +752,7 @@
         {
           "command": "codeQLQueryHistory.removeHistoryItem",
           "group": "9_qlCommands",
-          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem"
+          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
         },
         {
           "command": "codeQLQueryHistory.setLabel",
@@ -736,18 +777,28 @@
         {
           "command": "codeQLQueryHistory.showEvalLog",
           "group": "9_qlCommands",
-          "when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)"
+          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showEvalLogSummary",
           "group": "9_qlCommands",
-          "when": "codeql.supportsEvalLog && (viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem)"
+          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.showEvalLogViewer",
+          "group": "9_qlCommands",
+          "when": "config.codeQL.canary && codeql.supportsEvalLog && viewItem == rawResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == interpretedResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showQueryText",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
         },
+        {
+          "command": "codeQLQueryHistory.exportResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == remoteResultsItem"
+        },
         {
           "command": "codeQLQueryHistory.viewCsvResults",
           "group": "9_qlCommands",
@@ -776,7 +827,12 @@
         {
           "command": "codeQLQueryHistory.openOnGithub",
           "group": "9_qlCommands",
-          "when": "viewItem == remoteResultsItem || viewItem == inProgressRemoteResultsItem"
+          "when": "viewItem == remoteResultsItem || viewItem == inProgressRemoteResultsItem || viewItem == cancelledRemoteResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.copyRepoList",
+          "group": "9_qlCommands",
+          "when": "viewItem == remoteResultsItem"
         },
         {
           "command": "codeQLTests.showOutputDifferences",
@@ -838,6 +894,14 @@
           "command": "codeQL.runVariantAnalysis",
           "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.openVariantAnalysis",
+          "when": "config.codeQL.canary && config.codeQL.variantAnalysis.liveResults"
+        },
+        {
+          "command": "codeQL.exportVariantAnalysisResults",
+          "when": "config.codeQL.canary"
+        },
         {
           "command": "codeQL.runQueries",
           "when": "false"
@@ -867,7 +931,7 @@
           "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
         },
         {
-          "command": "codeQL.chooseDatabaseGithub",
+          "command": "codeQL.chooseDatabaseLgtm",
           "when": "config.codeQL.canary"
         },
         {
@@ -950,6 +1014,10 @@
           "command": "codeQLQueryHistory.showEvalLogSummary",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.showEvalLogViewer",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.openQueryDirectory",
           "when": "false"
@@ -962,10 +1030,18 @@
           "command": "codeQLQueryHistory.openOnGithub",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.copyRepoList",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.showQueryText",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.exportResults",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.viewCsvResults",
           "when": "false"
@@ -1010,6 +1086,10 @@
           "command": "codeQLAstViewer.clear",
           "when": "false"
         },
+        {
+          "command": "codeQLEvalLogViewer.clear",
+          "when": "false"
+        },
         {
           "command": "codeQLTests.acceptOutput",
           "when": "false"
@@ -1051,6 +1131,10 @@
         {
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
+        },
+        {
+          "command": "codeQL.gotoQL",
+          "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
     },
@@ -1076,6 +1160,11 @@
         {
           "id": "codeQLAstViewer",
           "name": "AST Viewer"
+        },
+        {
+          "id": "codeQLEvalLogViewer",
+          "name": "Evaluator Log Viewer",
+          "when": "config.codeQL.canary"
         }
       ]
     },
@@ -1090,7 +1179,11 @@
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)"
+      },
+      {
+        "view": "codeQLEvalLogViewer",
+        "contents": "Run the 'Show Evaluator Log (UI)' command on a CodeQL query run in the Query History view."
       }
     ]
   },
@@ -1099,27 +1192,34 @@
     "watch": "npm-run-all -p watch:*",
     "watch:extension": "tsc --watch",
     "watch:webpack": "gulp watchView",
-    "watch:css": "gulp watchCss",
-    "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
+    "test": "npm-run-all -p test:*",
+    "test:unit": "mocha --exit -r ts-node/register -r test/mocha.setup.js test/pure-tests/**/*.ts",
+    "test:view": "jest",
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
     "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
     "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
     "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
-    "format-staged": "lint-staged"
+    "format-staged": "lint-staged",
+    "storybook": "start-storybook -p 6006",
+    "build-storybook": "build-storybook"
   },
   "dependencies": {
-    "@octokit/rest": "^18.5.6",
-    "@primer/octicons-react": "^16.3.0",
+    "@octokit/plugin-retry": "^3.0.9",
+    "@octokit/rest": "^19.0.4",
+    "@primer/octicons-react": "^17.6.0",
     "@primer/react": "^35.0.0",
+    "@vscode/codicons": "^0.0.31",
+    "@vscode/webview-ui-toolkit": "^1.0.1",
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
-    "d3": "^6.3.1",
+    "d3": "^7.6.1",
     "d3-graphviz": "^2.6.1",
     "fs-extra": "^10.0.1",
-    "glob-promise": "^3.4.0",
-    "js-yaml": "^3.14.0",
+    "glob-promise": "^4.2.2",
+    "immutable": "^4.0.0",
+    "js-yaml": "^4.1.0",
     "minimist": "~1.2.6",
     "nanoid": "^3.2.0",
     "node-fetch": "~2.6.7",
@@ -1127,6 +1227,8 @@
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "semver": "~7.3.2",
+    "source-map": "^0.7.4",
+    "source-map-support": "^0.5.21",
     "stream": "^0.0.2",
     "stream-chain": "~2.2.4",
     "stream-json": "~1.7.3",
@@ -1140,14 +1242,28 @@
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",
     "vscode-test-adapter-util": "~0.7.0",
-    "zip-a-folder": "~0.0.12"
+    "zip-a-folder": "~1.1.3"
   },
   "devDependencies": {
+    "@babel/core": "^7.18.13",
+    "@babel/plugin-transform-modules-commonjs": "^7.18.6",
+    "@faker-js/faker": "^7.5.0",
+    "@storybook/addon-actions": "^6.5.10",
+    "@storybook/addon-essentials": "^6.5.10",
+    "@storybook/addon-interactions": "^6.5.10",
+    "@storybook/addon-links": "^6.5.10",
+    "@storybook/builder-webpack5": "^6.5.10",
+    "@storybook/manager-webpack5": "^6.5.10",
+    "@storybook/react": "^6.5.10",
+    "@storybook/testing-library": "^0.0.13",
+    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/react": "^12.1.5",
+    "@testing-library/user-event": "^14.4.3",
     "@types/chai": "^4.1.7",
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/d3": "^6.2.0",
+    "@types/d3": "^7.4.0",
     "@types/d3-graphviz": "^2.6.6",
     "@types/del": "^4.0.0",
     "@types/fs-extra": "^9.0.6",
@@ -1156,6 +1272,7 @@
     "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "^1.1.0",
     "@types/gulp-sourcemaps": "0.0.32",
+    "@types/jest": "^29.0.2",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
     "@types/mocha": "^9.0.0",
@@ -1176,35 +1293,42 @@
     "@types/unzipper": "~0.10.1",
     "@types/vscode": "^1.59.0",
     "@types/webpack": "^5.28.0",
+    "@types/webpack-env": "^1.18.0",
     "@types/xml2js": "~0.4.4",
     "@typescript-eslint/eslint-plugin": "^4.26.0",
     "@typescript-eslint/parser": "^4.26.0",
     "ansi-colors": "^4.1.1",
-    "applicationinsights": "^1.8.7",
+    "applicationinsights": "^2.3.5",
+    "babel-loader": "^8.2.5",
     "chai": "^4.2.0",
     "chai-as-promised": "~7.1.1",
     "css-loader": "~3.1.0",
     "del": "^6.0.0",
     "eslint": "~6.8.0",
+    "eslint-plugin-jest-dom": "^4.0.2",
     "eslint-plugin-react": "~7.19.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-storybook": "^0.6.4",
+    "file-loader": "^6.2.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",
     "gulp-replace": "^1.1.3",
     "gulp-sourcemaps": "^3.0.0",
     "gulp-typescript": "^5.0.1",
-    "husky": "~4.2.5",
-    "jsonc-parser": "^2.3.0",
+    "husky": "~4.3.8",
+    "jest": "^29.0.3",
+    "jest-environment-jsdom": "^29.0.3",
     "lint-staged": "~10.2.2",
-    "mocha": "^9.1.3",
+    "mini-css-extract-plugin": "^2.6.1",
+    "mocha": "^10.0.0",
     "mocha-sinon": "~2.1.2",
     "npm-run-all": "^4.1.5",
     "prettier": "~2.0.5",
     "proxyquire": "~2.1.3",
-    "sinon": "~13.0.1",
+    "sinon": "~14.0.0",
     "sinon-chai": "~3.5.0",
-    "source-map-support": "^0.5.21",
-    "style-loader": "~0.23.1",
     "through2": "^4.0.2",
+    "ts-jest": "^29.0.1",
     "ts-loader": "^8.1.0",
     "ts-node": "^10.7.0",
     "ts-protoc-gen": "^0.9.0",
@@ -1212,7 +1336,7 @@
     "typescript-formatter": "^7.2.2",
     "vsce": "^2.7.0",
     "vscode-test": "^1.4.0",
-    "webpack": "^5.28.0",
+    "webpack": "^5.62.2",
     "webpack-cli": "^4.6.0"
   },
   "husky": {
@@ -1222,7 +1346,7 @@
     }
   },
   "lint-staged": {
-    "./**/*.{json,css,scss,md}": [
+    "./**/*.{json,css,scss}": [
       "prettier --write"
     ],
     "./**/*.{ts,tsx}": [
@@ -1231,6 +1355,6 @@
     ]
   },
   "resolutions": {
-    "glob-parent": "~6.0.0"
+    "glob-parent": "6.0.0"
   }
 }

extensions/ql-vscode/src/abstract-webview.ts

@@ -0,0 +1,129 @@
+import {
+  WebviewPanel,
+  ExtensionContext,
+  window as Window,
+  ViewColumn,
+  Uri,
+  WebviewPanelOptions,
+  WebviewOptions
+} from 'vscode';
+import * as path from 'path';
+
+import { DisposableObject } from './pure/disposable-object';
+import { tmpDir } from './helpers';
+import { getHtmlForWebview, WebviewMessage, WebviewView } from './interface-utils';
+
+export type WebviewPanelConfig = {
+  viewId: string;
+  title: string;
+  viewColumn: ViewColumn;
+  view: WebviewView;
+  preserveFocus?: boolean;
+  additionalOptions?: WebviewPanelOptions & WebviewOptions;
+}
+
+export abstract class AbstractWebview<ToMessage extends WebviewMessage, FromMessage extends WebviewMessage> extends DisposableObject {
+  protected panel: WebviewPanel | undefined;
+  protected panelLoaded = false;
+  protected panelLoadedCallBacks: (() => void)[] = [];
+
+  constructor(
+    protected readonly ctx: ExtensionContext
+  ) {
+    super();
+  }
+
+  public async restoreView(panel: WebviewPanel): Promise<void> {
+    this.panel = panel;
+    this.setupPanel(panel);
+  }
+
+  protected get isShowingPanel() {
+    return !!this.panel;
+  }
+
+  protected getPanel(): WebviewPanel {
+    if (this.panel == undefined) {
+      const { ctx } = this;
+
+      const config = this.getPanelConfig();
+
+      this.panel = Window.createWebviewPanel(
+        config.viewId,
+        config.title,
+        { viewColumn: config.viewColumn, preserveFocus: config.preserveFocus },
+        {
+          enableScripts: true,
+          enableFindWidget: true,
+          retainContextWhenHidden: true,
+          ...config.additionalOptions,
+          localResourceRoots: [
+            ...(config.additionalOptions?.localResourceRoots ?? []),
+            Uri.file(tmpDir.name),
+            Uri.file(path.join(ctx.extensionPath, 'out'))
+          ],
+        }
+      );
+      this.setupPanel(this.panel);
+    }
+    return this.panel;
+  }
+
+  protected setupPanel(panel: WebviewPanel): void {
+    const config = this.getPanelConfig();
+
+    this.push(
+      panel.onDidDispose(
+        () => {
+          this.panel = undefined;
+          this.panelLoaded = false;
+          this.onPanelDispose();
+        },
+        null,
+        this.ctx.subscriptions
+      )
+    );
+
+    panel.webview.html = getHtmlForWebview(
+      this.ctx,
+      panel.webview,
+      config.view,
+      {
+        allowInlineStyles: true,
+      }
+    );
+    this.push(
+      panel.webview.onDidReceiveMessage(
+        async (e) => this.onMessage(e),
+        undefined,
+        this.ctx.subscriptions
+      )
+    );
+  }
+
+  protected abstract getPanelConfig(): WebviewPanelConfig;
+
+  protected abstract onPanelDispose(): void;
+
+  protected abstract onMessage(msg: FromMessage): Promise<void>;
+
+  protected waitForPanelLoaded(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.panelLoaded) {
+        resolve();
+      } else {
+        this.panelLoadedCallBacks.push(resolve);
+      }
+    });
+  }
+
+  protected onWebViewLoaded(): void {
+    this.panelLoaded = true;
+    this.panelLoadedCallBacks.forEach((cb) => cb());
+    this.panelLoadedCallBacks = [];
+  }
+
+  protected postMessage(msg: ToMessage): Thenable<boolean> {
+    return this.getPanel().webview.postMessage(msg);
+  }
+}

extensions/ql-vscode/src/archive-filesystem-provider.ts

@@ -167,21 +167,26 @@ type Archive = {
   dirMap: DirectoryHierarchyMap;
 };
 
+async function parse_zip(zipPath: string): Promise<Archive> {
+  if (!await fs.pathExists(zipPath))
+    throw vscode.FileSystemError.FileNotFound(zipPath);
+  const archive: Archive = { unzipped: await unzipper.Open.file(zipPath), dirMap: new Map };
+  archive.unzipped.files.forEach(f => { ensureFile(archive.dirMap, path.resolve('/', f.path)); });
+  return archive;
+}
+
 export class ArchiveFileSystemProvider implements vscode.FileSystemProvider {
   private readOnlyError = vscode.FileSystemError.NoPermissions('write operation attempted, but source archive filesystem is readonly');
-  private archives: Map<string, Archive> = new Map;
+  private archives: Map<string, Promise<Archive>> = new Map;
 
   private async getArchive(zipPath: string): Promise<Archive> {
     if (!this.archives.has(zipPath)) {
-      if (!await fs.pathExists(zipPath))
-        throw vscode.FileSystemError.FileNotFound(zipPath);
-      const archive: Archive = { unzipped: await unzipper.Open.file(zipPath), dirMap: new Map };
-      archive.unzipped.files.forEach(f => { ensureFile(archive.dirMap, path.resolve('/', f.path)); });
-      this.archives.set(zipPath, archive);
+      this.archives.set(zipPath, parse_zip(zipPath));
     }
-    return this.archives.get(zipPath)!;
+    return await this.archives.get(zipPath)!;
   }
 
+
   root = new Directory('');
 
   // metadata

extensions/ql-vscode/src/authentication.ts

@@ -1,13 +1,15 @@
 import * as vscode from 'vscode';
 import * as Octokit from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
 
 const GITHUB_AUTH_PROVIDER_ID = 'github';
 
-// 'repo' scope should be enough for triggering workflows. For a comprehensive list, see:
+// We need 'repo' scope for triggering workflows and 'gist' scope for exporting results to Gist.
+// For a comprehensive list of scopes, see:
 // https://docs.github.com/apps/building-oauth-apps/understanding-scopes-for-oauth-apps
-const SCOPES = ['repo'];
+const SCOPES = ['repo', 'gist'];
 
-/** 
+/**
  * Handles authentication to GitHub, using the VS Code [authentication API](https://code.visualstudio.com/api/references/vscode-api#authentication).
  */
 export class Credentials {
@@ -18,6 +20,15 @@ export class Credentials {
   // eslint-disable-next-line @typescript-eslint/no-empty-function
   private constructor() { }
 
+  /**
+   * Initializes an instance of credentials with an octokit instance.
+   *
+   * Do not call this method until you know you actually need an instance of credentials.
+   * since calling this method will require the user to log in.
+   *
+   * @param context The extension context.
+   * @returns An instance of credentials.
+   */
   static async initialize(context: vscode.ExtensionContext): Promise<Credentials> {
     const c = new Credentials();
     c.registerListeners(context);
@@ -25,12 +36,31 @@ export class Credentials {
     return c;
   }
 
-  private async createOctokit(createIfNone: boolean): Promise<Octokit.Octokit | undefined> {
+  /**
+   * Initializes an instance of credentials with an octokit instance using
+   * a token from the user's GitHub account. This method is meant to be
+   * used non-interactive environments such as tests.
+   *
+   * @param overrideToken The GitHub token to use for authentication.
+   * @returns An instance of credentials.
+   */
+  static async initializeWithToken(overrideToken: string) {
+    const c = new Credentials();
+    c.octokit = await c.createOctokit(false, overrideToken);
+    return c;
+  }
+
+  private async createOctokit(createIfNone: boolean, overrideToken?: string): Promise<Octokit.Octokit | undefined> {
+    if (overrideToken) {
+      return new Octokit.Octokit({ auth: overrideToken, retry });
+    }
+
     const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
 
     if (session) {
       return new Octokit.Octokit({
-        auth: session.accessToken
+        auth: session.accessToken,
+        retry
       });
     } else {
       return undefined;
@@ -46,16 +76,27 @@ export class Credentials {
     }));
   }
 
-  async getOctokit(): Promise<Octokit.Octokit> {
+  /**
+   * Creates or returns an instance of Octokit.
+   *
+   * @param requireAuthentication Whether the Octokit instance needs to be authenticated as user.
+   * @returns An instance of Octokit.
+   */
+  async getOctokit(requireAuthentication = true): Promise<Octokit.Octokit> {
     if (this.octokit) {
       return this.octokit;
     }
 
-    this.octokit = await this.createOctokit(true);
-    // octokit shouldn't be undefined, since we've set "createIfNone: true".
-    // The following block is mainly here to prevent a compiler error.
+    this.octokit = await this.createOctokit(requireAuthentication);
+
     if (!this.octokit) {
-      throw new Error('Did not initialize Octokit.');
+      if (requireAuthentication) {
+        throw new Error('Did not initialize Octokit.');
+      }
+
+      // We don't want to set this in this.octokit because that would prevent
+      // authenticating when requireCredentials is true.
+      return new Octokit.Octokit({ retry });
     }
     return this.octokit;
   }

extensions/ql-vscode/src/cli.ts

@@ -11,12 +11,12 @@ import { promisify } from 'util';
 import { CancellationToken, commands, Disposable, Uri } from 'vscode';
 
 import { BQRSInfo, DecodedBqrsChunk } from './pure/bqrs-cli-types';
-import { CliConfig } from './config';
+import { allowCanaryQueryServer, CliConfig } from './config';
 import { DistributionProvider, FindDistributionResultKind } from './distribution';
 import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
-import { CompilationMessage } from './pure/messages';
+import { CompilationMessage } from './pure/legacy-messages';
 import { sarifParser } from './sarif-parser';
 import { dbSchemeToLanguage, walkDirectory } from './helpers';
 
@@ -168,7 +168,7 @@ export class CodeQLCliServer implements Disposable {
   nullBuffer: Buffer;
 
   /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: SemVer | undefined;
+  private _version: Promise<SemVer> | undefined;
 
   /**
    * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
@@ -240,7 +240,7 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Restart the server when the current command terminates
    */
-  private restartCliServer(): void {
+  restartCliServer(): void {
     const callback = (): void => {
       try {
         this.killProcessIfRunning();
@@ -604,10 +604,14 @@ export class CodeQLCliServer implements Disposable {
   }
 
   /** Resolves the ML models that should be available when evaluating a query. */
-  async resolveMlModels(additionalPacks: string[]): Promise<MlModelsInfo> {
+  async resolveMlModels(additionalPacks: string[], queryPath: string): Promise<MlModelsInfo> {
+    const args = await this.cliConstraints.supportsPreciseResolveMlModels()
+      // use the dirname of the path so that we can handle query libraries
+      ? [...this.getAdditionalPacksArg(additionalPacks), path.dirname(queryPath)]
+      : this.getAdditionalPacksArg(additionalPacks);
     return await this.runJsonCodeQlCliCommand<MlModelsInfo>(
       ['resolve', 'ml-models'],
-      this.getAdditionalPacksArg(additionalPacks),
+      args,
       'Resolving ML models',
       false
     );
@@ -667,11 +671,11 @@ export class CodeQLCliServer implements Disposable {
 
   /**
   * Generate a summary of an evaluation log.
-  * @param endSummaryPath The path to write only the end of query part of the human-readable summary to. 
+  * @param endSummaryPath The path to write only the end of query part of the human-readable summary to.
   * @param inputPath The path of an evaluation event log.
   * @param outputPath The path to write a human-readable summary of it to.
   */
-   async generateLogSummary(
+  async generateLogSummary(
     inputPath: string,
     outputPath: string,
     endSummaryPath: string,
@@ -679,12 +683,30 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       '--format=text',
       `--end-summary=${endSummaryPath}`,
+      ...(await this.cliConstraints.supportsSourceMap() ? ['--sourcemap'] : []),
       inputPath,
       outputPath
     ];
     return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
   }
 
+  /**
+  * Generate a JSON summary of an evaluation log.
+  * @param inputPath The path of an evaluation event log.
+  * @param outputPath The path to write a JSON summary of it to.
+  */
+  async generateJsonLogSummary(
+    inputPath: string,
+    outputPath: string,
+  ): Promise<string> {
+    const subcommandArgs = [
+      '--format=predicates',
+      inputPath,
+      outputPath
+    ];
+    return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating JSON log summary');
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
@@ -914,8 +936,12 @@ export class CodeQLCliServer implements Disposable {
     return this.runJsonCodeQlCliCommand(['pack', 'download'], packs, 'Downloading packs');
   }
 
-  async packInstall(dir: string) {
-    return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
+  async packInstall(dir: string, forceUpdate = false) {
+    const args = [dir];
+    if (forceUpdate) {
+      args.push('--mode', 'update');
+    }
+    return this.runJsonCodeQlCliCommand(['pack', 'install'], args, 'Installing pack dependencies');
   }
 
   async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
@@ -959,13 +985,13 @@ export class CodeQLCliServer implements Disposable {
 
   public async getVersion() {
     if (!this._version) {
-      this._version = await this.refreshVersion();
+      this._version = this.refreshVersion();
       // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
       await commands.executeCommand(
         'setContext', 'codeql.supportsEvalLog', await this.cliConstraints.supportsPerQueryEvalLog()
       );
     }
-    return this._version;
+    return await this._version;
   }
 
   private async refreshVersion() {
@@ -1222,6 +1248,9 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
 
+
+  public static CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES = new SemVer('2.4.2');
+
   /**
    * CLI version where `codeql resolve upgrades` supports
    * the `--allow-downgrades` flag
@@ -1235,7 +1264,7 @@ export class CliVersionConstraint {
 
   /**
    * CLI version where database registration was introduced
-  */
+   */
   public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
 
   /**
@@ -1255,7 +1284,7 @@ export class CliVersionConstraint {
   public static CLI_VERSION_WITH_NO_PRECOMPILE = new SemVer('2.7.1');
 
   /**
-   * CLI version where remote queries are supported.
+   * CLI version where remote queries (variant analysis) are supported.
    */
   public static CLI_VERSION_REMOTE_QUERIES = new SemVer('2.6.3');
 
@@ -1264,6 +1293,11 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_RESOLVE_ML_MODELS = new SemVer('2.7.3');
 
+  /**
+   * CLI version where the `resolve ml-models` subcommand was enhanced to work with packaging.
+   */
+  public static CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS = new SemVer('2.10.0');
+
   /**
    * CLI version where the `--old-eval-stats` option to the query server was introduced.
    */
@@ -1280,16 +1314,26 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer('2.8.2');
 
-   /**
-    * CLI version that supports rotating structured logs to produce one per query.
-    * 
-    * Note that 2.8.4 supports generating the evaluation logs and summaries,
-    * but 2.9.0 includes a new option to produce the end-of-query summary logs to
-    * the query server console. For simplicity we gate all features behind 2.9.0,
-    * but if a user is tied to the 2.8 release, we can enable evaluator logs 
-    * and summaries for them.
-    */
-    public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.9.0');
+  /**
+   * CLI version that supports rotating structured logs to produce one per query.
+   *
+   * Note that 2.8.4 supports generating the evaluation logs and summaries,
+   * but 2.9.0 includes a new option to produce the end-of-query summary logs to
+   * the query server console. For simplicity we gate all features behind 2.9.0,
+   * but if a user is tied to the 2.8 release, we can enable evaluator logs
+   * and summaries for them.
+   */
+  public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer('2.9.0');
+
+  /**
+   * CLI version that supports the `--sourcemap` option for log generation.
+   */
+  public static CLI_VERSION_WITH_SOURCEMAP = new SemVer('2.10.3');
+
+  /**
+   * CLI version that supports the new query server.
+   */
+  public static CLI_VERSION_WITH_NEW_QUERY_SERVER = new SemVer('2.11.0');
 
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
@@ -1307,6 +1351,10 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_LANGUAGE);
   }
 
+  public async supportsNonDestructiveUpgrades() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NONDESTURCTIVE_UPGRADES);
+  }
+
   public async supportsDowngrades() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DOWNGRADES);
   }
@@ -1339,6 +1387,10 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_ML_MODELS);
   }
 
+  async supportsPreciseResolveMlModels() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS);
+  }
+
   async supportsOldEvalStats() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_OLD_EVAL_STATS);
   }
@@ -1354,4 +1406,16 @@ export class CliVersionConstraint {
   async supportsPerQueryEvalLog() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG);
   }
+
+  async supportsSourceMap() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_SOURCEMAP);
+  }
+
+  async supportsNewQueryServer() {
+    // TODO while under development, users _must_ opt-in to the new query server
+    // by setting the `codeql.canaryQueryServer` setting to `true`.
+    // Ignore the version check for now.
+    return allowCanaryQueryServer();
+    // return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER);
+  }
 }

extensions/ql-vscode/src/compare/compare-view.ts

@@ -1,14 +1,8 @@
-import { DisposableObject } from '../pure/disposable-object';
 import {
-  WebviewPanel,
   ExtensionContext,
-  window as Window,
   ViewColumn,
-  Uri,
 } from 'vscode';
-import * as path from 'path';
 
-import { tmpDir } from '../helpers';
 import {
   FromCompareViewMessage,
   ToCompareViewMessage,
@@ -17,33 +11,33 @@ import {
 import { Logger } from '../logging';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
-import { getHtmlForWebview, jumpToLocation } from '../interface-utils';
+import { jumpToLocation } from '../interface-utils';
 import { transformBqrsResultSet, RawResultSet, BQRSInfo } from '../pure/bqrs-cli-types';
 import resultsDiff from './resultsDiff';
 import { CompletedLocalQueryInfo } from '../query-results';
 import { getErrorMessage } from '../pure/helpers-pure';
+import { HistoryItemLabelProvider } from '../history-item-label-provider';
+import { AbstractWebview, WebviewPanelConfig } from '../abstract-webview';
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
   to: CompletedLocalQueryInfo;
 }
 
-export class CompareInterfaceManager extends DisposableObject {
+export class CompareView extends AbstractWebview<ToCompareViewMessage, FromCompareViewMessage> {
   private comparePair: ComparePair | undefined;
-  private panel: WebviewPanel | undefined;
-  private panelLoaded = false;
-  private panelLoadedCallBacks: (() => void)[] = [];
 
   constructor(
-    private ctx: ExtensionContext,
+    ctx: ExtensionContext,
     private databaseManager: DatabaseManager,
     private cliServer: CodeQLCliServer,
     private logger: Logger,
+    private labelProvider: HistoryItemLabelProvider,
     private showQueryResultsCallback: (
       item: CompletedLocalQueryInfo
     ) => Promise<void>
   ) {
-    super();
+    super(ctx);
   }
 
   async showResults(
@@ -81,12 +75,12 @@ export class CompareInterfaceManager extends DisposableObject {
             // since we split the description into several rows
             // only run interpolation if the label is user-defined
             // otherwise we will wind up with duplicated rows
-            name: from.getShortLabel(),
+            name: this.labelProvider.getShortLabel(from),
             status: from.completedQuery.statusString,
             time: from.startTime,
           },
           toQuery: {
-            name: to.getShortLabel(),
+            name: this.labelProvider.getShortLabel(to),
             status: to.completedQuery.statusString,
             time: to.startTime,
           },
@@ -101,73 +95,24 @@ export class CompareInterfaceManager extends DisposableObject {
     }
   }
 
-  getPanel(): WebviewPanel {
-    if (this.panel == undefined) {
-      const { ctx } = this;
-      const panel = (this.panel = Window.createWebviewPanel(
-        'compareView',
-        'Compare CodeQL Query Results',
-        { viewColumn: ViewColumn.Active, preserveFocus: true },
-        {
-          enableScripts: true,
-          enableFindWidget: true,
-          retainContextWhenHidden: true,
-          localResourceRoots: [
-            Uri.file(tmpDir.name),
-            Uri.file(path.join(this.ctx.extensionPath, 'out')),
-          ],
-        }
-      ));
-      this.push(this.panel.onDidDispose(
-        () => {
-          this.panel = undefined;
-          this.comparePair = undefined;
-        },
-        null,
-        ctx.subscriptions
-      ));
-
-      const scriptPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/compareView.js')
-      );
-
-      const stylesheetPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/view/resultsView.css')
-      );
-
-      panel.webview.html = getHtmlForWebview(
-        panel.webview,
-        scriptPathOnDisk,
-        [stylesheetPathOnDisk],
-        false
-      );
-      this.push(panel.webview.onDidReceiveMessage(
-        async (e) => this.handleMsgFromView(e),
-        undefined,
-        ctx.subscriptions
-      ));
-    }
-    return this.panel;
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: 'compareView',
+      title: 'Compare CodeQL Query Results',
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: 'compare',
+    };
   }
 
-  private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve) => {
-      if (this.panelLoaded) {
-        resolve();
-      } else {
-        this.panelLoadedCallBacks.push(resolve);
-      }
-    });
+  protected onPanelDispose(): void {
+    this.comparePair = undefined;
   }
 
-  private async handleMsgFromView(
-    msg: FromCompareViewMessage
-  ): Promise<void> {
+  protected async onMessage(msg: FromCompareViewMessage): Promise<void> {
     switch (msg.t) {
-      case 'compareViewLoaded':
-        this.panelLoaded = true;
-        this.panelLoadedCallBacks.forEach((cb) => cb());
-        this.panelLoadedCallBacks = [];
+      case 'viewLoaded':
+        this.onWebViewLoaded();
         break;
 
       case 'changeCompare':
@@ -184,10 +129,6 @@ export class CompareInterfaceManager extends DisposableObject {
     }
   }
 
-  private postMessage(msg: ToCompareViewMessage): Thenable<boolean> {
-    return this.getPanel().webview.postMessage(msg);
-  }
-
   private async findCommonResultSetNames(
     from: CompletedLocalQueryInfo,
     to: CompletedLocalQueryInfo,

extensions/ql-vscode/src/compare/view/.eslintrc.js

@@ -1,13 +0,0 @@
-module.exports = {
-  env: {
-    browser: true
-  },
-  extends: [
-    "plugin:react/recommended"
-  ],
-  settings: {
-    react: {
-      version: 'detect'
-    }
-  }
-}

extensions/ql-vscode/src/compare/view/tsconfig.json

@@ -1,18 +0,0 @@
-{
-  "compilerOptions": {
-    "module": "esnext",
-    "moduleResolution": "node",
-    "target": "es6",
-    "outDir": "out",
-    "lib": ["ES2021", "dom"],
-    "jsx": "react",
-    "sourceMap": true,
-    "rootDir": "..",
-    "strict": true,
-    "noUnusedLocals": true,
-    "noImplicitReturns": true,
-    "noFallthroughCasesInSwitch": true,
-    "experimentalDecorators": true
-  },
-  "exclude": ["node_modules"]
-}

extensions/ql-vscode/src/config.ts

@@ -2,7 +2,7 @@ import { DisposableObject } from './pure/disposable-object';
 import { workspace, Event, EventEmitter, ConfigurationChangeEvent, ConfigurationTarget } from 'vscode';
 import { DistributionManager } from './distribution';
 import { logger } from './logging';
-import { ONE_DAY_IN_MS } from './pure/helpers-pure';
+import { ONE_DAY_IN_MS } from './pure/time';
 
 /** Helper class to look up a labelled (and possibly nested) setting. */
 export class Setting {
@@ -59,7 +59,7 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIB
 // Query History configuration
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
 const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
-const QUERY_HISTORY_TTL = new Setting('format', QUERY_HISTORY_SETTING);
+const QUERY_HISTORY_TTL = new Setting('ttl', QUERY_HISTORY_SETTING);
 
 /** When these settings change, the distribution should be updated. */
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
@@ -317,12 +317,23 @@ export function isCanary() {
   return !!CANARY_FEATURES.getValue<boolean>();
 }
 
+/**
+ * Enables the experimental query server
+ */
+export const CANARY_QUERY_SERVER = new Setting('canaryQueryServer', ROOT_SETTING);
+
+
+export function allowCanaryQueryServer() {
+  return !!CANARY_QUERY_SERVER.getValue<boolean>();
+}
+
+
 /**
  * Avoids caching in the AST viewer if the user is also a canary user.
  */
 export const NO_CACHE_AST_VIEWER = new Setting('disableCache', AST_VIEWER_SETTING);
 
-// Settings for remote queries
+// Settings for variant analysis
 const REMOTE_QUERIES_SETTING = new Setting('variantAnalysis', ROOT_SETTING);
 
 /**
@@ -342,6 +353,21 @@ export async function setRemoteRepositoryLists(lists: Record<string, string[]> |
   await REMOTE_REPO_LISTS.updateValue(lists, ConfigurationTarget.Global);
 }
 
+/**
+ * Path to a file that contains lists of GitHub repositories that you want to query remotely via
+ * the "Run Variant Analysis" command.
+ * Note: This command is only available for internal users.
+ *
+ * This setting should be a path to a JSON file that contains a JSON object where each key is a
+ * user-specified name (string), and the value is an array of GitHub repositories
+ * (of the form `<owner>/<repo>`).
+ */
+const REPO_LISTS_PATH = new Setting('repositoryListsPath', REMOTE_QUERIES_SETTING);
+
+export function getRemoteRepositoryListsPath(): string | undefined {
+  return REPO_LISTS_PATH.getValue<string>() || undefined;
+}
+
 /**
  * The name of the "controller" repository that you want to use with the "Run Variant Analysis" command.
  * Note: This command is only available for internal users.
@@ -368,3 +394,17 @@ const ACTION_BRANCH = new Setting('actionBranch', REMOTE_QUERIES_SETTING);
 export function getActionBranch(): string {
   return ACTION_BRANCH.getValue<string>() || 'main';
 }
+
+export function isIntegrationTestMode() {
+  return process.env.INTEGRATION_TEST_MODE === 'true';
+}
+
+/**
+ * A flag indicating whether to enable the experimental "live results" feature
+ * for multi-repo variant analyses.
+ */
+const LIVE_RESULTS = new Setting('liveResults', REMOTE_QUERIES_SETTING);
+
+export function isVariantAnalysisLiveResultsEnabled(): boolean {
+  return !!LIVE_RESULTS.getValue<boolean>();
+}

extensions/ql-vscode/src/contextual/astBuilder.ts

@@ -1,10 +1,10 @@
-import { QueryWithResults } from '../run-queries';
 import { CodeQLCliServer } from '../cli';
 import { DecodedBqrsChunk, BqrsId, EntityValue } from '../pure/bqrs-cli-types';
 import { DatabaseItem } from '../databases';
 import { ChildAstItem, AstItem } from '../astViewer';
 import fileRangeFromURI from './fileRangeFromURI';
 import { Uri } from 'vscode';
+import { QueryWithResults } from '../run-queries-shared';
 
 /**
  * A class that wraps a tree of QL results from a query that

extensions/ql-vscode/src/contextual/locationFinder.ts

@@ -3,13 +3,12 @@ import { ColumnKindCode, EntityValue, getResultSetSchema, ResultSetSchema } from
 import { CodeQLCliServer } from '../cli';
 import { DatabaseManager, DatabaseItem } from '../databases';
 import fileRangeFromURI from './fileRangeFromURI';
-import * as messages from '../pure/messages';
-import { QueryServerClient } from '../queryserver-client';
-import { QueryWithResults, compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../run-queries';
 import { ProgressCallback } from '../commandRunner';
 import { KeyType } from './keyType';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
 import { CancellationToken, LocationLink, Uri } from 'vscode';
+import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared';
+import { QueryRunner } from '../queryRunner';
 
 export const SELECT_QUERY_NAME = '#select';
 export const TEMPLATE_NAME = 'selectedSourceFile';
@@ -35,7 +34,7 @@ export interface FullLocationLink extends LocationLink {
  */
 export async function getLocationsForUriString(
   cli: CodeQLCliServer,
-  qs: QueryServerClient,
+  qs: QueryRunner,
   dbm: DatabaseManager,
   uriString: string,
   keyType: KeyType,
@@ -65,19 +64,8 @@ export async function getLocationsForUriString(
       },
       false
     );
-
-    const results = await compileAndRunQueryAgainstDatabase(
-      cli,
-      qs,
-      db,
-      initialInfo,
-      queryStorageDir,
-      progress,
-      token,
-      templates
-    );
-
-    if (results.result.resultType == messages.QueryResultType.SUCCESS) {
+    const results = await qs.compileAndRunQueryAgainstDatabase(db, initialInfo, queryStorageDir, progress, token, templates);
+    if (results.sucessful) {
       links.push(...await getLinksFromResults(results, cli, db, filter));
     }
   }
@@ -114,15 +102,9 @@ async function getLinksFromResults(
   return localLinks;
 }
 
-function createTemplates(path: string): messages.TemplateDefinitions {
+function createTemplates(path: string): Record<string, string> {
   return {
-    [TEMPLATE_NAME]: {
-      values: {
-        tuples: [[{
-          stringValue: path
-        }]]
-      }
-    }
+    [TEMPLATE_NAME]: path
   };
 }
 

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -45,7 +45,7 @@ async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[],
       }
     });
   }
-  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
+  await fs.writeFile(suiteFile, yaml.dump(suiteYaml), 'utf8');
 
   const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
   return queries;

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -16,9 +16,6 @@ import { CodeQLCliServer } from '../cli';
 import { DatabaseManager } from '../databases';
 import { CachedOperation } from '../helpers';
 import { ProgressCallback, withProgress } from '../commandRunner';
-import * as messages from '../pure/messages';
-import { QueryServerClient } from '../queryserver-client';
-import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo, QueryWithResults } from '../run-queries';
 import AstBuilder from './astBuilder';
 import {
   KeyType,
@@ -26,6 +23,8 @@ import {
 import { FullLocationLink, getLocationsForUriString, TEMPLATE_NAME } from './locationFinder';
 import { qlpackOfDatabase, resolveQueries } from './queryResolver';
 import { isCanary, NO_CACHE_AST_VIEWER } from '../config';
+import { createInitialQueryInfo, QueryWithResults } from '../run-queries-shared';
+import { QueryRunner } from '../queryRunner';
 
 /**
  * Run templated CodeQL queries to find definitions and references in
@@ -39,7 +38,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
 
   constructor(
     private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
+    private qs: QueryRunner,
     private dbm: DatabaseManager,
     private queryStorageDir: string,
   ) {
@@ -83,7 +82,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
 
   constructor(
     private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
+    private qs: QueryRunner,
     private dbm: DatabaseManager,
     private queryStorageDir: string,
   ) {
@@ -137,7 +136,7 @@ export class TemplatePrintAstProvider {
 
   constructor(
     private cli: CodeQLCliServer,
-    private qs: QueryServerClient,
+    private qs: QueryRunner,
     private dbm: DatabaseManager,
     private queryStorageDir: string,
   ) {
@@ -195,14 +194,9 @@ export class TemplatePrintAstProvider {
     }
 
     const query = queries[0];
-    const templates: messages.TemplateDefinitions = {
-      [TEMPLATE_NAME]: {
-        values: {
-          tuples: [[{
-            stringValue: zippedArchive.pathWithinSourceArchive
-          }]]
-        }
-      }
+    const templates: Record<string, string> = {
+      [TEMPLATE_NAME]:
+        zippedArchive.pathWithinSourceArchive
     };
 
     const initialInfo = await createInitialQueryInfo(
@@ -215,9 +209,7 @@ export class TemplatePrintAstProvider {
     );
 
     return {
-      query: await compileAndRunQueryAgainstDatabase(
-        this.cli,
-        this.qs,
+      query: await this.qs.compileAndRunQueryAgainstDatabase(
         db,
         initialInfo,
         this.queryStorageDir,
@@ -231,23 +223,23 @@ export class TemplatePrintAstProvider {
 }
 
 export class TemplatePrintCfgProvider {
-  private cache: CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>;
+  private cache: CachedOperation<[Uri, Record<string, string>] | undefined>;
 
   constructor(
     private cli: CodeQLCliServer,
     private dbm: DatabaseManager,
   ) {
-    this.cache = new CachedOperation<[Uri, messages.TemplateDefinitions] | undefined>(this.getCfgUri.bind(this));
+    this.cache = new CachedOperation<[Uri, Record<string, string>] | undefined>(this.getCfgUri.bind(this));
   }
 
-  async provideCfgUri(document?: TextDocument): Promise<[Uri, messages.TemplateDefinitions] | undefined> {
+  async provideCfgUri(document?: TextDocument): Promise<[Uri, Record<string, string>] | undefined> {
     if (!document) {
       return;
     }
     return await this.cache.get(document.uri.toString());
   }
 
-  private async getCfgUri(uriString: string): Promise<[Uri, messages.TemplateDefinitions]> {
+  private async getCfgUri(uriString: string): Promise<[Uri, Record<string, string>]> {
     const uri = Uri.parse(uriString, true);
     if (uri.scheme !== zipArchiveScheme) {
       throw new Error('CFG Viewing is only available for databases with zipped source archives.');
@@ -275,14 +267,8 @@ export class TemplatePrintCfgProvider {
 
     const queryUri = Uri.file(queries[0]);
 
-    const templates: messages.TemplateDefinitions = {
-      [TEMPLATE_NAME]: {
-        values: {
-          tuples: [[{
-            stringValue: zippedArchive.pathWithinSourceArchive
-          }]]
-        }
-      }
+    const templates: Record<string, string> = {
+      [TEMPLATE_NAME]: zippedArchive.pathWithinSourceArchive
     };
 
     return [queryUri, templates];

extensions/ql-vscode/src/databaseFetcher.ts

@@ -10,6 +10,8 @@ import {
 import { CodeQLCliServer } from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
+import * as Octokit from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
 
 import { DatabaseManager, DatabaseItem } from './databases';
 import {
@@ -51,6 +53,7 @@ export async function promptImportInternetDatabase(
     {},
     databaseManager,
     storagePath,
+    undefined,
     progress,
     token,
     cli
@@ -75,7 +78,7 @@ export async function promptImportInternetDatabase(
 export async function promptImportGithubDatabase(
   databaseManager: DatabaseManager,
   storagePath: string,
-  credentials: Credentials,
+  credentials: Credentials | undefined,
   progress: ProgressCallback,
   token: CancellationToken,
   cli?: CodeQLCliServer
@@ -98,12 +101,15 @@ export async function promptImportGithubDatabase(
     throw new Error(`Invalid GitHub repository: ${githubRepo}`);
   }
 
-  const databaseUrl = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress);
-  if (!databaseUrl) {
+  const octokit = credentials ? await credentials.getOctokit(true) : new Octokit.Octokit({ retry });
+
+  const result = await convertGithubNwoToDatabaseUrl(githubRepo, octokit, progress);
+  if (!result) {
     return;
   }
 
-  const octokit = await credentials.getOctokit();
+  const { databaseUrl, name, owner } = result;
+
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
    * The object is undocumented, but looks something like this:
@@ -115,16 +121,12 @@ export async function promptImportGithubDatabase(
    * We only need the actual token string.
    */
   const octokitToken = (await octokit.auth() as { token: string })?.token;
-  if (!octokitToken) {
-    // Just print a generic error message for now. Ideally we could show more debugging info, like the
-    // octokit object, but that would expose a user token.
-    throw new Error('Unable to get GitHub token.');
-  }
   const item = await databaseArchiveFetcher(
     databaseUrl,
-    { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` },
+    { 'Accept': 'application/zip', 'Authorization': octokitToken ? `Bearer ${octokitToken}` : '' },
     databaseManager,
     storagePath,
+    `${owner}/${name}`,
     progress,
     token,
     cli
@@ -173,6 +175,7 @@ export async function promptImportLgtmDatabase(
         {},
         databaseManager,
         storagePath,
+        undefined,
         progress,
         token,
         cli
@@ -220,6 +223,7 @@ export async function importArchiveDatabase(
       {},
       databaseManager,
       storagePath,
+      undefined,
       progress,
       token,
       cli
@@ -247,6 +251,7 @@ export async function importArchiveDatabase(
  * @param requestHeaders Headers to send with the request
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
+ * @param nameOverride a name for the database that overrides the default
  * @param progress callback to send progress messages to
  * @param token cancellation token
  */
@@ -255,6 +260,7 @@ async function databaseArchiveFetcher(
   requestHeaders: { [key: string]: string },
   databaseManager: DatabaseManager,
   storagePath: string,
+  nameOverride: string | undefined,
   progress: ProgressCallback,
   token: CancellationToken,
   cli?: CodeQLCliServer,
@@ -296,7 +302,7 @@ async function databaseArchiveFetcher(
     });
     await ensureZippedSourceLocation(dbPath);
 
-    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath));
+    const item = await databaseManager.openDatabase(progress, token, Uri.file(dbPath), nameOverride);
     await databaseManager.setCurrentDatabaseItem(item);
     return item;
   } else {
@@ -409,7 +415,6 @@ async function fetchAndUnzip(
 
   await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, cli, progress);
 
-
   // remove archivePath eagerly since these archives can be large.
   await fs.remove(archivePath);
 }
@@ -516,13 +521,16 @@ function convertGitHubUrlToNwo(githubUrl: string): string | undefined {
 
 export async function convertGithubNwoToDatabaseUrl(
   githubRepo: string,
-  credentials: Credentials,
-  progress: ProgressCallback): Promise<string | undefined> {
+  octokit: Octokit.Octokit,
+  progress: ProgressCallback): Promise<{
+    databaseUrl: string,
+    owner: string,
+    name: string
+  } | undefined> {
   try {
     const nwo = convertGitHubUrlToNwo(githubRepo) || githubRepo;
     const [owner, repo] = nwo.split('/');
 
-    const octokit = await credentials.getOctokit();
     const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo });
 
     const languages = response.data.map((db: any) => db.language);
@@ -532,7 +540,11 @@ export async function convertGithubNwoToDatabaseUrl(
       return;
     }
 
-    return `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`;
+    return {
+      databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
+      owner,
+      name: repo
+    };
 
   } catch (e) {
     void logger.log(`Error: ${getErrorMessage(e)}`);

extensions/ql-vscode/src/databases-ui.ts

@@ -28,9 +28,6 @@ import {
   showAndLogErrorMessage
 } from './helpers';
 import { logger } from './logging';
-import { clearCacheInDatabase } from './run-queries';
-import * as qsClient from './queryserver-client';
-import { upgradeDatabaseExplicit } from './upgrades';
 import {
   importArchiveDatabase,
   promptImportGithubDatabase,
@@ -40,6 +37,8 @@ import {
 import { CancellationToken } from 'vscode';
 import { asyncFilter, getErrorMessage } from './pure/helpers-pure';
 import { Credentials } from './authentication';
+import { QueryRunner } from './queryRunner';
+import { isCanary } from './config';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -219,7 +218,7 @@ export class DatabaseUI extends DisposableObject {
 
   public constructor(
     private databaseManager: DatabaseManager,
-    private readonly queryServer: qsClient.QueryServerClient | undefined,
+    private readonly queryServer: QueryRunner | undefined,
     private readonly storagePath: string,
     readonly extensionPath: string,
     private readonly getCredentials: () => Promise<Credentials>
@@ -301,7 +300,7 @@ export class DatabaseUI extends DisposableObject {
           progress: ProgressCallback,
           token: CancellationToken
         ) => {
-          const credentials = await this.getCredentials();
+          const credentials = isCanary() ? await this.getCredentials() : undefined;
           await this.handleChooseDatabaseGithub(credentials, progress, token);
         },
         {
@@ -389,12 +388,11 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseFolder = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<DatabaseItem | undefined> => {
+  ): Promise<void> => {
     try {
-      return await this.chooseAndSetDatabase(true, progress, token);
+      await this.chooseAndSetDatabase(true, progress, token);
     } catch (e) {
       void showAndLogErrorMessage(getErrorMessage(e));
-      return undefined;
     }
   };
 
@@ -457,12 +455,11 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseArchive = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<DatabaseItem | undefined> => {
+  ): Promise<void> => {
     try {
-      return await this.chooseAndSetDatabase(false, progress, token);
+      await this.chooseAndSetDatabase(false, progress, token);
     } catch (e) {
       void showAndLogErrorMessage(getErrorMessage(e));
-      return undefined;
     }
   };
 
@@ -480,7 +477,7 @@ export class DatabaseUI extends DisposableObject {
   };
 
   handleChooseDatabaseGithub = async (
-    credentials: Credentials,
+    credentials: Credentials | undefined,
     progress: ProgressCallback,
     token: CancellationToken
   ): Promise<DatabaseItem | undefined> => {
@@ -575,8 +572,7 @@ export class DatabaseUI extends DisposableObject {
 
     // Search for upgrade scripts in any workspace folders available
 
-    await upgradeDatabaseExplicit(
-      this.queryServer,
+    await this.queryServer.upgradeDatabaseExplicit(
       databaseItem,
       progress,
       token
@@ -591,8 +587,7 @@ export class DatabaseUI extends DisposableObject {
       this.queryServer !== undefined &&
       this.databaseManager.currentDatabaseItem !== undefined
     ) {
-      await clearCacheInDatabase(
-        this.queryServer,
+      await this.queryServer.clearCacheInDatabase(
         this.databaseManager.currentDatabaseItem,
         progress,
         token
@@ -755,7 +750,7 @@ export class DatabaseUI extends DisposableObject {
    * Perform some heuristics to ensure a proper database location is chosen.
    *
    * 1. If the selected URI to add is a file, choose the containing directory
-   * 2. If the selected URI is a directory matching db-*, choose the containing directory
+   * 2. If the selected URI appears to be a db language folder, choose the containing directory
    * 3. choose the current directory
    *
    * @param uri a URI that is a database folder or inside it
@@ -768,7 +763,7 @@ export class DatabaseUI extends DisposableObject {
       dbPath = path.dirname(dbPath);
     }
 
-    if (isLikelyDbLanguageFolder(dbPath)) {
+    if (await isLikelyDbLanguageFolder(dbPath)) {
       dbPath = path.dirname(dbPath);
     }
     return Uri.file(dbPath);

extensions/ql-vscode/src/databases.ts

@@ -17,9 +17,8 @@ import {
 import { zipArchiveScheme, encodeArchiveBasePath, decodeSourceArchiveUri, encodeSourceArchiveUri } from './archive-filesystem-provider';
 import { DisposableObject } from './pure/disposable-object';
 import { Logger, logger } from './logging';
-import { registerDatabases, Dataset, deregisterDatabases } from './pure/messages';
-import { QueryServerClient } from './queryserver-client';
 import { getErrorMessage } from './pure/helpers-pure';
+import { QueryRunner } from './queryRunner';
 
 /**
  * databases.ts
@@ -148,7 +147,7 @@ export async function findSourceArchive(
 }
 
 async function resolveDatabase(
-  databasePath: string
+  databasePath: string,
 ): Promise<DatabaseContents> {
 
   const name = path.basename(databasePath);
@@ -170,7 +169,9 @@ async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
   return await glob('*.dbscheme', { cwd: dbDirectory });
 }
 
-async function resolveDatabaseContents(uri: vscode.Uri): Promise<DatabaseContents> {
+async function resolveDatabaseContents(
+  uri: vscode.Uri,
+): Promise<DatabaseContents> {
   if (uri.scheme !== 'file') {
     throw new Error(`Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`);
   }
@@ -357,14 +358,12 @@ export class DatabaseItemImpl implements DatabaseItem {
       try {
         this._contents = await resolveDatabaseContents(this.databaseUri);
         this._error = undefined;
-      }
-      catch (e) {
+      } catch (e) {
         this._contents = undefined;
         this._error = e instanceof Error ? e : new Error(String(e));
         throw e;
       }
-    }
-    finally {
+    } finally {
       this.onChanged({
         kind: DatabaseEventKind.Refresh,
         item: this
@@ -553,13 +552,13 @@ export class DatabaseManager extends DisposableObject {
 
   constructor(
     private readonly ctx: ExtensionContext,
-    private readonly qs: QueryServerClient,
+    private readonly qs: QueryRunner,
     private readonly cli: cli.CodeQLCliServer,
     public logger: Logger
   ) {
     super();
 
-    qs.onDidStartQueryServer(this.reregisterDatabases.bind(this));
+    qs.onStart(this.reregisterDatabases.bind(this));
 
     // Let this run async.
     void this.loadPersistedState();
@@ -569,14 +568,15 @@ export class DatabaseManager extends DisposableObject {
     progress: ProgressCallback,
     token: vscode.CancellationToken,
     uri: vscode.Uri,
+    displayName?: string
   ): Promise<DatabaseItem> {
     const contents = await resolveDatabaseContents(uri);
     // Ignore the source archive for QLTest databases by default.
     const isQLTestDatabase = path.extname(uri.fsPath) === '.testproj';
     const fullOptions: FullDatabaseOptions = {
       ignoreSourceArchive: isQLTestDatabase,
-      // displayName is only set if a user explicitly renames a database
-      displayName: undefined,
+      // If a displayName is not passed in, the basename of folder containing the database is used.
+      displayName,
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath)
     };
@@ -705,6 +705,7 @@ export class DatabaseManager extends DisposableObject {
           step
         });
         try {
+          void this.logger.log(`Found ${databases.length} persisted databases: ${databases.map(db => db.uri).join(', ')}`);
           for (const database of databases) {
             progress({
               maxStep: databases.length,
@@ -719,16 +720,19 @@ export class DatabaseManager extends DisposableObject {
               if (currentDatabaseUri === database.uri) {
                 await this.setCurrentDatabaseItem(databaseItem, true);
               }
-            }
-            catch (e) {
+              void this.logger.log(`Loaded database ${databaseItem.name} at URI ${database.uri}.`);
+            } catch (e) {
               // When loading from persisted state, leave invalid databases in the list. They will be
               // marked as invalid, and cannot be set as the current database.
+              void this.logger.log(`Error loading database ${database.uri}: ${e}.`);
             }
           }
         } catch (e) {
           // database list had an unexpected type - nothing to be done?
           void showAndLogErrorMessage(`Database list loading failed: ${getErrorMessage(e)}`);
         }
+
+        void this.logger.log('Finished loading persisted databases.');
       });
   }
 
@@ -857,27 +861,14 @@ export class DatabaseManager extends DisposableObject {
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
-      const databases: Dataset[] = [{
-        dbDir: dbItem.contents.datasetUri.fsPath,
-        workingSet: 'default'
-      }];
-      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
-    }
+    await this.qs.deregisterDatabase(progress, token, dbItem);
   }
-
   private async registerDatabase(
     progress: ProgressCallback,
     token: vscode.CancellationToken,
     dbItem: DatabaseItem,
   ) {
-    if (dbItem.contents && (await this.cli.cliConstraints.supportsDatabaseRegistration())) {
-      const databases: Dataset[] = [{
-        dbDir: dbItem.contents.datasetUri.fsPath,
-        workingSet: 'default'
-      }];
-      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
-    }
+    await this.qs.registerDatabase(progress, token, dbItem);
   }
 
   private updatePersistedCurrentDatabaseItem(): void {

extensions/ql-vscode/src/eval-log-tree-builder.ts

@@ -0,0 +1,67 @@
+import { ChildEvalLogTreeItem, EvalLogTreeItem } from './eval-log-viewer';
+import { EvalLogData as EvalLogData } from './pure/log-summary-parser';
+
+/** Builds the tree data for the evaluator log viewer for a single query run. */
+export default class EvalLogTreeBuilder {
+  private queryName: string;
+  private evalLogDataItems: EvalLogData[];
+
+  constructor(queryName: string, evaluatorLogDataItems: EvalLogData[]) {
+    this.queryName = queryName;
+    this.evalLogDataItems = evaluatorLogDataItems;
+  }
+
+  async getRoots(): Promise<EvalLogTreeItem[]> {
+    return await this.parseRoots();
+  }
+
+  private async parseRoots(): Promise<EvalLogTreeItem[]> {
+    const roots: EvalLogTreeItem[] = [];
+
+    // Once the viewer can show logs for multiple queries, there will be more than 1 item at the root
+    // level. For now, there will always be one root (the one query being shown).
+    const queryItem: EvalLogTreeItem = {
+      label: this.queryName,
+      children: [] // Will assign predicate items as children shortly.
+    };
+
+    // Display descriptive message when no data exists 
+    if (this.evalLogDataItems.length === 0) {
+      const noResultsItem: ChildEvalLogTreeItem = {
+        label: 'No predicates evaluated in this query run.',
+        parent: queryItem,
+        children: [],
+      };
+      queryItem.children.push(noResultsItem);
+    }
+
+    // For each predicate, create a TreeItem object with appropriate parents/children 
+    this.evalLogDataItems.forEach(logDataItem => {
+      const predicateLabel = `${logDataItem.predicateName} (${logDataItem.resultSize} tuples, ${logDataItem.millis} ms)`;
+      const predicateItem: ChildEvalLogTreeItem = {
+        label: predicateLabel,
+        parent: queryItem,
+        children: [] // Will assign pipeline items as children shortly.
+      };
+      for (const [pipelineName, steps] of Object.entries(logDataItem.ra)) {
+        const pipelineLabel = `Pipeline: ${pipelineName}`;
+        const pipelineItem: ChildEvalLogTreeItem = {
+          label: pipelineLabel,
+          parent: predicateItem,
+          children: [] // Will assign step items as children shortly.
+        };
+        predicateItem.children.push(pipelineItem);
+
+        pipelineItem.children = steps.map((step: string) => ({
+          label: step,
+          parent: pipelineItem,
+          children: []
+        }));
+      }
+      queryItem.children.push(predicateItem);
+    });
+
+    roots.push(queryItem);
+    return roots;
+  }
+}

extensions/ql-vscode/src/eval-log-viewer.ts

@@ -0,0 +1,92 @@
+import { window, TreeDataProvider, TreeView, TreeItem, ProviderResult, Event, EventEmitter, TreeItemCollapsibleState } from 'vscode';
+import { commandRunner } from './commandRunner';
+import { DisposableObject } from './pure/disposable-object';
+import { showAndLogErrorMessage } from './helpers';
+
+export interface EvalLogTreeItem {
+  label?: string;
+  children: ChildEvalLogTreeItem[];
+}
+
+export interface ChildEvalLogTreeItem extends EvalLogTreeItem {
+  parent: ChildEvalLogTreeItem | EvalLogTreeItem;
+}
+
+/** Provides data from parsed CodeQL evaluator logs to be rendered in a tree view. */
+class EvalLogDataProvider extends DisposableObject implements TreeDataProvider<EvalLogTreeItem> {
+  public roots: EvalLogTreeItem[] = [];
+
+  private _onDidChangeTreeData: EventEmitter<EvalLogTreeItem | undefined | null | void> = new EventEmitter<EvalLogTreeItem | undefined | null | void>();
+  readonly onDidChangeTreeData: Event<EvalLogTreeItem | undefined | null | void> = this._onDidChangeTreeData.event;
+
+  refresh(): void {
+    this._onDidChangeTreeData.fire();
+  }
+
+  getTreeItem(element: EvalLogTreeItem): TreeItem | Thenable<TreeItem> {
+    const state = element.children.length
+      ? TreeItemCollapsibleState.Collapsed
+      : TreeItemCollapsibleState.None;
+    const treeItem = new TreeItem(element.label || '', state);
+    treeItem.tooltip = `${treeItem.label} || ''}`;
+    return treeItem;
+  }
+
+  getChildren(element?: EvalLogTreeItem): ProviderResult<EvalLogTreeItem[]> {
+    // If no item is passed, return the root.
+    if (!element) {
+      return this.roots || [];
+    }
+    // Otherwise it is called with an existing item, to load its children.
+    return element.children;
+  }
+
+  getParent(element: ChildEvalLogTreeItem): ProviderResult<EvalLogTreeItem> {
+    return element.parent;
+  }
+}
+
+/** Manages a tree viewer of structured evaluator logs. */
+export class EvalLogViewer extends DisposableObject {
+  private treeView: TreeView<EvalLogTreeItem>;
+  private treeDataProvider: EvalLogDataProvider;
+
+  constructor() {
+    super();
+
+    this.treeDataProvider = new EvalLogDataProvider();
+    this.treeView = window.createTreeView('codeQLEvalLogViewer', {
+      treeDataProvider: this.treeDataProvider,
+      showCollapseAll: true
+    });
+
+    this.push(this.treeView);
+    this.push(this.treeDataProvider);
+    this.push(
+      commandRunner('codeQLEvalLogViewer.clear', async () => {
+        this.clear();
+      })
+    );
+  }
+
+  private clear(): void {
+    this.treeDataProvider.roots = [];
+    this.treeDataProvider.refresh();
+    this.treeView.message = undefined;
+  }
+
+  // Called when the Show Evaluator Log (UI) command is run on a new query.
+  updateRoots(roots: EvalLogTreeItem[]): void {
+    this.treeDataProvider.roots = roots;
+    this.treeDataProvider.refresh();
+
+    this.treeView.message = 'Viewer for query run:'; // Currently only one query supported at a time. 
+
+    // Handle error on reveal. This could happen if
+    // the tree view is disposed during the reveal.
+    this.treeView.reveal(roots[0], { focus: false })?.then(
+      () => { /**/ },
+      err => showAndLogErrorMessage(err)
+    );
+  }
+}

extensions/ql-vscode/src/extension.ts

@@ -68,17 +68,17 @@ import {
 } from './helpers';
 import { asError, assertNever, getErrorMessage } from './pure/helpers-pure';
 import { spawnIdeServer } from './ide-server';
-import { InterfaceManager } from './interface';
+import { ResultsView } from './interface';
 import { WebviewReveal } from './interface-utils';
-import { ideServerLogger, logger, queryServerLogger } from './logging';
+import { ideServerLogger, logger, ProgressReporter, queryServerLogger } from './logging';
 import { QueryHistoryManager } from './query-history';
 import { CompletedLocalQueryInfo, LocalQueryInfo } from './query-results';
-import * as qsClient from './queryserver-client';
+import * as legacyQueryServer from './legacy-query-server/queryserver-client';
+import * as newQueryServer from './query-server/queryserver-client';
 import { displayQuickQuery } from './quick-query';
-import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from './run-queries';
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
-import { CompareInterfaceManager } from './compare/compare-interface';
+import { CompareView } from './compare/compare-view';
 import { gatherQlFiles } from './pure/files';
 import { initializeTelemetry } from './telemetry';
 import {
@@ -95,7 +95,25 @@ import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
 import { RemoteQueryResult } from './remote-queries/remote-query-result';
 import { URLSearchParams } from 'url';
 import { handleDownloadPacks, handleInstallPackDependencies } from './packaging';
-import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
+import { HistoryItemLabelProvider } from './history-item-label-provider';
+import { exportRemoteQueryResults } from './remote-queries/export-results';
+import { RemoteQuery } from './remote-queries/remote-query';
+import { EvalLogViewer } from './eval-log-viewer';
+import { SummaryLanguageSupport } from './log-insights/summary-language-support';
+import { JoinOrderScannerProvider } from './log-insights/join-order';
+import { LogScannerService } from './log-insights/log-scanner-service';
+import { createInitialQueryInfo } from './run-queries-shared';
+import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
+import { NewQueryRunner } from './query-server/query-runner';
+import { QueryRunner } from './queryRunner';
+import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
+import { VariantAnalysisViewSerializer } from './remote-queries/variant-analysis-view-serializer';
+import { VariantAnalysis } from './remote-queries/shared/variant-analysis';
+import {
+  VariantAnalysis as VariantAnalysisApiResponse,
+  VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository
+} from './remote-queries/gh-api/variant-analysis';
+import { VariantAnalysisManager } from './remote-queries/variant-analysis-manager';
 
 /**
  * extension.ts
@@ -158,10 +176,11 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 export interface CodeQLExtensionInterface {
   readonly ctx: ExtensionContext;
   readonly cliServer: CodeQLCliServer;
-  readonly qs: qsClient.QueryServerClient;
+  readonly qs: QueryRunner;
   readonly distributionManager: DistributionManager;
   readonly databaseManager: DatabaseManager;
   readonly databaseUI: DatabaseUI;
+  readonly variantAnalysisManager: VariantAnalysisManager;
   readonly dispose: () => void;
 }
 
@@ -372,7 +391,10 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     allowAutoUpdating: true
   })));
 
-  return await installOrUpdateThenTryActivate({
+  const variantAnalysisViewSerializer = new VariantAnalysisViewSerializer(ctx);
+  Window.registerWebviewPanelSerializer(VariantAnalysisView.viewType, variantAnalysisViewSerializer);
+
+  const codeQlExtension = await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
 
@@ -380,8 +402,14 @@ export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionIn
     // otherwise, ask user to accept the update
     allowAutoUpdating: !!ctx.globalState.get(shouldUpdateOnNextActivationKey)
   });
+
+  variantAnalysisViewSerializer.onExtensionLoaded(codeQlExtension.variantAnalysisManager);
+
+  return codeQlExtension;
 }
 
+const PACK_GLOBS = ['**/codeql-pack.yml', '**/qlpack.yml', '**/queries.xml', '**/codeql-pack.lock.yml', '**/qlpack.lock.yml', '.codeqlmanifest.json', 'codeql-workspace.yml'];
+
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
   distributionManager: DistributionManager,
@@ -410,21 +438,16 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(statusBar);
 
   void logger.log('Initializing query server client.');
-  const qs = new qsClient.QueryServerClient(
-    qlConfigurationListener,
-    cliServer,
-    {
-      logger: queryServerLogger,
-      contextStoragePath: getContextStoragePath(ctx),
-    },
-    (task) =>
-      Window.withProgress(
-        { title: 'CodeQL query server', location: ProgressLocation.Window },
-        task
-      )
-  );
-  ctx.subscriptions.push(qs);
-  await qs.startQueryServer();
+  const qs = await createQueryServer(qlConfigurationListener, cliServer, ctx);
+
+
+  for (const glob of PACK_GLOBS) {
+    const fsWatcher = workspace.createFileSystemWatcher(glob);
+    ctx.subscriptions.push(fsWatcher);
+    fsWatcher.onDidChange(async (_uri) => {
+      await qs.clearPackCache();
+    });
+  }
 
   void logger.log('Initializing database manager.');
   const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
@@ -440,6 +463,10 @@ async function activateWithInstalledDistribution(
   databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
+  void logger.log('Initializing evaluator log viewer.');
+  const evalLogViewer = new EvalLogViewer();
+  ctx.subscriptions.push(evalLogViewer);
+
   void logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
   ctx.subscriptions.push(queryHistoryConfigurationListener);
@@ -447,38 +474,58 @@ async function activateWithInstalledDistribution(
     showResultsForCompletedQuery(item, WebviewReveal.Forced);
   const queryStorageDir = path.join(ctx.globalStorageUri.fsPath, 'queries');
   await fs.ensureDir(queryStorageDir);
+  const labelProvider = new HistoryItemLabelProvider(queryHistoryConfigurationListener);
+
+  void logger.log('Initializing results panel interface.');
+  const localQueryResultsView = new ResultsView(ctx, dbm, cliServer, queryServerLogger, labelProvider);
+  ctx.subscriptions.push(localQueryResultsView);
+
+  void logger.log('Initializing variant analysis manager.');
+  const variantAnalysisStorageDir = path.join(ctx.globalStorageUri.fsPath, 'variant-analyses');
+  await fs.ensureDir(variantAnalysisStorageDir);
+  const variantAnalysisManager = new VariantAnalysisManager(ctx, cliServer, variantAnalysisStorageDir, logger);
+  ctx.subscriptions.push(variantAnalysisManager);
+
+  void logger.log('Initializing remote queries manager.');
+  const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger, variantAnalysisManager);
+  ctx.subscriptions.push(rqm);
 
   void logger.log('Initializing query history.');
   const qhm = new QueryHistoryManager(
     qs,
     dbm,
+    localQueryResultsView,
+    rqm,
+    evalLogViewer,
     queryStorageDir,
     ctx,
     queryHistoryConfigurationListener,
+    labelProvider,
     async (from: CompletedLocalQueryInfo, to: CompletedLocalQueryInfo) =>
       showResultsForComparison(from, to),
   );
 
-  qhm.onWillOpenQueryItem(async item => {
-    if (item.t === 'local' && item.completed) {
-      await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.Forced);
-    }
-  });
 
   ctx.subscriptions.push(qhm);
-  void logger.log('Initializing results panel interface.');
-  const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
-  ctx.subscriptions.push(intm);
 
-  void logger.log('Initializing compare panel interface.');
-  const cmpm = new CompareInterfaceManager(
+  void logger.log('Initializing evaluation log scanners.');
+  const logScannerService = new LogScannerService(qhm);
+  ctx.subscriptions.push(logScannerService);
+  ctx.subscriptions.push(logScannerService.scanners.registerLogScannerProvider(new JoinOrderScannerProvider()));
+
+  void logger.log('Reading query history');
+  await qhm.readQueryHistory();
+
+  void logger.log('Initializing compare view.');
+  const compareView = new CompareView(
     ctx,
     dbm,
     cliServer,
     queryServerLogger,
+    labelProvider,
     showResults
   );
-  ctx.subscriptions.push(cmpm);
+  ctx.subscriptions.push(compareView);
 
   void logger.log('Initializing source archive filesystem provider.');
   archiveFilesystemProvider.activate(ctx);
@@ -488,7 +535,7 @@ async function activateWithInstalledDistribution(
     to: CompletedLocalQueryInfo
   ): Promise<void> {
     try {
-      await cmpm.showResults(from, to);
+      await compareView.showResults(from, to);
     } catch (e) {
       void showAndLogErrorMessage(getErrorMessage(e));
     }
@@ -498,7 +545,7 @@ async function activateWithInstalledDistribution(
     query: CompletedLocalQueryInfo,
     forceReveal: WebviewReveal
   ): Promise<void> {
-    await intm.showResults(query, forceReveal, false);
+    await localQueryResultsView.showResults(query, forceReveal, false);
   }
 
   async function compileAndRunQuery(
@@ -525,12 +572,10 @@ async function activateWithInstalledDistribution(
       token.onCancellationRequested(() => source.cancel());
 
       const initialInfo = await createInitialQueryInfo(selectedQuery, databaseInfo, quickEval, range);
-      const item = new LocalQueryInfo(initialInfo, queryHistoryConfigurationListener, source);
+      const item = new LocalQueryInfo(initialInfo, source);
       qhm.addQuery(item);
       try {
-        const completedQueryInfo = await compileAndRunQueryAgainstDatabase(
-          cliServer,
-          qs,
+        const completedQueryInfo = await qs.compileAndRunQueryAgainstDatabase(
           databaseItem,
           initialInfo,
           queryStorageDir,
@@ -539,8 +584,8 @@ async function activateWithInstalledDistribution(
           undefined,
           item,
         );
-        item.completeThisQuery(completedQueryInfo);
-        await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced);
+        qhm.completeQuery(item, completedQueryInfo);
+        await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.Forced);
         // Note we must update the query history view after showing results as the
         // display and sorting might depend on the number of results
       } catch (e) {
@@ -758,12 +803,13 @@ async function activateWithInstalledDistribution(
           });
         }
 
-        if (queryUris.length > 1) {
+        if (queryUris.length > 1 && !await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
           // Try to upgrade the current database before running any queries
           // so that the user isn't confronted with multiple upgrade
           // requests for each query to run.
           // Only do it if running multiple queries since this check is
           // performed on each query run anyway.
+          // Don't do this with non destructive upgrades as the user won't see anything anyway.
           await databaseUI.tryUpgradeCurrentDatabase(progress, token);
         }
 
@@ -838,14 +884,6 @@ async function activateWithInstalledDistribution(
     )
   );
 
-  void logger.log('Initializing remote queries interface.');
-  const rqm = new RemoteQueriesManager(ctx, cliServer, qhm, queryStorageDir, logger);
-  ctx.subscriptions.push(rqm);
-
-  // wait until after the remote queries manager is initialized to read the query history
-  // since the rqm is notified of queries being added.
-  await qhm.readQueryHistory();
-
 
   registerRemoteQueryTextProvider();
 
@@ -868,7 +906,7 @@ async function activateWithInstalledDistribution(
           token
         );
       } else {
-        throw new Error('Remote queries require the CodeQL Canary version to run.');
+        throw new Error('Variant analysis requires the CodeQL Canary version to run.');
       }
     }, {
       title: 'Run Variant Analysis',
@@ -878,11 +916,43 @@ async function activateWithInstalledDistribution(
 
   ctx.subscriptions.push(
     commandRunner('codeQL.monitorRemoteQuery', async (
-      queryItem: RemoteQueryHistoryItem,
+      queryId: string,
+      query: RemoteQuery,
       token: CancellationToken) => {
-      await rqm.monitorRemoteQuery(queryItem, token);
+      await rqm.monitorRemoteQuery(queryId, query, token);
     }));
 
+  ctx.subscriptions.push(
+    commandRunner('codeQL.copyRepoList', async (queryId: string) => {
+      await rqm.copyRemoteQueryRepoListToClipboard(queryId);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.monitorVariantAnalysis', async (
+      variantAnalysis: VariantAnalysis,
+      token: CancellationToken
+    ) => {
+      await variantAnalysisManager.monitorVariantAnalysis(variantAnalysis, token);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.autoDownloadVariantAnalysisResult', async (
+      scannedRepo: ApiVariantAnalysisScannedRepository,
+      variantAnalysisSummary: VariantAnalysisApiResponse,
+      token: CancellationToken
+    ) => {
+      await variantAnalysisManager.autoDownloadVariantAnalysisResult(scannedRepo, variantAnalysisSummary, token);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openVariantAnalysis', async () => {
+      await variantAnalysisManager.promptOpenVariantAnalysis();
+    })
+  );
+
   ctx.subscriptions.push(
     commandRunner('codeQL.autoDownloadRemoteQueryResults', async (
       queryResult: RemoteQueryResult,
@@ -890,6 +960,25 @@ async function activateWithInstalledDistribution(
       await rqm.autoDownloadRemoteQueryResults(queryResult, token);
     }));
 
+  ctx.subscriptions.push(
+    commandRunner('codeQL.exportVariantAnalysisResults', async (queryId?: string) => {
+      await exportRemoteQueryResults(qhm, rqm, ctx, queryId);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.loadVariantAnalysisRepoResults', async (variantAnalysisId: number, repositoryFullName: string) => {
+      await variantAnalysisManager.loadResults(variantAnalysisId, repositoryFullName);
+    })
+  );
+
+  // The "openVariantAnalysisView" command is internal-only.
+  ctx.subscriptions.push(
+    commandRunner('codeQL.openVariantAnalysisView', async (variantAnalysisId: number) => {
+      await variantAnalysisManager.showView(variantAnalysisId);
+    })
+  );
+
   ctx.subscriptions.push(
     commandRunner(
       'codeQL.openReferencedFile',
@@ -909,6 +998,8 @@ async function activateWithInstalledDistribution(
       progress: ProgressCallback,
       token: CancellationToken
     ) => {
+      // We restart the CLI server too, to ensure they are the same version
+      cliServer.restartCliServer();
       await qs.restartQueryServer(progress, token);
       void showAndLogInformationMessage('CodeQL Query Server restarted.', {
         outputLogger: queryServerLogger,
@@ -941,7 +1032,7 @@ async function activateWithInstalledDistribution(
       progress: ProgressCallback,
       token: CancellationToken
     ) => {
-      const credentials = await Credentials.initialize(ctx);
+      const credentials = isCanary() ? await Credentials.initialize(ctx) : undefined;
       await databaseUI.handleChooseDatabaseGithub(credentials, progress, token);
     },
       {
@@ -989,19 +1080,16 @@ async function activateWithInstalledDistribution(
     }
   };
 
-  // The "authenticateToGitHub" command is internal-only.
   ctx.subscriptions.push(
     commandRunner('codeQL.authenticateToGitHub', async () => {
-      if (isCanary()) {
-        /**
-         * Credentials for authenticating to GitHub.
-         * These are used when making API calls.
-         */
-        const credentials = await Credentials.initialize(ctx);
-        const octokit = await credentials.getOctokit();
-        const userInfo = await octokit.users.getAuthenticated();
-        void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
-      }
+      /**
+       * Credentials for authenticating to GitHub.
+       * These are used when making API calls.
+       */
+      const credentials = await Credentials.initialize(ctx);
+      const octokit = await credentials.getOctokit();
+      const userInfo = await octokit.users.getAuthenticated();
+      void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
     }));
 
   ctx.subscriptions.push(
@@ -1030,6 +1118,8 @@ async function activateWithInstalledDistribution(
     })
   );
 
+  ctx.subscriptions.push(new SummaryLanguageSupport());
+
   void logger.log('Starting language server.');
   ctx.subscriptions.push(client.start());
 
@@ -1103,12 +1193,46 @@ async function activateWithInstalledDistribution(
     distributionManager,
     databaseManager: dbm,
     databaseUI,
+    variantAnalysisManager,
     dispose: () => {
       ctx.subscriptions.forEach(d => d.dispose());
     }
   };
 }
 
+async function createQueryServer(qlConfigurationListener: QueryServerConfigListener, cliServer: CodeQLCliServer, ctx: ExtensionContext): Promise<QueryRunner> {
+  const qsOpts = {
+    logger: queryServerLogger,
+    contextStoragePath: getContextStoragePath(ctx),
+  };
+  const progressCallback = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Window.withProgress(
+    { title: 'CodeQL query server', location: ProgressLocation.Window },
+    task
+  );
+  if (await cliServer.cliConstraints.supportsNewQueryServer()) {
+    const qs = new newQueryServer.QueryServerClient(
+      qlConfigurationListener,
+      cliServer,
+      qsOpts,
+      progressCallback
+    );
+    ctx.subscriptions.push(qs);
+    await qs.startQueryServer();
+    return new NewQueryRunner(qs);
+
+  } else {
+    const qs = new legacyQueryServer.QueryServerClient(
+      qlConfigurationListener,
+      cliServer,
+      qsOpts,
+      progressCallback
+    );
+    ctx.subscriptions.push(qs);
+    await qs.startQueryServer();
+    return new LegacyQueryRunner(qs);
+  }
+}
+
 function getContextStoragePath(ctx: ExtensionContext) {
   return ctx.storageUri?.fsPath || ctx.globalStorageUri.fsPath;
 }

extensions/ql-vscode/src/helpers.ts

@@ -76,9 +76,10 @@ export async function showAndLogWarningMessage(message: string, {
  */
 export async function showAndLogInformationMessage(message: string, {
   outputLogger = logger,
-  items = [] as string[]
+  items = [] as string[],
+  fullMessage = ''
 } = {}): Promise<string | undefined> {
-  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage);
+  return internalShowAndLog(message, items, outputLogger, Window.showInformationMessage, fullMessage);
 }
 
 type ShowMessageFn = (message: string, ...items: string[]) => Thenable<string | undefined>;
@@ -288,7 +289,7 @@ interface QlPackWithPath {
 async function findDbschemePack(packs: QlPackWithPath[], dbschemePath: string): Promise<{ name: string; isLibraryPack: boolean; }> {
   for (const { packDir, packName } of packs) {
     if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
+      const qlpack = yaml.load(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme?: string; library?: boolean; };
       if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
         return {
           name: packName,
@@ -469,9 +470,9 @@ export function getInitialQueryContents(language: string, dbscheme: string) {
 
 /**
  * Heuristically determines if the directory passed in corresponds
- * to a database root.
- *
- * @param maybeRoot
+ * to a database root. A database root is a directory that contains
+ * a codeql-database.yml or (historically) a .dbinfo file. It also
+ * contains a folder starting with `db-`.
  */
 export async function isLikelyDatabaseRoot(maybeRoot: string) {
   const [a, b, c] = (await Promise.all([
@@ -483,11 +484,14 @@ export async function isLikelyDatabaseRoot(maybeRoot: string) {
     glob('db-*/', { cwd: maybeRoot })
   ]));
 
-  return !!((a || b) && c);
+  return ((a || b) && c.length > 0);
 }
 
-export function isLikelyDbLanguageFolder(dbPath: string) {
-  return !!path.basename(dbPath).startsWith('db-');
+/**
+ * A language folder is any folder starting with `db-` that is itself not a database root.
+ */
+export async function isLikelyDbLanguageFolder(dbPath: string) {
+  return path.basename(dbPath).startsWith('db-') && !(await isLikelyDatabaseRoot(dbPath));
 }
 
 /**
@@ -580,3 +584,11 @@ export async function* walkDirectory(dir: string): AsyncIterableIterator<string>
     }
   }
 }
+
+/**
+ * Pluralizes a word.
+ * Example: Returns "N repository" if N is one, "N repositories" otherwise.
+ */
+export function pluralize(numItems: number | undefined, singular: string, plural: string): string {
+  return numItems ? `${numItems} ${numItems === 1 ? singular : plural}` : '';
+}

extensions/ql-vscode/src/history-item-label-provider.ts

@@ -0,0 +1,93 @@
+import { env } from 'vscode';
+import * as path from 'path';
+import { QueryHistoryConfig } from './config';
+import { LocalQueryInfo, QueryHistoryInfo } from './query-results';
+import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
+import { pluralize } from './helpers';
+
+interface InterpolateReplacements {
+  t: string; // Start time
+  q: string; // Query name
+  d: string; // Database/Controller repo name
+  r: string; // Result count/Empty
+  s: string; // Status
+  f: string; // Query file name
+  '%': '%'; // Percent sign
+}
+
+export class HistoryItemLabelProvider {
+  constructor(private config: QueryHistoryConfig) {
+    /**/
+  }
+
+  getLabel(item: QueryHistoryInfo) {
+    const replacements = item.t === 'local'
+      ? this.getLocalInterpolateReplacements(item)
+      : this.getRemoteInterpolateReplacements(item);
+
+    const rawLabel = item.userSpecifiedLabel ?? (this.config.format || '%q');
+
+    return this.interpolate(rawLabel, replacements);
+  }
+
+  /**
+   * If there is a user-specified label for this query, interpolate and use that.
+   * Otherwise, use the raw name of this query.
+   *
+   * @returns the name of the query, unless there is a custom label for this query.
+   */
+  getShortLabel(item: QueryHistoryInfo): string {
+    return item.userSpecifiedLabel
+      ? this.getLabel(item)
+      : item.t === 'local'
+        ? item.getQueryName()
+        : item.remoteQuery.queryName;
+  }
+
+
+  private interpolate(rawLabel: string, replacements: InterpolateReplacements): string {
+    const label = rawLabel.replace(/%(.)/g, (match, key: keyof InterpolateReplacements) => {
+      const replacement = replacements[key];
+      return replacement !== undefined ? replacement : match;
+    });
+
+    return label.replace(/\s+/g, ' ');
+  }
+
+  private getLocalInterpolateReplacements(item: LocalQueryInfo): InterpolateReplacements {
+    const { resultCount = 0, statusString = 'in progress' } = item.completedQuery || {};
+    return {
+      t: item.startTime,
+      q: item.getQueryName(),
+      d: item.initialInfo.databaseInfo.name,
+      r: `(${resultCount} results)`,
+      s: statusString,
+      f: item.getQueryFileName(),
+      '%': '%',
+    };
+  }
+
+  // Return the number of repositories queried if available. Otherwise, use the controller repository name.
+  private buildRepoLabel(item: RemoteQueryHistoryItem): string {
+    const repositoryCount = item.remoteQuery.repositoryCount;
+
+    if (repositoryCount) {
+      return pluralize(repositoryCount, 'repository', 'repositories');
+    }
+
+    return `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`;
+  }
+
+  private getRemoteInterpolateReplacements(item: RemoteQueryHistoryItem): InterpolateReplacements {
+    const resultCount = item.resultCount ? `(${pluralize(item.resultCount, 'result', 'results')})` : '';
+    return {
+      t: new Date(item.remoteQuery.executionStartTime).toLocaleString(env.language),
+      q: `${item.remoteQuery.queryName} (${item.remoteQuery.language})`,
+      d: this.buildRepoLabel(item),
+      r: resultCount,
+      s: item.status,
+      f: path.basename(item.remoteQuery.queryFilePath),
+      '%': '%'
+    };
+  }
+}

extensions/ql-vscode/src/interface-utils.ts

@@ -4,6 +4,7 @@ import {
   Uri,
   Location,
   Range,
+  ExtensionContext,
   WebviewPanel,
   Webview,
   workspace,
@@ -111,16 +112,36 @@ export function tryResolveLocation(
   }
 }
 
+export type WebviewView = 'results' | 'compare' | 'remote-queries' | 'variant-analysis';
+
+export interface WebviewMessage {
+  t: string;
+}
+
 /**
  * Returns HTML to populate the given webview.
  * Uses a content security policy that only loads the given script.
  */
 export function getHtmlForWebview(
+  ctx: ExtensionContext,
   webview: Webview,
-  scriptUriOnDisk: Uri,
-  stylesheetUrisOnDisk: Uri[],
-  allowInlineStyles: boolean
+  view: WebviewView,
+  {
+    allowInlineStyles,
+  }: {
+    allowInlineStyles?: boolean;
+  } = {
+      allowInlineStyles: false,
+    }
 ): string {
+  const scriptUriOnDisk = Uri.file(
+    ctx.asAbsolutePath('out/webview.js')
+  );
+
+  const stylesheetUrisOnDisk = [
+    Uri.file(ctx.asAbsolutePath('out/webview.css'))
+  ];
+
   // Convert the on-disk URIs into webview URIs.
   const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
   const stylesheetWebviewUris = stylesheetUrisOnDisk.map(stylesheetUriOnDisk =>
@@ -137,6 +158,8 @@ export function getHtmlForWebview(
     ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
     : `'nonce-${nonce}'`;
 
+  const fontSrc = webview.cspSource;
+
   /*
    * Content security policy:
    * default-src: allow nothing by default.
@@ -149,11 +172,11 @@ export function getHtmlForWebview(
 <html>
   <head>
     <meta http-equiv="Content-Security-Policy"
-          content="default-src 'none'; script-src 'nonce-${nonce}'; style-src ${styleSrc}; connect-src ${webview.cspSource};">
+          content="default-src 'none'; script-src 'nonce-${nonce}'; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${webview.cspSource};">
         ${stylesheetsHtmlLines.join(`    ${os.EOL}`)}
   </head>
   <body>
-    <div id=root>
+    <div id=root data-view="${view}">
     </div>
       <script nonce="${nonce}" src="${scriptWebviewUri}">
     </script>

extensions/ql-vscode/src/interface.ts

@@ -1,6 +1,4 @@
-import * as path from 'path';
 import * as Sarif from 'sarif';
-import { DisposableObject } from './pure/disposable-object';
 import * as vscode from 'vscode';
 import {
   Diagnostic,
@@ -14,7 +12,7 @@ import {
 import * as cli from './cli';
 import { CodeQLCliServer } from './cli';
 import { DatabaseEventKind, DatabaseItem, DatabaseManager } from './databases';
-import { showAndLogErrorMessage, tmpDir } from './helpers';
+import { showAndLogErrorMessage } from './helpers';
 import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import {
   FromResultsViewMsg,
@@ -31,24 +29,24 @@ import {
   RawResultsSortState,
 } from './pure/interface-types';
 import { Logger } from './logging';
-import * as messages from './pure/messages';
 import { commandRunner } from './commandRunner';
 import { CompletedQueryInfo, interpretResultsSarif, interpretGraphResults } from './query-results';
-import { QueryEvaluationInfo } from './run-queries';
+import { QueryEvaluationInfo } from './run-queries-shared';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './pure/sarif-utils';
 import {
   WebviewReveal,
   fileUriToWebviewUri,
   tryResolveLocation,
-  getHtmlForWebview,
   shownLocationDecoration,
   shownLocationLineDecoration,
   jumpToLocation,
 } from './interface-utils';
 import { getDefaultResultSetName, ParsedResultSets } from './pure/interface-types';
 import { RawResultSet, transformBqrsResultSet, ResultSetSchema } from './pure/bqrs-cli-types';
+import { AbstractWebview, WebviewPanelConfig } from './abstract-webview';
 import { PAGE_SIZE } from './config';
 import { CompletedLocalQueryInfo } from './query-results';
+import { HistoryItemLabelProvider } from './history-item-label-provider';
 
 /**
  * interface.ts
@@ -121,12 +119,9 @@ function numInterpretedPages(interpretation: Interpretation | undefined): number
   return Math.ceil(n / pageSize);
 }
 
-export class InterfaceManager extends DisposableObject {
+export class ResultsView extends AbstractWebview<IntoResultsViewMsg, FromResultsViewMsg> {
   private _displayedQuery?: CompletedLocalQueryInfo;
   private _interpretation?: Interpretation;
-  private _panel: vscode.WebviewPanel | undefined;
-  private _panelLoaded = false;
-  private _panelLoadedCallBacks: (() => void)[] = [];
 
   private readonly _diagnosticCollection = languages.createDiagnosticCollection(
     'codeql-query-results'
@@ -136,9 +131,10 @@ export class InterfaceManager extends DisposableObject {
     public ctx: vscode.ExtensionContext,
     private databaseManager: DatabaseManager,
     public cliServer: CodeQLCliServer,
-    public logger: Logger
+    public logger: Logger,
+    private labelProvider: HistoryItemLabelProvider
   ) {
-    super();
+    super(ctx);
     this.push(this._diagnosticCollection);
     this.push(
       vscode.window.onDidChangeTextEditorSelection(
@@ -163,7 +159,7 @@ export class InterfaceManager extends DisposableObject {
       this.databaseManager.onDidChangeDatabaseItem(({ kind }) => {
         if (kind === DatabaseEventKind.Remove) {
           this._diagnosticCollection.clear();
-          if (this.isShowingPanel()) {
+          if (this.isShowingPanel) {
             void this.postMessage({
               t: 'untoggleShowProblems'
             });
@@ -177,59 +173,81 @@ export class InterfaceManager extends DisposableObject {
     await this.postMessage({ t: 'navigatePath', direction });
   }
 
-  private isShowingPanel() {
-    return !!this._panel;
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: 'resultsView',
+      title: 'CodeQL Query Results',
+      viewColumn: this.chooseColumnForWebview(),
+      preserveFocus: true,
+      view: 'results',
+    };
   }
 
-  // Returns the webview panel, creating it if it doesn't already
-  // exist.
-  getPanel(): vscode.WebviewPanel {
-    if (this._panel == undefined) {
-      const { ctx } = this;
-      const webViewColumn = this.chooseColumnForWebview();
-      const panel = (this._panel = Window.createWebviewPanel(
-        'resultsView', // internal name
-        'CodeQL Query Results', // user-visible name
-        { viewColumn: webViewColumn, preserveFocus: true },
-        {
-          enableScripts: true,
-          enableFindWidget: true,
-          retainContextWhenHidden: true,
-          localResourceRoots: [
-            vscode.Uri.file(tmpDir.name),
-            vscode.Uri.file(path.join(this.ctx.extensionPath, 'out'))
-          ]
-        }
-      ));
+  protected onPanelDispose(): void {
+    this._displayedQuery = undefined;
+  }
 
-      this.push(this._panel.onDidDispose(
-        () => {
-          this._panel = undefined;
-          this._displayedQuery = undefined;
-          this._panelLoaded = false;
-        },
-        null,
-        ctx.subscriptions
-      ));
-      const scriptPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath('out/resultsView.js')
-      );
-      const stylesheetPathOnDisk = vscode.Uri.file(
-        ctx.asAbsolutePath('out/view/resultsView.css')
-      );
-      panel.webview.html = getHtmlForWebview(
-        panel.webview,
-        scriptPathOnDisk,
-        [stylesheetPathOnDisk],
-        false
-      );
-      this.push(panel.webview.onDidReceiveMessage(
-        async (e) => this.handleMsgFromView(e),
-        undefined,
-        ctx.subscriptions
-      ));
+  protected async onMessage(msg: FromResultsViewMsg): Promise<void> {
+    try {
+      switch (msg.t) {
+        case 'viewLoaded':
+          this.onWebViewLoaded();
+          break;
+        case 'viewSourceFile': {
+          await jumpToLocation(msg, this.databaseManager, this.logger);
+          break;
+        }
+        case 'toggleDiagnostics': {
+          if (msg.visible) {
+            const databaseItem = this.databaseManager.findDatabaseItem(
+              Uri.parse(msg.databaseUri)
+            );
+            if (databaseItem !== undefined) {
+              await this.showResultsAsDiagnostics(
+                msg.origResultsPaths,
+                msg.metadata,
+                databaseItem
+              );
+            }
+          } else {
+            // TODO: Only clear diagnostics on the same database.
+            this._diagnosticCollection.clear();
+          }
+          break;
+        }
+        case 'changeSort':
+          await this.changeRawSortState(msg.resultSetName, msg.sortState);
+          break;
+        case 'changeInterpretedSort':
+          await this.changeInterpretedSortState(msg.sortState);
+          break;
+        case 'changePage':
+          if (msg.selectedTable === ALERTS_TABLE_NAME || msg.selectedTable === GRAPH_TABLE_NAME) {
+            await this.showPageOfInterpretedResults(msg.pageNumber);
+          }
+          else {
+            await this.showPageOfRawResults(
+              msg.selectedTable,
+              msg.pageNumber,
+              // When we are in an unsorted state, we guarantee that
+              // sortedResultsInfo doesn't have an entry for the current
+              // result set. Use this to determine whether or not we use
+              // the sorted bqrs file.
+              !!this._displayedQuery?.completedQuery.sortedResultsInfo[msg.selectedTable]
+            );
+          }
+          break;
+        case 'openFile':
+          await this.openFile(msg.filePath);
+          break;
+        default:
+          assertNever(msg);
+      }
+    } catch (e) {
+      void showAndLogErrorMessage(getErrorMessage(e), {
+        fullMessage: getErrorStack(e)
+      });
     }
-    return this._panel;
   }
 
   /**
@@ -294,85 +312,6 @@ export class InterfaceManager extends DisposableObject {
     await this.showPageOfRawResults(resultSetName, 0, true);
   }
 
-  private async handleMsgFromView(msg: FromResultsViewMsg): Promise<void> {
-    try {
-      switch (msg.t) {
-        case 'viewSourceFile': {
-          await jumpToLocation(msg, this.databaseManager, this.logger);
-          break;
-        }
-        case 'toggleDiagnostics': {
-          if (msg.visible) {
-            const databaseItem = this.databaseManager.findDatabaseItem(
-              Uri.parse(msg.databaseUri)
-            );
-            if (databaseItem !== undefined) {
-              await this.showResultsAsDiagnostics(
-                msg.origResultsPaths,
-                msg.metadata,
-                databaseItem
-              );
-            }
-          } else {
-            // TODO: Only clear diagnostics on the same database.
-            this._diagnosticCollection.clear();
-          }
-          break;
-        }
-        case 'resultViewLoaded':
-          this._panelLoaded = true;
-          this._panelLoadedCallBacks.forEach((cb) => cb());
-          this._panelLoadedCallBacks = [];
-          break;
-        case 'changeSort':
-          await this.changeRawSortState(msg.resultSetName, msg.sortState);
-          break;
-        case 'changeInterpretedSort':
-          await this.changeInterpretedSortState(msg.sortState);
-          break;
-        case 'changePage':
-          if (msg.selectedTable === ALERTS_TABLE_NAME || msg.selectedTable === GRAPH_TABLE_NAME) {
-            await this.showPageOfInterpretedResults(msg.pageNumber);
-          }
-          else {
-            await this.showPageOfRawResults(
-              msg.selectedTable,
-              msg.pageNumber,
-              // When we are in an unsorted state, we guarantee that
-              // sortedResultsInfo doesn't have an entry for the current
-              // result set. Use this to determine whether or not we use
-              // the sorted bqrs file.
-              !!this._displayedQuery?.completedQuery.sortedResultsInfo[msg.selectedTable]
-            );
-          }
-          break;
-        case 'openFile':
-          await this.openFile(msg.filePath);
-          break;
-        default:
-          assertNever(msg);
-      }
-    } catch (e) {
-      void showAndLogErrorMessage(getErrorMessage(e), {
-        fullMessage: getErrorStack(e)
-      });
-    }
-  }
-
-  postMessage(msg: IntoResultsViewMsg): Thenable<boolean> {
-    return this.getPanel().webview.postMessage(msg);
-  }
-
-  private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve) => {
-      if (this._panelLoaded) {
-        resolve();
-      } else {
-        this._panelLoadedCallBacks.push(resolve);
-      }
-    });
-  }
-
   /**
    * Show query results in webview panel.
    * @param fullQuery Evaluation info for the executed query.
@@ -387,7 +326,7 @@ export class InterfaceManager extends DisposableObject {
     forceReveal: WebviewReveal,
     shouldKeepOldResultsWhileRendering = false
   ): Promise<void> {
-    if (fullQuery.completedQuery.result.resultType !== messages.QueryResultType.SUCCESS) {
+    if (!fullQuery.completedQuery.sucessful) {
       return;
     }
 
@@ -416,7 +355,7 @@ export class InterfaceManager extends DisposableObject {
         // more asynchronous message to not so abruptly interrupt
         // user's workflow by immediately revealing the panel.
         const showButton = 'View Results';
-        const queryName = fullQuery.getShortLabel();
+        const queryName = this.labelProvider.getShortLabel(fullQuery);
         const resultPromise = vscode.window.showInformationMessage(
           `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ''
           }.`,
@@ -483,7 +422,7 @@ export class InterfaceManager extends DisposableObject {
       database: fullQuery.initialInfo.databaseInfo,
       shouldKeepOldResultsWhileRendering,
       metadata: fullQuery.completedQuery.query.metadata,
-      queryName: fullQuery.label,
+      queryName: this.labelProvider.getLabel(fullQuery),
       queryPath: fullQuery.initialInfo.queryPath
     });
   }
@@ -516,7 +455,7 @@ export class InterfaceManager extends DisposableObject {
       resultSetNames,
       pageSize: interpretedPageSize(this._interpretation),
       numPages: numInterpretedPages(this._interpretation),
-      queryName: this._displayedQuery.label,
+      queryName: this.labelProvider.getLabel(this._displayedQuery),
       queryPath: this._displayedQuery.initialInfo.queryPath
     });
   }
@@ -601,7 +540,7 @@ export class InterfaceManager extends DisposableObject {
       database: results.initialInfo.databaseInfo,
       shouldKeepOldResultsWhileRendering: false,
       metadata: results.completedQuery.query.metadata,
-      queryName: results.label,
+      queryName: this.labelProvider.getLabel(results),
       queryPath: results.initialInfo.queryPath
     });
   }

extensions/ql-vscode/src/json-rpc-server.ts

@@ -0,0 +1,30 @@
+import { Logger } from './logging';
+import * as cp from 'child_process';
+import { Disposable } from 'vscode';
+import { MessageConnection } from 'vscode-jsonrpc';
+
+
+/** A running query server process and its associated message connection. */
+export class ServerProcess implements Disposable {
+  child: cp.ChildProcess;
+  connection: MessageConnection;
+  logger: Logger;
+
+  constructor(child: cp.ChildProcess, connection: MessageConnection, private name: string, logger: Logger) {
+    this.child = child;
+    this.connection = connection;
+    this.logger = logger;
+  }
+
+  dispose(): void {
+    void this.logger.log(`Stopping ${this.name}...`);
+    this.connection.dispose();
+    this.child.stdin!.end();
+    this.child.stderr!.destroy();
+    // TODO kill the process if it doesn't terminate after a certain time limit.
+
+    // On Windows, we usually have to terminate the process before closing its stdout.
+    this.child.stdout!.destroy();
+    void this.logger.log(`Stopped ${this.name}.`);
+  }
+}

extensions/ql-vscode/src/legacy-query-server/legacyRunner.ts

@@ -0,0 +1,65 @@
+import { CancellationToken } from 'vscode';
+import { ProgressCallback } from '../commandRunner';
+import { DatabaseItem } from '../databases';
+import { Dataset, deregisterDatabases, registerDatabases } from '../pure/legacy-messages';
+import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
+import { QueryRunner } from '../queryRunner';
+import { QueryWithResults } from '../run-queries-shared';
+import { QueryServerClient } from './queryserver-client';
+import { clearCacheInDatabase, compileAndRunQueryAgainstDatabase } from './run-queries';
+import { upgradeDatabaseExplicit } from './upgrades';
+
+export class LegacyQueryRunner extends QueryRunner {
+
+
+  constructor(public readonly qs: QueryServerClient) {
+    super();
+  }
+
+  get cliServer() {
+    return this.qs.cliServer;
+  }
+
+  async restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await this.qs.restartQueryServer(progress, token);
+  }
+
+  onStart(callBack: (progress: ProgressCallback, token: CancellationToken) => Promise<void>) {
+    this.qs.onDidStartQueryServer(callBack);
+  }
+  async clearCacheInDatabase(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await clearCacheInDatabase(this.qs, dbItem, progress, token);
+  }
+  async compileAndRunQueryAgainstDatabase(dbItem: DatabaseItem, initialInfo: InitialQueryInfo, queryStorageDir: string, progress: ProgressCallback, token: CancellationToken, templates?: Record<string, string>, queryInfo?: LocalQueryInfo): Promise<QueryWithResults> {
+    return await compileAndRunQueryAgainstDatabase(this.qs.cliServer, this.qs, dbItem, initialInfo, queryStorageDir, progress, token, templates, queryInfo);
+  }
+
+  async deregisterDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
+    if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
+    }
+  }
+  async registerDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
+    if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: Dataset[] = [{
+        dbDir: dbItem.contents.datasetUri.fsPath,
+        workingSet: 'default'
+      }];
+      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
+    }
+  }
+
+  async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await upgradeDatabaseExplicit(this.qs, dbItem, progress, token);
+  }
+
+  async clearPackCache(): Promise<void> {
+    /**
+     * Nothing needs to be done
+     */
+  }
+}

extensions/ql-vscode/src/legacy-query-server/queryserver-client.ts

@@ -1,49 +1,25 @@
-import * as cp from 'child_process';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 
-import { DisposableObject } from './pure/disposable-object';
-import { Disposable, CancellationToken, commands } from 'vscode';
-import { createMessageConnection, MessageConnection, RequestType } from 'vscode-jsonrpc';
-import * as cli from './cli';
-import { QueryServerConfig } from './config';
-import { Logger, ProgressReporter } from './logging';
-import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from './pure/messages';
-import * as messages from './pure/messages';
-import { ProgressCallback, ProgressTask } from './commandRunner';
+import { DisposableObject } from '../pure/disposable-object';
+import { CancellationToken, commands } from 'vscode';
+import { createMessageConnection, RequestType } from 'vscode-jsonrpc';
+import * as cli from '../cli';
+import { QueryServerConfig } from '../config';
+import { Logger, ProgressReporter } from '../logging';
+import { completeQuery, EvaluationResult, progress, ProgressMessage, WithProgressId } from '../pure/legacy-messages';
+import * as messages from '../pure/legacy-messages';
+import { ProgressCallback, ProgressTask } from '../commandRunner';
+import { findQueryLogFile } from '../run-queries-shared';
+import { ServerProcess } from '../json-rpc-server';
+
+type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
 
 type ServerOpts = {
   logger: Logger;
   contextStoragePath: string;
 }
 
-/** A running query server process and its associated message connection. */
-class ServerProcess implements Disposable {
-  child: cp.ChildProcess;
-  connection: MessageConnection;
-  logger: Logger;
-
-  constructor(child: cp.ChildProcess, connection: MessageConnection, logger: Logger) {
-    this.child = child;
-    this.connection = connection;
-    this.logger = logger;
-  }
-
-  dispose(): void {
-    void this.logger.log('Stopping query server...');
-    this.connection.dispose();
-    this.child.stdin!.end();
-    this.child.stderr!.destroy();
-    // TODO kill the process if it doesn't terminate after a certain time limit.
-
-    // On Windows, we usually have to terminate the process before closing its stdout.
-    this.child.stdout!.destroy();
-    void this.logger.log('Stopped query server.');
-  }
-}
-
-type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
-
 /**
  * Client that manages a query server process.
  * The server process is started upon initialization and tracked during its lifetime.
@@ -200,7 +176,7 @@ export class QueryServerClient extends DisposableObject {
         callback(res);
       }
     });
-    this.serverProcess = new ServerProcess(child, connection, this.logger);
+    this.serverProcess = new ServerProcess(child, connection, 'Query server', this.logger);
     // Ensure the server process is disposed together with this client.
     this.track(this.serverProcess);
     connection.listen();
@@ -254,19 +230,3 @@ export class QueryServerClient extends DisposableObject {
     }
   }
 }
-
-export function findQueryLogFile(resultPath: string): string {
-  return path.join(resultPath, 'query.log');
-}
-
-export function findQueryEvalLogFile(resultPath: string): string {
-  return path.join(resultPath, 'evaluator-log.jsonl');
-}
-
-export function findQueryEvalLogSummaryFile(resultPath: string): string {
-  return path.join(resultPath, 'evaluator-log.summary');
-}
-
-export function findQueryEvalLogEndSummaryFile(resultPath: string): string {
-  return path.join(resultPath, 'evaluator-log-end.summary');
-}

extensions/ql-vscode/src/legacy-query-server/run-queries.ts

@@ -0,0 +1,526 @@
+import * as crypto from 'crypto';
+import * as fs from 'fs-extra';
+import * as tmp from 'tmp-promise';
+import * as path from 'path';
+import {
+  CancellationToken,
+  Uri,
+} from 'vscode';
+import { ErrorCodes, ResponseError } from 'vscode-languageclient';
+
+import * as cli from '../cli';
+import { DatabaseItem, } from '../databases';
+import {
+  getOnDiskWorkspaceFolders,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  tryGetQueryMetadata,
+  upgradesTmpDir
+} from '../helpers';
+import { ProgressCallback } from '../commandRunner';
+import { QueryMetadata } from '../pure/interface-types';
+import { logger } from '../logging';
+import * as messages from '../pure/legacy-messages';
+import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
+import * as qsClient from './queryserver-client';
+import { getErrorMessage } from '../pure/helpers-pure';
+import { compileDatabaseUpgradeSequence, upgradeDatabaseExplicit } from './upgrades';
+import { QueryEvaluationInfo, QueryWithResults } from '../run-queries-shared';
+
+/**
+ * A collection of evaluation-time information about a query,
+ * including the query itself, and where we have decided to put
+ * temporary files associated with it, such as the compiled query
+ * output and results.
+ */
+export class QueryInProgress {
+
+  public queryEvalInfo: QueryEvaluationInfo;
+  /**
+   * Note that in the {@link slurpQueryHistory} method, we create a QueryEvaluationInfo instance
+   * by explicitly setting the prototype in order to avoid calling this constructor.
+   */
+  constructor(
+    readonly querySaveDir: string,
+    readonly dbItemPath: string,
+    databaseHasMetadataFile: boolean,
+    readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
+    readonly quickEvalPosition?: messages.Position,
+    readonly metadata?: QueryMetadata,
+    readonly templates?: Record<string, string>,
+  ) {
+    this.queryEvalInfo = new QueryEvaluationInfo(querySaveDir, dbItemPath, databaseHasMetadataFile, quickEvalPosition, metadata);
+    /**/
+  }
+
+  get compiledQueryPath() {
+    return path.join(this.querySaveDir, 'compiledQuery.qlo');
+  }
+
+
+  async run(
+    qs: qsClient.QueryServerClient,
+    upgradeQlo: string | undefined,
+    availableMlModels: cli.MlModelInfo[],
+    dbItem: DatabaseItem,
+    progress: ProgressCallback,
+    token: CancellationToken,
+    queryInfo?: LocalQueryInfo,
+  ): Promise<messages.EvaluationResult> {
+    if (!dbItem.contents || dbItem.error) {
+      throw new Error('Can\'t run query on invalid database.');
+    }
+
+    let result: messages.EvaluationResult | null = null;
+
+    const callbackId = qs.registerCallback(res => {
+      result = {
+        ...res,
+        logFileLocation: this.queryEvalInfo.logPath
+      };
+    });
+
+    const availableMlModelUris: messages.MlModel[] = availableMlModels.map(model => ({ uri: Uri.file(model.path).toString(true) }));
+
+    const queryToRun: messages.QueryToRun = {
+      resultsPath: this.queryEvalInfo.resultsPaths.resultsPath,
+      qlo: Uri.file(this.compiledQueryPath).toString(),
+      compiledUpgrade: upgradeQlo && Uri.file(upgradeQlo).toString(),
+      allowUnknownTemplates: true,
+      templateValues: createSimpleTemplates(this.templates),
+      availableMlModels: availableMlModelUris,
+      id: callbackId,
+      timeoutSecs: qs.config.timeoutSecs,
+    };
+
+    const dataset: messages.Dataset = {
+      dbDir: dbItem.contents.datasetUri.fsPath,
+      workingSet: 'default'
+    };
+    if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
+      await qs.sendRequest(messages.startLog, {
+        db: dataset,
+        logPath: this.queryEvalInfo.evalLogPath,
+      });
+
+    }
+    const params: messages.EvaluateQueriesParams = {
+      db: dataset,
+      evaluateId: callbackId,
+      queries: [queryToRun],
+      stopOnError: false,
+      useSequenceHint: false
+    };
+    try {
+      await qs.sendRequest(messages.runQueries, params, token, progress);
+      if (qs.config.customLogDirectory) {
+        void showAndLogWarningMessage(
+          `Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${this.queryEvalInfo.logPath}.`
+        );
+      }
+    } finally {
+      qs.unRegisterCallback(callbackId);
+      if (queryInfo && await qs.cliServer.cliConstraints.supportsPerQueryEvalLog()) {
+        await qs.sendRequest(messages.endLog, {
+          db: dataset,
+          logPath: this.queryEvalInfo.evalLogPath,
+        });
+        if (await this.queryEvalInfo.hasEvalLog()) {
+          await this.queryEvalInfo.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
+        } else {
+          void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.queryEvalInfo.evalLogPath}.`);
+        }
+      }
+    }
+    return result || {
+      evaluationTime: 0,
+      message: 'No result from server',
+      queryId: -1,
+      runId: callbackId,
+      resultType: messages.QueryResultType.OTHER_ERROR
+    };
+  }
+
+  async compile(
+    qs: qsClient.QueryServerClient,
+    program: messages.QlProgram,
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<messages.CompilationMessage[]> {
+    let compiled: messages.CheckQueryResult | undefined;
+    try {
+      const target = this.quickEvalPosition ? {
+        quickEval: { quickEvalPos: this.quickEvalPosition }
+      } : { query: {} };
+      const params: messages.CompileQueryParams = {
+        compilationOptions: {
+          computeNoLocationUrls: true,
+          failOnWarnings: false,
+          fastCompilation: false,
+          includeDilInQlo: true,
+          localChecking: false,
+          noComputeGetUrl: false,
+          noComputeToString: false,
+          computeDefaultStrings: true,
+          emitDebugInfo: true
+        },
+        extraOptions: {
+          timeoutSecs: qs.config.timeoutSecs
+        },
+        queryToCheck: program,
+        resultPath: this.compiledQueryPath,
+        target,
+      };
+
+      compiled = await qs.sendRequest(messages.compileQuery, params, token, progress);
+    } finally {
+      void qs.logger.log(' - - - COMPILATION DONE - - - ', { additionalLogLocation: this.queryEvalInfo.logPath });
+    }
+    return (compiled?.messages || []).filter(msg => msg.severity === messages.Severity.ERROR);
+  }
+}
+
+export async function clearCacheInDatabase(
+  qs: qsClient.QueryServerClient,
+  dbItem: DatabaseItem,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<messages.ClearCacheResult> {
+  if (dbItem.contents === undefined) {
+    throw new Error('Can\'t clear the cache in an invalid database.');
+  }
+
+  const db: messages.Dataset = {
+    dbDir: dbItem.contents.datasetUri.fsPath,
+    workingSet: 'default',
+  };
+
+  const params: messages.ClearCacheParams = {
+    dryRun: false,
+    db,
+  };
+
+  return qs.sendRequest(messages.clearCache, params, token, progress);
+}
+
+
+/**
+ * Compare the dbscheme implied by the query `query` and that of the current database.
+ * - If they are compatible, do nothing.
+ * - If they are incompatible but the database can be upgraded, suggest that upgrade.
+ * - If they are incompatible and the database cannot be upgraded, throw an error.
+ */
+async function checkDbschemeCompatibility(
+  cliServer: cli.CodeQLCliServer,
+  qs: qsClient.QueryServerClient,
+  query: QueryInProgress,
+  qlProgram: messages.QlProgram,
+  dbItem: DatabaseItem,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<void> {
+  const searchPath = getOnDiskWorkspaceFolders();
+
+  if (dbItem.contents?.dbSchemeUri !== undefined) {
+    const { finalDbscheme } = await cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, false);
+    const hash = async function(filename: string): Promise<string> {
+      return crypto.createHash('sha256').update(await fs.readFile(filename)).digest('hex');
+    };
+
+    // At this point, we have learned about three dbschemes:
+
+    // the dbscheme of the actual database we're querying.
+    const dbschemeOfDb = await hash(dbItem.contents.dbSchemeUri.fsPath);
+
+    // the dbscheme of the query we're running, including the library we've resolved it to use.
+    const dbschemeOfLib = await hash(query.queryDbscheme);
+
+    // the database we're able to upgrade to
+    const upgradableTo = await hash(finalDbscheme);
+
+    if (upgradableTo != dbschemeOfLib) {
+      reportNoUpgradePath(qlProgram, query);
+    }
+
+    if (upgradableTo == dbschemeOfLib &&
+      dbschemeOfDb != dbschemeOfLib) {
+      // Try to upgrade the database
+      await upgradeDatabaseExplicit(
+        qs,
+        dbItem,
+        progress,
+        token
+      );
+    }
+  }
+}
+
+function reportNoUpgradePath(qlProgram: messages.QlProgram, query: QueryInProgress): void {
+  throw new Error(
+    `Query ${qlProgram.queryPath} expects database scheme ${query.queryDbscheme}, but the current database has a different scheme, and no database upgrades are available. The current database scheme may be newer than the CodeQL query libraries in your workspace.\n\nPlease try using a newer version of the query libraries.`
+  );
+}
+
+/**
+ * Compile a non-destructive upgrade.
+ */
+async function compileNonDestructiveUpgrade(
+  qs: qsClient.QueryServerClient,
+  upgradeTemp: tmp.DirectoryResult,
+  query: QueryInProgress,
+  qlProgram: messages.QlProgram,
+  dbItem: DatabaseItem,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<string> {
+
+  if (!dbItem?.contents?.dbSchemeUri) {
+    throw new Error('Database is invalid, and cannot be upgraded.');
+  }
+
+  // When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
+  // When packaging is not used, all dependencies are in the workspace.
+  const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
+    ? qlProgram.libraryPath
+    : getOnDiskWorkspaceFolders();
+
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
+    dbItem.contents.dbSchemeUri.fsPath,
+    upgradesPath,
+    true,
+    query.queryDbscheme
+  );
+
+  if (!matchesTarget) {
+    reportNoUpgradePath(qlProgram, query);
+  }
+  const result = await compileDatabaseUpgradeSequence(qs, dbItem, scripts, upgradeTemp, progress, token);
+  if (result.compiledUpgrade === undefined) {
+    const error = result.error || '[no error message available]';
+    throw new Error(error);
+  }
+  // We can upgrade to the actual target
+  qlProgram.dbschemePath = query.queryDbscheme;
+  // We are new enough that we will always support single file upgrades.
+  return result.compiledUpgrade;
+}
+
+
+
+export async function compileAndRunQueryAgainstDatabase(
+  cliServer: cli.CodeQLCliServer,
+  qs: qsClient.QueryServerClient,
+  dbItem: DatabaseItem,
+  initialInfo: InitialQueryInfo,
+  queryStorageDir: string,
+  progress: ProgressCallback,
+  token: CancellationToken,
+  templates?: Record<string, string>,
+  queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
+): Promise<QueryWithResults> {
+  if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
+    throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
+  }
+
+  // Get the workspace folder paths.
+  const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
+  // Figure out the library path for the query.
+  const packConfig = await cliServer.resolveLibraryPath(diskWorkspaceFolders, initialInfo.queryPath);
+
+  if (!packConfig.dbscheme) {
+    throw new Error('Could not find a database scheme for this query. Please check that you have a valid qlpack.yml file for this query, which refers to a database scheme either in the `dbscheme` field or through one of its dependencies.');
+  }
+
+  // Check whether the query has an entirely different schema from the
+  // database. (Queries that merely need the database to be upgraded
+  // won't trigger this check)
+  // This test will produce confusing results if we ever change the name of the database schema files.
+  const querySchemaName = path.basename(packConfig.dbscheme);
+  const dbSchemaName = path.basename(dbItem.contents.dbSchemeUri.fsPath);
+  if (querySchemaName != dbSchemaName) {
+    void logger.log(`Query schema was ${querySchemaName}, but database schema was ${dbSchemaName}.`);
+    throw new Error(`The query ${path.basename(initialInfo.queryPath)} cannot be run against the selected database (${dbItem.name}): their target languages are different. Please select a different database and try again.`);
+  }
+
+  const qlProgram: messages.QlProgram = {
+    // The project of the current document determines which library path
+    // we use. The `libraryPath` field in this server message is relative
+    // to the workspace root, not to the project root.
+    libraryPath: packConfig.libraryPath,
+    // Since we are compiling and running a query against a database,
+    // we use the database's DB scheme here instead of the DB scheme
+    // from the current document's project.
+    dbschemePath: dbItem.contents.dbSchemeUri.fsPath,
+    queryPath: initialInfo.queryPath
+  };
+
+  // Read the query metadata if possible, to use in the UI.
+  const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
+
+  let availableMlModels: cli.MlModelInfo[] = [];
+  if (!await cliServer.cliConstraints.supportsResolveMlModels()) {
+    void logger.log('Resolving ML models is unsupported by this version of the CLI. Running the query without any ML models.');
+  } else {
+    try {
+      availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders, initialInfo.queryPath)).models;
+      if (availableMlModels.length) {
+        void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
+      } else {
+        void logger.log('Did not find any available ML models.');
+      }
+    } catch (e) {
+      const message = `Couldn't resolve available ML models for ${qlProgram.queryPath}. Running the ` +
+        `query without any ML models: ${e}.`;
+      void showAndLogErrorMessage(message);
+    }
+  }
+
+  const hasMetadataFile = (await dbItem.hasMetadataFile());
+  const query = new QueryInProgress(
+    path.join(queryStorageDir, initialInfo.id),
+    dbItem.databaseUri.fsPath,
+    hasMetadataFile,
+    packConfig.dbscheme,
+    initialInfo.quickEvalPosition,
+    metadata,
+    templates
+  );
+  await query.queryEvalInfo.createTimestampFile();
+
+  let upgradeDir: tmp.DirectoryResult | undefined;
+  try {
+    let upgradeQlo;
+    if (await cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
+      upgradeDir = await tmp.dir({ dir: upgradesTmpDir, unsafeCleanup: true });
+      upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
+    } else {
+      await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
+    }
+    let errors;
+    try {
+      errors = await query.compile(qs, qlProgram, progress, token);
+    } catch (e) {
+      if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
+        return createSyntheticResult(query, 'Query cancelled');
+      } else {
+        throw e;
+      }
+    }
+
+    if (errors.length === 0) {
+      const result = await query.run(qs, upgradeQlo, availableMlModels, dbItem, progress, token, queryInfo);
+      if (result.resultType !== messages.QueryResultType.SUCCESS) {
+        const message = result.message || 'Failed to run query';
+        void logger.log(message);
+        void showAndLogErrorMessage(message);
+      }
+      const message = formatLegacyMessage(result);
+
+      return {
+        query: query.queryEvalInfo,
+        message,
+        result,
+        sucessful: result.resultType == messages.QueryResultType.SUCCESS,
+        logFileLocation: result.logFileLocation,
+        dispose: () => {
+          qs.logger.removeAdditionalLogLocation(result.logFileLocation);
+        }
+      };
+    } else {
+      // Error dialogs are limited in size and scrollability,
+      // so we include a general description of the problem,
+      // and direct the user to the output window for the detailed compilation messages.
+      // However we don't show quick eval errors there so we need to display them anyway.
+      void qs.logger.log(
+        `Failed to compile query ${initialInfo.queryPath} against database scheme ${qlProgram.dbschemePath}:`,
+        { additionalLogLocation: query.queryEvalInfo.logPath }
+      );
+
+      const formattedMessages: string[] = [];
+
+      for (const error of errors) {
+        const message = error.message || '[no error message available]';
+        const formatted = `ERROR: ${message} (${error.position.fileName}:${error.position.line}:${error.position.column}:${error.position.endLine}:${error.position.endColumn})`;
+        formattedMessages.push(formatted);
+        void qs.logger.log(formatted, { additionalLogLocation: query.queryEvalInfo.logPath });
+      }
+      if (initialInfo.isQuickEval && formattedMessages.length <= 2) {
+        // If there are more than 2 error messages, they will not be displayed well in a popup
+        // and will be trimmed by the function displaying the error popup. Accordingly, we only
+        // try to show the errors if there are 2 or less, otherwise we direct the user to the log.
+        void showAndLogErrorMessage('Quick evaluation compilation failed: ' + formattedMessages.join('\n'));
+      } else {
+        void showAndLogErrorMessage((initialInfo.isQuickEval ? 'Quick evaluation' : 'Query') + compilationFailedErrorTail);
+      }
+      return createSyntheticResult(query, 'Query had compilation errors');
+    }
+  } finally {
+    try {
+      await upgradeDir?.cleanup();
+    } catch (e) {
+      void qs.logger.log(
+        `Could not clean up the upgrades dir. Reason: ${getErrorMessage(e)}`,
+        { additionalLogLocation: query.queryEvalInfo.logPath }
+      );
+    }
+  }
+}
+
+
+const compilationFailedErrorTail = ' compilation failed. Please make sure there are no errors in the query, the database is up to date,' +
+  ' and the query and database use the same target language. For more details on the error, go to View > Output,' +
+  ' and choose CodeQL Query Server from the dropdown.';
+
+export function formatLegacyMessage(result: messages.EvaluationResult) {
+  switch (result.resultType) {
+    case messages.QueryResultType.CANCELLATION:
+      return `cancelled after ${Math.round(result.evaluationTime / 1000)} seconds`;
+    case messages.QueryResultType.OOM:
+      return 'out of memory';
+    case messages.QueryResultType.SUCCESS:
+      return `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
+    case messages.QueryResultType.TIMEOUT:
+      return `timed out after ${Math.round(result.evaluationTime / 1000)} seconds`;
+    case messages.QueryResultType.OTHER_ERROR:
+    default:
+      return result.message ? `failed: ${result.message}` : 'failed';
+  }
+}
+
+/**
+ * Create a synthetic result for a query that failed to compile.
+ */
+function createSyntheticResult(
+  query: QueryInProgress,
+  message: string,
+): QueryWithResults {
+  return {
+    query: query.queryEvalInfo,
+    message,
+    result:{
+      evaluationTime:0,
+      queryId: 0,
+      resultType: messages.QueryResultType.OTHER_ERROR,
+      message,
+      runId: 0,
+    },
+    sucessful: false,
+    dispose: () => { /**/ },
+  };
+}
+
+
+function createSimpleTemplates(templates: Record<string, string> | undefined): messages.TemplateDefinitions | undefined {
+  if (!templates) {
+    return undefined;
+  }
+  const result: messages.TemplateDefinitions = {};
+  for (const key of Object.keys(templates)) {
+    result[key] = {
+      values: {
+        tuples: [[{ stringValue: templates[key] }]]
+      }
+    };
+  }
+  return result;
+}

extensions/ql-vscode/src/legacy-query-server/upgrades.ts

@@ -1,13 +1,12 @@
 import * as vscode from 'vscode';
-import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from './helpers';
-import { ProgressCallback, UserCancellationException } from './commandRunner';
-import { logger } from './logging';
-import * as messages from './pure/messages';
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tmpDir } from '../helpers';
+import { ProgressCallback, UserCancellationException } from '../commandRunner';
+import { logger } from '../logging';
+import * as messages from '../pure/legacy-messages';
 import * as qsClient from './queryserver-client';
 import * as tmp from 'tmp-promise';
 import * as path from 'path';
-import * as semver from 'semver';
-import { DatabaseItem } from './databases';
+import { DatabaseItem } from '../databases';
 
 /**
  * Maximum number of lines to include from database upgrade message,
@@ -16,17 +15,6 @@ import { DatabaseItem } from './databases';
  */
 const MAX_UPGRADE_MESSAGE_LINES = 10;
 
-/**
- * Check that we support non-destructive upgrades.
- *
- * This requires 3 features. The ability to compile an upgrade sequence; The ability to
- * run a non-destructive upgrades as a query; the ability to specify a target when
- * resolving upgrades. We check for a version of codeql that has all three features.
- */
-export async function hasNondestructiveUpgradeCapabilities(qs: qsClient.QueryServerClient): Promise<boolean> {
-  return semver.gte(await qs.cliServer.getVersion(), '2.4.2');
-}
-
 
 /**
  * Compile a database upgrade sequence.
@@ -43,7 +31,7 @@ export async function compileDatabaseUpgradeSequence(
   if (dbItem.contents === undefined || dbItem.contents.dbSchemeUri === undefined) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
-  if (!await hasNondestructiveUpgradeCapabilities(qs)) {
+  if (!await qs.cliServer.cliConstraints.supportsNonDestructiveUpgrades()) {
     throw new Error('The version of codeql is too old to run non-destructive upgrades.');
   }
   // If possible just compile the upgrade sequence
@@ -205,7 +193,14 @@ export async function upgradeDatabaseExplicit(
       void qs.logger.log('Running the following database upgrade:');
 
       getUpgradeDescriptions(compileUpgradeResult.compiledUpgrades).map(s => s.description).join('\n');
-      return await runDatabaseUpgrade(qs, dbItem, compileUpgradeResult.compiledUpgrades, progress, token);
+      const result = await runDatabaseUpgrade(qs, dbItem, compileUpgradeResult.compiledUpgrades, progress, token);
+
+      // TODO Can remove the next lines when https://github.com/github/codeql-team/issues/1241 is fixed
+      // restart the query server to avoid a bug in the CLI where the upgrade is applied, but the old dbscheme
+      // is still cached in memory.
+
+      await qs.restartQueryServer(progress, token);
+      return result;
     }
     catch (e) {
       void showAndLogErrorMessage(`Database upgrade failed: ${e}`);

extensions/ql-vscode/src/log-insights/join-order.ts

@@ -0,0 +1,460 @@
+import * as I from 'immutable';
+import { EvaluationLogProblemReporter, EvaluationLogScanner, EvaluationLogScannerProvider } from './log-scanner';
+import { InLayer, ComputeRecursive, SummaryEvent, PipelineRun, ComputeSimple } from './log-summary';
+
+const DEFAULT_WARNING_THRESHOLD = 50;
+
+/**
+ * Like `max`, but returns 0 if no meaningful maximum can be computed.
+ */
+function safeMax(it?: Iterable<number>) {
+  const m = Math.max(...(it || []));
+  return Number.isFinite(m) ? m : 0;
+}
+
+/**
+ * Compute a key for the maps that that is sent to report generation.
+ * Should only be used on events that are known to define queryCausingWork.
+ */
+function makeKey(
+  queryCausingWork: string | undefined,
+  predicate: string,
+  suffix = ''
+): string {
+  if (queryCausingWork === undefined) {
+    throw new Error(
+      'queryCausingWork was not defined on an event we expected it to be defined for!'
+    );
+  }
+  return `${queryCausingWork}:${predicate}${suffix ? ' ' + suffix : ''}`;
+}
+
+const DEPENDENT_PREDICATES_REGEXP = (() => {
+  const regexps = [
+    // SCAN id
+    String.raw`SCAN\s+([0-9a-zA-Z:#_]+)\s`,
+    // JOIN id WITH id
+    String.raw`JOIN\s+([0-9a-zA-Z:#_]+)\s+WITH\s+([0-9a-zA-Z:#_]+)\s`,
+    // AGGREGATE id, id
+    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+)\s*,\s+([0-9a-zA-Z:#_]+)`,
+    // id AND NOT id
+    String.raw`([0-9a-zA-Z:#_]+)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+)`,
+    // INVOKE HIGHER-ORDER RELATION rel ON <id, ..., id>
+    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+)((?:,[0-9a-zA-Z:#_<>]+)*)>`,
+    // SELECT id
+    String.raw`SELECT\s+([0-9a-zA-Z:#_]+)`
+  ];
+  return new RegExp(
+    `${String.raw`\{[0-9]+\}\s+[0-9a-zA-Z]+\s=\s(?:` + regexps.join('|')})`
+  );
+})();
+
+function getDependentPredicates(operations: string[]): I.List<string> {
+  return I.List(operations).flatMap(operation => {
+    const matches = DEPENDENT_PREDICATES_REGEXP.exec(operation.trim());
+    if (matches !== null) {
+      return I.List(matches)
+        .rest() // Skip the first group as it's just the entire string
+        .filter(x => !!x && !x.match('r[0-9]+|PRIMITIVE')) // Only keep the references to predicates.
+        .flatMap(x => x.split(',')) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
+        .filter(x => !!x); // Remove empty strings
+    } else {
+      return I.List();
+    }
+  });
+}
+
+function getMainHash(event: InLayer | ComputeRecursive): string {
+  switch (event.evaluationStrategy) {
+    case 'IN_LAYER':
+      return event.mainHash;
+    case 'COMPUTE_RECURSIVE':
+      return event.raHash;
+  }
+}
+
+/**
+ * Sum arrays a and b element-wise. The shorter array is padded with 0s if the arrays are not the same length.
+ */
+function pointwiseSum(a: Int32Array, b: Int32Array, problemReporter: EvaluationLogProblemReporter): Int32Array {
+  function reportIfInconsistent(ai: number, bi: number) {
+    if (ai === -1 && bi !== -1) {
+      problemReporter.log(
+        `Operation was not evaluated in the first pipeline, but it was evaluated in the accumulated pipeline (with tuple count ${bi}).`
+      );
+    }
+    if (ai !== -1 && bi === -1) {
+      problemReporter.log(
+        `Operation was evaluated in the first pipeline (with tuple count ${ai}), but it was not evaluated in the accumulated pipeline.`
+      );
+    }
+  }
+
+  const length = Math.max(a.length, b.length);
+  const result = new Int32Array(length);
+  for (let i = 0; i < length; i++) {
+    const ai = a[i] || 0;
+    const bi = b[i] || 0;
+    // -1 is used to represent the absence of a tuple count for a line in the pretty-printed RA (e.g. an empty line), so we ignore those.
+    if (i < a.length && i < b.length && (ai === -1 || bi === -1)) {
+      result[i] = -1;
+      reportIfInconsistent(ai, bi);
+    } else {
+      result[i] = ai + bi;
+    }
+  }
+  return result;
+}
+
+function pushValue<K, V>(m: Map<K, V[]>, k: K, v: V) {
+  if (!m.has(k)) {
+    m.set(k, []);
+  }
+  m.get(k)!.push(v);
+  return m;
+}
+
+function computeJoinOrderBadness(
+  maxTupleCount: number,
+  maxDependentPredicateSize: number,
+  resultSize: number
+): number {
+  return maxTupleCount / Math.max(maxDependentPredicateSize, resultSize);
+}
+
+/**
+ * A bucket contains the pointwise sum of the tuple counts, result sizes and dependent predicate sizes
+ * For each (predicate, order) in an SCC, we will compute a bucket.
+ */
+interface Bucket {
+  tupleCounts: Int32Array;
+  resultSize: number;
+  dependentPredicateSizes: I.Map<string, number>;
+}
+
+class JoinOrderScanner implements EvaluationLogScanner {
+  // Map a predicate hash to its result size
+  private readonly predicateSizes = new Map<string, number>();
+  private readonly layerEvents = new Map<string, (ComputeRecursive | InLayer)[]>();
+  // Map a key of the form 'query-with-demand : predicate name' to its badness input.
+  private readonly maxTupleCountMap = new Map<string, number[]>();
+  private readonly resultSizeMap = new Map<string, number[]>();
+  private readonly maxDependentPredicateSizeMap = new Map<string, number[]>();
+  private readonly joinOrderMetricMap = new Map<string, number>();
+
+  constructor(
+    private readonly problemReporter: EvaluationLogProblemReporter,
+    private readonly warningThreshold: number) {
+  }
+
+  public onEvent(event: SummaryEvent): void {
+    if (
+      event.completionType !== undefined &&
+      event.completionType !== 'SUCCESS'
+    ) {
+      return; // Skip any evaluation that wasn't successful
+    }
+
+    this.recordPredicateSizes(event);
+    this.computeBadnessMetric(event);
+  }
+
+  public onDone(): void {
+    void this;
+  }
+
+  private recordPredicateSizes(event: SummaryEvent): void {
+    switch (event.evaluationStrategy) {
+      case 'EXTENSIONAL':
+      case 'COMPUTED_EXTENSIONAL':
+      case 'COMPUTE_SIMPLE':
+      case 'CACHACA':
+      case 'CACHE_HIT': {
+        this.predicateSizes.set(event.raHash, event.resultSize);
+        break;
+      }
+      case 'SENTINEL_EMPTY': {
+        this.predicateSizes.set(event.raHash, 0);
+        break;
+      }
+      case 'COMPUTE_RECURSIVE':
+      case 'IN_LAYER': {
+        this.predicateSizes.set(event.raHash, event.resultSize);
+        // layerEvents are indexed by the mainHash.
+        const hash = getMainHash(event);
+        if (!this.layerEvents.has(hash)) {
+          this.layerEvents.set(hash, []);
+        }
+        this.layerEvents.get(hash)!.push(event);
+        break;
+      }
+    }
+  }
+
+  private reportProblemIfNecessary(event: SummaryEvent, iteration: number, metric: number): void {
+    if (metric >= this.warningThreshold) {
+      this.problemReporter.reportProblem(event.predicateName, event.raHash, iteration,
+        `Relation '${event.predicateName}' has an inefficient join order. Its join order metric is ${metric.toFixed(2)}, which is larger than the threshold of ${this.warningThreshold.toFixed(2)}.`);
+    }
+  }
+
+  private computeBadnessMetric(event: SummaryEvent): void {
+    if (
+      event.completionType !== undefined &&
+      event.completionType !== 'SUCCESS'
+    ) {
+      return; // Skip any evaluation that wasn't successful
+    }
+    switch (event.evaluationStrategy) {
+      case 'COMPUTE_SIMPLE': {
+        if (!event.pipelineRuns) {
+          // skip if the optional pipelineRuns field is not present.
+          break;
+        }
+        // Compute the badness metric for a non-recursive predicate. The metric in this case is defined as:
+        // badness = (max tuple count in the pipeline) / (largest predicate this pipeline depends on)
+        const key = makeKey(event.queryCausingWork, event.predicateName);
+        const resultSize = event.resultSize;
+
+        // There is only one entry in `pipelineRuns` if it's a non-recursive predicate.
+        const { maxTupleCount, maxDependentPredicateSize } =
+          this.badnessInputsForNonRecursiveDelta(event.pipelineRuns[0], event);
+
+        if (maxDependentPredicateSize > 0) {
+          pushValue(this.maxTupleCountMap, key, maxTupleCount);
+          pushValue(this.resultSizeMap, key, resultSize);
+          pushValue(
+            this.maxDependentPredicateSizeMap,
+            key,
+            maxDependentPredicateSize
+          );
+          const metric = computeJoinOrderBadness(maxTupleCount, maxDependentPredicateSize, resultSize!);
+          this.joinOrderMetricMap.set(key, metric);
+          this.reportProblemIfNecessary(event, 0, metric);
+        }
+        break;
+      }
+
+      case 'COMPUTE_RECURSIVE': {
+        // Compute the badness metric for a recursive predicate for each ordering.
+        const sccMetricInput = this.badnessInputsForRecursiveDelta(event);
+        // Loop through each predicate in the SCC
+        sccMetricInput.forEach((buckets, predicate) => {
+          // Loop through each ordering of the predicate
+          buckets.forEach((bucket, raReference) => {
+            // Format the key as demanding-query:name (ordering)
+            const key = makeKey(
+              event.queryCausingWork,
+              predicate,
+              `(${raReference})`
+            );
+            const maxTupleCount = Math.max(...bucket.tupleCounts);
+            const resultSize = bucket.resultSize;
+            const maxDependentPredicateSize = Math.max(
+              ...bucket.dependentPredicateSizes.values()
+            );
+
+            if (maxDependentPredicateSize > 0) {
+              pushValue(this.maxTupleCountMap, key, maxTupleCount);
+              pushValue(this.resultSizeMap, key, resultSize);
+              pushValue(
+                this.maxDependentPredicateSizeMap,
+                key,
+                maxDependentPredicateSize
+              );
+              const metric = computeJoinOrderBadness(maxTupleCount, maxDependentPredicateSize, resultSize);
+              const oldMetric = this.joinOrderMetricMap.get(key);
+              if ((oldMetric === undefined) || (metric > oldMetric)) {
+                this.joinOrderMetricMap.set(key, metric);
+              }
+            }
+          });
+        });
+        break;
+      }
+    }
+  }
+
+  /**
+   * Iterate through an SCC with main node `event`.
+   */
+  private iterateSCC(
+    event: ComputeRecursive,
+    func: (
+      inLayerEvent: ComputeRecursive | InLayer,
+      run: PipelineRun,
+      iteration: number
+    ) => void
+  ): void {
+    const sccEvents = this.layerEvents.get(event.raHash)!;
+    const nextPipeline: number[] = new Array(sccEvents.length).fill(0);
+
+    const maxIteration = Math.max(
+      ...sccEvents.map(e => e.predicateIterationMillis.length)
+    );
+
+    for (let iteration = 0; iteration < maxIteration; ++iteration) {
+      // Loop through each predicate in this iteration
+      for (let predicate = 0; predicate < sccEvents.length; ++predicate) {
+        const inLayerEvent = sccEvents[predicate];
+        const iterationTime =
+          inLayerEvent.predicateIterationMillis.length <= iteration
+            ? -1
+            : inLayerEvent.predicateIterationMillis[iteration];
+        if (iterationTime != -1) {
+          const run: PipelineRun =
+            inLayerEvent.pipelineRuns[nextPipeline[predicate]++];
+          func(inLayerEvent, run, iteration);
+        }
+      }
+    }
+  }
+
+  /**
+   * Compute the maximum tuple count and maximum dependent predicate size for a non-recursive pipeline
+   */
+  private badnessInputsForNonRecursiveDelta(
+    pipelineRun: PipelineRun,
+    event: ComputeSimple
+  ): { maxTupleCount: number; maxDependentPredicateSize: number } {
+    const dependentPredicateSizes = Object.values(event.dependencies).map(hash =>
+      this.predicateSizes.get(hash) ?? 0  // Should always be present, but zero is a safe default.
+    );
+    const maxDependentPredicateSize = safeMax(dependentPredicateSizes);
+    return {
+      maxTupleCount: safeMax(pipelineRun.counts),
+      maxDependentPredicateSize: maxDependentPredicateSize
+    };
+  }
+
+  private prevDeltaSizes(event: ComputeRecursive, predicate: string, i: number) {
+    // If an iteration isn't present in the map it means it was skipped because the optimizer
+    // inferred that it was empty. So its size is 0.
+    return this.curDeltaSizes(event, predicate, i - 1);
+  }
+
+  private curDeltaSizes(event: ComputeRecursive, predicate: string, i: number) {
+    // If an iteration isn't present in the map it means it was skipped because the optimizer
+    // inferred that it was empty. So its size is 0.
+    return (
+      this.layerEvents.get(event.raHash)?.find(x => x.predicateName === predicate)?.deltaSizes[i] ?? 0
+    );
+  }
+
+  /**
+   * Compute the metric dependent predicate sizes and the result size for a predicate in an SCC.
+   */
+  private badnessInputsForLayer(
+    event: ComputeRecursive,
+    inLayerEvent: InLayer | ComputeRecursive,
+    raReference: string,
+    iteration: number
+  ) {
+    const dependentPredicates = getDependentPredicates(
+      inLayerEvent.ra[raReference]
+    );
+    let dependentPredicateSizes: I.Map<string, number>;
+    // We treat the base case as a non-recursive pipeline. In that case, the dependent predicates are
+    // the dependencies of the base case and the cur_deltas.
+    if (raReference === 'base') {
+      dependentPredicateSizes = I.Map(
+        dependentPredicates.map((pred): [string, number] => {
+          // A base case cannot contain a `prev_delta`, but it can contain a `cur_delta`.
+          let size = 0;
+          if (pred.endsWith('#cur_delta')) {
+            size = this.curDeltaSizes(
+              event,
+              pred.slice(0, -'#cur_delta'.length),
+              iteration
+            );
+          } else {
+            const hash = event.dependencies[pred];
+            size = this.predicateSizes.get(hash)!;
+          }
+          return [pred, size];
+        })
+      );
+    } else {
+      // It's a non-base case in a recursive pipeline. In that case, the dependent predicates are
+      // only the prev_deltas.
+      dependentPredicateSizes = I.Map(
+        dependentPredicates
+          .flatMap(pred => {
+            // If it's actually a prev_delta
+            if (pred.endsWith('#prev_delta')) {
+              // Return the predicate without the #prev_delta suffix.
+              return [pred.slice(0, -'#prev_delta'.length)];
+            } else {
+              // Not a recursive delta. Skip it.
+              return [];
+            }
+          })
+          .map((prev): [string, number] => {
+            const size = this.prevDeltaSizes(event, prev, iteration);
+            return [prev, size];
+          })
+      );
+    }
+
+    const deltaSize = inLayerEvent.deltaSizes[iteration];
+    return { dependentPredicateSizes, deltaSize };
+  }
+
+  /**
+   * Compute the metric input for all the events in a SCC that starts with main node `event`
+   */
+  private badnessInputsForRecursiveDelta(event: ComputeRecursive): Map<string, Map<string, Bucket>> {
+    // nameToOrderToBucket : predicate name -> ordering (i.e., standard, order_500000, etc.) -> bucket
+    const nameToOrderToBucket = new Map<string, Map<string, Bucket>>();
+
+    // Iterate through the SCC and compute the metric inputs
+    this.iterateSCC(event, (inLayerEvent, run, iteration) => {
+      const raReference = run.raReference;
+      const predicateName = inLayerEvent.predicateName;
+      if (!nameToOrderToBucket.has(predicateName)) {
+        nameToOrderToBucket.set(predicateName, new Map());
+      }
+      const orderTobucket = nameToOrderToBucket.get(predicateName)!;
+      if (!orderTobucket.has(raReference)) {
+        orderTobucket.set(raReference, {
+          tupleCounts: new Int32Array(0),
+          resultSize: 0,
+          dependentPredicateSizes: I.Map()
+        });
+      }
+
+      const { dependentPredicateSizes, deltaSize } = this.badnessInputsForLayer(
+        event,
+        inLayerEvent,
+        raReference,
+        iteration
+      );
+
+      const bucket = orderTobucket.get(raReference)!;
+      // Pointwise sum the tuple counts
+      const newTupleCounts = pointwiseSum(
+        bucket.tupleCounts,
+        new Int32Array(run.counts),
+        this.problemReporter
+      );
+      const resultSize = bucket.resultSize + deltaSize;
+      // Pointwise sum the deltas.
+      const newDependentPredicateSizes = bucket.dependentPredicateSizes.mergeWith(
+        (oldSize, newSize) => oldSize + newSize,
+        dependentPredicateSizes
+      );
+      orderTobucket.set(raReference, {
+        tupleCounts: newTupleCounts,
+        resultSize: resultSize,
+        dependentPredicateSizes: newDependentPredicateSizes
+      });
+    });
+    return nameToOrderToBucket;
+  }
+}
+
+export class JoinOrderScannerProvider implements EvaluationLogScannerProvider {
+  public createScanner(problemReporter: EvaluationLogProblemReporter): EvaluationLogScanner {
+    return new JoinOrderScanner(problemReporter, DEFAULT_WARNING_THRESHOLD);
+  }
+}

extensions/ql-vscode/src/log-insights/jsonl-reader.ts

@@ -0,0 +1,23 @@
+import * as fs from 'fs-extra';
+
+/**
+ * Read a file consisting of multiple JSON objects. Each object is separated from the previous one
+ * by a double newline sequence. This is basically a more human-readable form of JSONL.
+ *
+ * The current implementation reads the entire text of the document into memory, but in the future
+ * it will stream the document to improve the performance with large documents.
+ *
+ * @param path The path to the file.
+ * @param handler Callback to be invoked for each top-level JSON object in order.
+ */
+export async function readJsonlFile(path: string, handler: (value: any) => Promise<void>): Promise<void> {
+  const logSummary = await fs.readFile(path, 'utf-8');
+
+  // Remove newline delimiters because summary is in .jsonl format.
+  const jsonSummaryObjects: string[] = logSummary.split(/\r?\n\r?\n/g);
+
+  for (const obj of jsonSummaryObjects) {
+    const jsonObj = JSON.parse(obj);
+    await handler(jsonObj);
+  }
+}

extensions/ql-vscode/src/log-insights/log-scanner-service.ts

@@ -0,0 +1,109 @@
+import { Diagnostic, DiagnosticSeverity, languages, Range, Uri } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+import { QueryHistoryManager } from '../query-history';
+import { QueryHistoryInfo } from '../query-results';
+import { EvaluationLogProblemReporter, EvaluationLogScannerSet } from './log-scanner';
+import { PipelineInfo, SummarySymbols } from './summary-parser';
+import * as fs from 'fs-extra';
+import { logger } from '../logging';
+
+/**
+ * Compute the key used to find a predicate in the summary symbols.
+ * @param name The name of the predicate.
+ * @param raHash The RA hash of the predicate.
+ * @returns The key of the predicate, consisting of `name@shortHash`, where `shortHash` is the first
+ * eight characters of `raHash`.
+ */
+function predicateSymbolKey(name: string, raHash: string): string {
+  return `${name}@${raHash.substring(0, 8)}`;
+}
+
+/**
+ * Implementation of `EvaluationLogProblemReporter` that generates `Diagnostic` objects to display
+ * in the VS Code "Problems" view.
+ */
+class ProblemReporter implements EvaluationLogProblemReporter {
+  public readonly diagnostics: Diagnostic[] = [];
+
+  constructor(private readonly symbols: SummarySymbols | undefined) {
+  }
+
+  public reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void {
+    const nameWithHash = predicateSymbolKey(predicateName, raHash);
+    const predicateSymbol = this.symbols?.predicates[nameWithHash];
+    let predicateInfo: PipelineInfo | undefined = undefined;
+    if (predicateSymbol !== undefined) {
+      predicateInfo = predicateSymbol.iterations[iteration];
+    }
+    if (predicateInfo !== undefined) {
+      const range = new Range(predicateInfo.raStartLine, 0, predicateInfo.raEndLine + 1, 0);
+      this.diagnostics.push(new Diagnostic(range, message, DiagnosticSeverity.Error));
+    }
+  }
+
+  public log(message: string): void {
+    void logger.log(message);
+  }
+}
+
+export class LogScannerService extends DisposableObject {
+  public readonly scanners = new EvaluationLogScannerSet();
+  private readonly diagnosticCollection = this.push(languages.createDiagnosticCollection('ql-eval-log'));
+  private currentItem: QueryHistoryInfo | undefined = undefined;
+
+  constructor(qhm: QueryHistoryManager) {
+    super();
+
+    this.push(qhm.onDidChangeCurrentQueryItem(async (item) => {
+      if (item !== this.currentItem) {
+        this.currentItem = item;
+        await this.scanEvalLog(item);
+      }
+    }));
+
+    this.push(qhm.onDidCompleteQuery(async (item) => {
+      if (item === this.currentItem) {
+        await this.scanEvalLog(item);
+      }
+    }));
+  }
+
+  /**
+   * Scan the evaluation log for a query, and report any diagnostics.
+   *
+   * @param query The query whose log is to be scanned.
+   */
+  public async scanEvalLog(
+    query: QueryHistoryInfo | undefined
+  ): Promise<void> {
+    this.diagnosticCollection.clear();
+
+    if ((query?.t !== 'local')
+      || (query.evalLogSummaryLocation === undefined)
+      || (query.jsonEvalLogSummaryLocation === undefined)) {
+      return;
+    }
+
+    const diagnostics = await this.scanLog(query.jsonEvalLogSummaryLocation, query.evalLogSummarySymbolsLocation);
+    const uri = Uri.file(query.evalLogSummaryLocation);
+    this.diagnosticCollection.set(uri, diagnostics);
+  }
+
+  /**
+   * Scan the evaluator summary log for problems, using the scanners for all registered providers.
+   * @param jsonSummaryLocation The file path of the JSON summary log.
+   * @param symbolsLocation The file path of the symbols file for the human-readable log summary.
+   * @returns An array of `Diagnostic`s representing the problems found by scanners.
+   */
+  private async scanLog(jsonSummaryLocation: string, symbolsLocation: string | undefined): Promise<Diagnostic[]> {
+    let symbols: SummarySymbols | undefined = undefined;
+    if (symbolsLocation !== undefined) {
+      symbols = JSON.parse(await fs.readFile(symbolsLocation, { encoding: 'utf-8' }));
+    }
+    const problemReporter = new ProblemReporter(symbols);
+
+    await this.scanners.scanLog(jsonSummaryLocation, problemReporter);
+
+    return problemReporter.diagnostics;
+  }
+}

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -0,0 +1,103 @@
+import { SummaryEvent } from './log-summary';
+import { readJsonlFile } from './jsonl-reader';
+
+/**
+ * Callback interface used to report diagnostics from a log scanner.
+ */
+export interface EvaluationLogProblemReporter {
+  /**
+   * Report a potential problem detected in the evaluation log.
+   *
+   * @param predicateName The mangled name of the predicate with the problem.
+   * @param raHash The RA hash of the predicate with the problem.
+   * @param iteration The iteration number with the problem. For a non-recursive predicate, this
+   * must be zero.
+   * @param message The problem message.
+   */
+  reportProblem(predicateName: string, raHash: string, iteration: number, message: string): void;
+
+  /**
+   * Log a message about a problem in the implementation of the scanner. These will typically be
+   * displayed separate from any problems reported via `reportProblem()`.
+   */
+  log(message: string): void;
+}
+
+/**
+ * Interface implemented by a log scanner. Instances are created via
+ * `EvaluationLogScannerProvider.createScanner()`.
+ */
+export interface EvaluationLogScanner {
+  /**
+   * Called for each event in the log summary, in order. The implementation can report problems via
+   * the `EvaluationLogProblemReporter` interface that was supplied to `createScanner()`.
+   * @param event The log summary event.
+   */
+  onEvent(event: SummaryEvent): void;
+  /**
+   * Called after all events in the log summary have been processed. The implementation can report
+   * problems via the `EvaluationLogProblemReporter` interface that was supplied to
+   * `createScanner()`.
+   */
+  onDone(): void;
+}
+
+/**
+ * A factory for log scanners. When a log is to be scanned, all registered
+ * `EvaluationLogScannerProviders` will be asked to create a new instance of `EvaluationLogScanner`
+ * to do the scanning.
+ */
+export interface EvaluationLogScannerProvider {
+  /**
+   * Create a new instance of `EvaluationLogScanner` to scan a single summary log.
+   * @param problemReporter Callback interface for reporting any problems discovered.
+   */
+  createScanner(problemReporter: EvaluationLogProblemReporter): EvaluationLogScanner;
+}
+
+/**
+ * Same as VSCode's `Disposable`, but avoids a dependency on VS Code.
+ */
+export interface Disposable {
+  dispose(): void;
+}
+
+export class EvaluationLogScannerSet {
+  private readonly scannerProviders = new Map<number, EvaluationLogScannerProvider>();
+  private nextScannerProviderId = 0;
+
+  /**
+   * Register a provider that can create instances of `EvaluationLogScanner` to scan evaluation logs
+   * for problems.
+   * @param provider The provider.
+   * @returns A `Disposable` that, when disposed, will unregister the provider.
+   */
+  public registerLogScannerProvider(provider: EvaluationLogScannerProvider): Disposable {
+    const id = this.nextScannerProviderId;
+    this.nextScannerProviderId++;
+
+    this.scannerProviders.set(id, provider);
+    return {
+      dispose: () => {
+        this.scannerProviders.delete(id);
+      }
+    };
+  }
+
+  /**
+   * Scan the evaluator summary log for problems, using the scanners for all registered providers.
+   * @param jsonSummaryLocation The file path of the JSON summary log.
+   * @param problemReporter Callback interface for reporting any problems discovered.
+   */
+  public async scanLog(jsonSummaryLocation: string, problemReporter: EvaluationLogProblemReporter): Promise<void> {
+    const scanners = [...this.scannerProviders.values()].map(p => p.createScanner(problemReporter));
+
+    await readJsonlFile(jsonSummaryLocation, async obj => {
+      scanners.forEach(scanner => {
+        scanner.onEvent(obj);
+      });
+    });
+
+    scanners.forEach(scanner => scanner.onDone());
+  }
+}

extensions/ql-vscode/src/log-insights/log-summary.ts

@@ -0,0 +1,93 @@
+export interface PipelineRun {
+  raReference: string;
+  counts: number[];
+  duplicationPercentages: number[];
+}
+
+export interface Ra {
+  [key: string]: string[];
+}
+
+export type EvaluationStrategy =
+  'COMPUTE_SIMPLE' |
+  'COMPUTE_RECURSIVE' |
+  'IN_LAYER' |
+  'COMPUTED_EXTENSIONAL' |
+  'EXTENSIONAL' |
+  'SENTINEL_EMPTY' |
+  'CACHACA' |
+  'CACHE_HIT';
+
+interface SummaryEventBase {
+  evaluationStrategy: EvaluationStrategy;
+  predicateName: string;
+  raHash: string;
+  appearsAs: { [key: string]: { [key: string]: number[] } };
+  completionType?: string;
+}
+
+interface ResultEventBase extends SummaryEventBase {
+  resultSize: number;
+}
+
+export interface ComputeSimple extends ResultEventBase {
+  evaluationStrategy: 'COMPUTE_SIMPLE';
+  ra: Ra;
+  pipelineRuns?: [PipelineRun];
+  queryCausingWork?: string;
+  dependencies: { [key: string]: string };
+}
+
+export interface ComputeRecursive extends ResultEventBase {
+  evaluationStrategy: 'COMPUTE_RECURSIVE';
+  deltaSizes: number[];
+  ra: Ra;
+  pipelineRuns: PipelineRun[];
+  queryCausingWork?: string;
+  dependencies: { [key: string]: string };
+  predicateIterationMillis: number[];
+}
+
+export interface InLayer extends ResultEventBase {
+  evaluationStrategy: 'IN_LAYER';
+  deltaSizes: number[];
+  ra: Ra;
+  pipelineRuns: PipelineRun[];
+  queryCausingWork?: string;
+  mainHash: string;
+  predicateIterationMillis: number[];
+}
+
+export interface ComputedExtensional extends ResultEventBase {
+  evaluationStrategy: 'COMPUTED_EXTENSIONAL';
+  queryCausingWork?: string;
+}
+
+export interface NonComputedExtensional extends ResultEventBase {
+  evaluationStrategy: 'EXTENSIONAL';
+  queryCausingWork?: string;
+}
+
+export interface SentinelEmpty extends SummaryEventBase {
+  evaluationStrategy: 'SENTINEL_EMPTY';
+  sentinelRaHash: string;
+}
+
+export interface Cachaca extends ResultEventBase {
+  evaluationStrategy: 'CACHACA';
+}
+
+export interface CacheHit extends ResultEventBase {
+  evaluationStrategy: 'CACHE_HIT';
+}
+
+export type Extensional = ComputedExtensional | NonComputedExtensional;
+
+export type SummaryEvent =
+  | ComputeSimple
+  | ComputeRecursive
+  | InLayer
+  | Extensional
+  | SentinelEmpty
+  | Cachaca
+  | CacheHit;

extensions/ql-vscode/src/log-insights/summary-language-support.ts

@@ -0,0 +1,154 @@
+import * as fs from 'fs-extra';
+import { RawSourceMap, SourceMapConsumer } from 'source-map';
+import { commands, Position, Selection, TextDocument, TextEditor, TextEditorRevealType, TextEditorSelectionChangeEvent, ViewColumn, window, workspace } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+import { commandRunner } from '../commandRunner';
+import { logger } from '../logging';
+import { getErrorMessage } from '../pure/helpers-pure';
+
+/** A `Position` within a specified file on disk. */
+interface PositionInFile {
+  filePath: string;
+  position: Position;
+}
+
+/**
+ * Opens the specified source location in a text editor.
+ * @param position The position (including file path) to show.
+ */
+async function showSourceLocation(position: PositionInFile): Promise<void> {
+  const document = await workspace.openTextDocument(position.filePath);
+  const editor = await window.showTextDocument(document, ViewColumn.Active);
+  editor.selection = new Selection(position.position, position.position);
+  editor.revealRange(editor.selection, TextEditorRevealType.InCenterIfOutsideViewport);
+}
+
+/**
+ * Simple language support for human-readable evaluator log summaries.
+ *
+ * This class implements the `codeQL.gotoQL` command, which jumps from RA code to the corresponding
+ * QL code that generated it. It also tracks the current selection and active editor to enable and
+ * disable that command based on whether there is a QL mapping for the current selection.
+ */
+export class SummaryLanguageSupport extends DisposableObject {
+  /**
+   * The last `TextDocument` (with language `ql-summary`) for which we tried to find a sourcemap, or
+   * `undefined` if we have not seen such a document yet.
+   */
+  private lastDocument: TextDocument | undefined = undefined;
+  /**
+   * The sourcemap for `lastDocument`, or `undefined` if there was no such sourcemap or document.
+   */
+  private sourceMap: SourceMapConsumer | undefined = undefined;
+
+  constructor() {
+    super();
+
+    this.push(window.onDidChangeActiveTextEditor(this.handleDidChangeActiveTextEditor));
+    this.push(window.onDidChangeTextEditorSelection(this.handleDidChangeTextEditorSelection));
+    this.push(workspace.onDidCloseTextDocument(this.handleDidCloseTextDocument));
+
+    this.push(commandRunner('codeQL.gotoQL', this.handleGotoQL));
+  }
+
+  /**
+   * Gets the location of the QL code that generated the RA at the current selection in the active
+   * editor, or `undefined` if there is no mapping.
+   */
+  private async getQLSourceLocation(): Promise<PositionInFile | undefined> {
+    const editor = window.activeTextEditor;
+    if (editor === undefined) {
+      return undefined;
+    }
+
+    const document = editor.document;
+    if (document.languageId !== 'ql-summary') {
+      return undefined;
+    }
+
+    if (document.uri.scheme !== 'file') {
+      return undefined;
+    }
+
+    if (this.lastDocument !== document) {
+      this.clearCache();
+
+      const mapPath = document.uri.fsPath + '.map';
+
+      try {
+        const sourceMapText = await fs.readFile(mapPath, 'utf-8');
+        const rawMap: RawSourceMap = JSON.parse(sourceMapText);
+        this.sourceMap = await new SourceMapConsumer(rawMap);
+      } catch (e: unknown) {
+        // Error reading sourcemap. Pretend there was no sourcemap.
+        void logger.log(`Error reading sourcemap file '${mapPath}': ${getErrorMessage(e)}`);
+        this.sourceMap = undefined;
+      }
+      this.lastDocument = document;
+    }
+
+    if (this.sourceMap === undefined) {
+      return undefined;
+    }
+
+    const qlPosition = this.sourceMap.originalPositionFor({
+      line: editor.selection.start.line + 1,
+      column: editor.selection.start.character,
+      bias: SourceMapConsumer.GREATEST_LOWER_BOUND
+    });
+
+    if ((qlPosition.source === null) || (qlPosition.line === null)) {
+      // No position found.
+      return undefined;
+    }
+    const line = qlPosition.line - 1;  // In `source-map`, lines are 1-based...
+    const column = qlPosition.column ?? 0;  // ...but columns are 0-based :(
+
+    return {
+      filePath: qlPosition.source,
+      position: new Position(line, column)
+    };
+  }
+
+  /**
+   * Clears the cached sourcemap and its corresponding `TextDocument`.
+   */
+  private clearCache(): void {
+    if (this.sourceMap !== undefined) {
+      this.sourceMap.destroy();
+      this.sourceMap = undefined;
+      this.lastDocument = undefined;
+    }
+  }
+
+  /**
+   * Updates the `codeql.hasQLSource` context variable based on the current selection. This variable
+   * controls whether or not the `codeQL.gotoQL` command is enabled.
+   */
+  private async updateContext(): Promise<void> {
+    const position = await this.getQLSourceLocation();
+
+    await commands.executeCommand('setContext', 'codeql.hasQLSource', position !== undefined);
+  }
+
+  handleDidChangeActiveTextEditor = async (_editor: TextEditor | undefined): Promise<void> => {
+    await this.updateContext();
+  }
+
+  handleDidChangeTextEditorSelection = async (_e: TextEditorSelectionChangeEvent): Promise<void> => {
+    await this.updateContext();
+  }
+
+  handleDidCloseTextDocument = (document: TextDocument): void => {
+    if (this.lastDocument === document) {
+      this.clearCache();
+    }
+  }
+
+  handleGotoQL = async (): Promise<void> => {
+    const position = await this.getQLSourceLocation();
+    if (position !== undefined) {
+      await showSourceLocation(position);
+    }
+  };
+}

extensions/ql-vscode/src/log-insights/summary-parser.ts

@@ -0,0 +1,113 @@
+import * as fs from 'fs-extra';
+
+/**
+ * Location information for a single pipeline invocation in the RA.
+ */
+export interface PipelineInfo {
+  startLine: number;
+  raStartLine: number;
+  raEndLine: number;
+}
+
+/**
+ * Location information for a single predicate in the RA.
+ */
+export interface PredicateSymbol {
+  /**
+   * `PipelineInfo` for each iteration. A non-recursive predicate will have a single iteration `0`.
+   */
+  iterations: Record<number, PipelineInfo>;
+}
+
+/**
+ * Location information for the RA from an evaluation log. Line numbers point into the
+ * human-readable log summary.
+ */
+export interface SummarySymbols {
+  predicates: Record<string, PredicateSymbol>;
+}
+
+// Tuple counts for Expr::Expr::getParent#dispred#f0820431#ff@76d6745o:
+const NON_RECURSIVE_TUPLE_COUNT_REGEXP = /^Evaluated relational algebra for predicate (?<predicateName>\S+) with tuple counts:$/;
+// Tuple counts for Expr::Expr::getEnclosingStmt#f0820431#bf@923ddwj9 on iteration 0 running pipeline base:
+const RECURSIVE_TUPLE_COUNT_REGEXP = /^Evaluated relational algebra for predicate (?<predicateName>\S+) on iteration (?<iteration>\d+) running pipeline (?<pipeline>\S+) with tuple counts:$/;
+const RETURN_REGEXP = /^\s*return /;
+
+/**
+ * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline
+ * run.
+ *
+ * TODO: Once we're more certain about the symbol format, we should have the CLI generate this as it
+ * generates the human-readabe summary to avoid having to rely on regular expression matching of the
+ * human-readable text.
+ *
+ * @param summaryPath The path to the summary file.
+ * @param symbolsPath The path to the symbols file to generate.
+ */
+export async function generateSummarySymbolsFile(summaryPath: string, symbolsPath: string): Promise<void> {
+  const symbols = await generateSummarySymbols(summaryPath);
+  await fs.writeFile(symbolsPath, JSON.stringify(symbols));
+}
+
+/**
+ * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline
+ * run.
+ *
+ * @param fileLocation The path to the summary file.
+ * @returns Symbol information for the summary file.
+ */
+async function generateSummarySymbols(summaryPath: string): Promise<SummarySymbols> {
+  const summary = await fs.promises.readFile(summaryPath, { encoding: 'utf-8' });
+  const symbols: SummarySymbols = {
+    predicates: {}
+  };
+
+  const lines = summary.split(/\r?\n/);
+  let lineNumber = 0;
+  while (lineNumber < lines.length) {
+    const startLineNumber = lineNumber;
+    lineNumber++;
+    const startLine = lines[startLineNumber];
+    const nonRecursiveMatch = startLine.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP);
+    let predicateName: string | undefined = undefined;
+    let iteration = 0;
+    if (nonRecursiveMatch) {
+      predicateName = nonRecursiveMatch.groups!.predicateName;
+    } else {
+      const recursiveMatch = startLine.match(RECURSIVE_TUPLE_COUNT_REGEXP);
+      if (recursiveMatch?.groups) {
+        predicateName = recursiveMatch.groups.predicateName;
+        iteration = parseInt(recursiveMatch.groups.iteration);
+      }
+    }
+
+    if (predicateName !== undefined) {
+      const raStartLine = lineNumber;
+      let raEndLine: number | undefined = undefined;
+      while ((lineNumber < lines.length) && (raEndLine === undefined)) {
+        const raLine = lines[lineNumber];
+        const returnMatch = raLine.match(RETURN_REGEXP);
+        if (returnMatch) {
+          raEndLine = lineNumber;
+        }
+        lineNumber++;
+      }
+      if (raEndLine !== undefined) {
+        let symbol = symbols.predicates[predicateName];
+        if (symbol === undefined) {
+          symbol = {
+            iterations: {}
+          };
+          symbols.predicates[predicateName] = symbol;
+        }
+        symbol.iterations[iteration] = {
+          startLine: lineNumber,
+          raStartLine: raStartLine,
+          raEndLine: raEndLine
+        };
+      }
+    }
+  }
+
+  return symbols;
+}

extensions/ql-vscode/src/pure/bqrs-cli-types.ts

@@ -103,7 +103,14 @@ export function transformBqrsResultSet(
   };
 }
 
+type BqrsKind = 'String' | 'Float' | 'Integer' | 'String' | 'Boolean' | 'Date' | 'Entity';
+
+interface BqrsColumn {
+  name: string;
+  kind: BqrsKind;
+}
 export interface DecodedBqrsChunk {
   tuples: CellValue[][];
   next?: number;
+  columns: BqrsColumn[];
 }

extensions/ql-vscode/src/pure/bqrs-utils.ts

@@ -97,20 +97,35 @@ export function isStringLoc(loc: UrlValue): loc is string {
 
 export function tryGetRemoteLocation(
   loc: UrlValue | undefined,
-  fileLinkPrefix: string
+  fileLinkPrefix: string,
+  sourceLocationPrefix: string | undefined,
 ): string | undefined {
   const resolvableLocation = tryGetResolvableLocation(loc);
   if (!resolvableLocation) {
     return undefined;
   }
 
-  // Remote locations have the following format:
-  // file:/home/runner/work/<repo>/<repo/relative/path/to/file
-  // So we need to drop the first 6 parts of the path.
+  let trimmedLocation: string;
 
-  // TODO: We can make this more robust to other path formats.
-  const locationParts = resolvableLocation.uri.split('/');
-  const trimmedLocation = locationParts.slice(6, locationParts.length).join('/');
+  // Remote locations have the following format:
+  // "file:${sourceLocationPrefix}/relative/path/to/file"
+  // So we need to strip off the first part to get the relative path.
+  if (sourceLocationPrefix) {
+    if (!resolvableLocation.uri.startsWith(`file:${sourceLocationPrefix}/`)) {
+      return undefined;
+    }
+    trimmedLocation = resolvableLocation.uri.replace(`file:${sourceLocationPrefix}/`, '');
+  } else {
+    // If the source location prefix is empty (e.g. for older remote queries), we assume that the database
+    // was created on a Linux actions runner and has the format:
+    // "file:/home/runner/work/<repo>/<repo>/relative/path/to/file"
+    // So we need to drop the first 6 parts of the path.
+    if (!resolvableLocation.uri.startsWith('file:/home/runner/work/')) {
+      return undefined;
+    }
+    const locationParts = resolvableLocation.uri.split('/');
+    trimmedLocation = locationParts.slice(6, locationParts.length).join('/');
+  }
 
   const fileLink = {
     fileLinkPrefix,

extensions/ql-vscode/src/pure/date.ts

@@ -0,0 +1,26 @@
+/*
+ * Contains an assortment of helper constants and functions for working with dates.
+ */
+
+const dateWithoutYearFormatter = new Intl.DateTimeFormat(undefined, {
+  month: 'short',
+  day: 'numeric',
+  hour: 'numeric',
+  minute: '2-digit',
+});
+
+const dateFormatter = new Intl.DateTimeFormat(undefined, {
+  year: 'numeric',
+  month: 'short',
+  day: 'numeric',
+  hour: 'numeric',
+  minute: '2-digit',
+});
+
+export function formatDate(value: Date): string {
+  if (value.getFullYear() === new Date().getFullYear()) {
+    return dateWithoutYearFormatter.format(value);
+  }
+
+  return dateFormatter.format(value);
+}

extensions/ql-vscode/src/pure/helpers-pure.ts

@@ -31,17 +31,18 @@ export const asyncFilter = async function <T>(arr: T[], predicate: (arg0: T) =>
   return arr.filter((_, index) => results[index]);
 };
 
-export const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000;
-export const ONE_HOUR_IN_MS = 1000 * 60 * 60;
-export const TWO_HOURS_IN_MS = 1000 * 60 * 60 * 2;
-export const THREE_HOURS_IN_MS = 1000 * 60 * 60 * 3;
-
 /**
  * This regex matches strings of the form `owner/repo` where:
- * - `owner` is made up of alphanumeric characters or single hyphens, starting and ending in an alphanumeric character
- * - `repo` is made up of alphanumeric characters, hyphens, or underscores
+ * - `owner` is made up of alphanumeric characters, hyphens, underscores, or periods
+ * - `repo` is made up of alphanumeric characters, hyphens, underscores, or periods
  */
-export const REPO_REGEX = /^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+\/[a-zA-Z0-9-_]+$/;
+export const REPO_REGEX = /^[a-zA-Z0-9-_\.]+\/[a-zA-Z0-9-_\.]+$/;
+
+/**
+ * This regex matches GiHub organization and user strings. These are made up for alphanumeric 
+ * characters, hyphens, underscores or periods.
+ */
+export const OWNER_REGEX = /^[a-zA-Z0-9-_\.]+$/;
 
 export function getErrorMessage(e: any) {
   return e instanceof Error ? e.message : String(e);

extensions/ql-vscode/src/pure/interface-types.ts

@@ -2,6 +2,11 @@ import * as sarif from 'sarif';
 import { AnalysisResults } from '../remote-queries/shared/analysis-result';
 import { AnalysisSummary, RemoteQueryResult } from '../remote-queries/shared/remote-query-result';
 import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationValue } from './bqrs-cli-types';
+import {
+  VariantAnalysis,
+  VariantAnalysisScannedRepositoryResult,
+  VariantAnalysisScannedRepositoryState,
+} from '../remote-queries/shared/variant-analysis';
 
 /**
  * This module contains types and code that are shared between
@@ -174,7 +179,7 @@ export type FromResultsViewMsg =
   | ToggleDiagnostics
   | ChangeRawResultsSortMsg
   | ChangeInterpretedResultsSortMsg
-  | ResultViewLoaded
+  | ViewLoadedMsg
   | ChangePage
   | OpenFileMsg;
 
@@ -216,11 +221,11 @@ interface ToggleDiagnostics {
 }
 
 /**
- * Message from the results view to signal that loading the results
- * is complete.
+ * Message from a view signal that loading is complete.
  */
-interface ResultViewLoaded {
-  t: 'resultViewLoaded';
+interface ViewLoadedMsg {
+  t: 'viewLoaded';
+  viewName: string;
 }
 
 /**
@@ -279,18 +284,11 @@ interface ChangeInterpretedResultsSortMsg {
  * Message from the compare view to the extension.
  */
 export type FromCompareViewMessage =
-  | CompareViewLoadedMessage
+  | ViewLoadedMsg
   | ChangeCompareMessage
   | ViewSourceFileMsg
   | OpenQueryMessage;
 
-/**
- * Message from the compare view to signal the completion of loading results.
- */
-interface CompareViewLoadedMessage {
-  t: 'compareViewLoaded';
-}
-
 /**
  * Message from the compare view to request opening a query.
  */
@@ -389,21 +387,19 @@ export interface ParsedResultSets {
 }
 
 export type FromRemoteQueriesMessage =
-  | RemoteQueryLoadedMessage
+  | ViewLoadedMsg
   | RemoteQueryErrorMessage
   | OpenFileMsg
   | OpenVirtualFileMsg
   | RemoteQueryDownloadAnalysisResultsMessage
-  | RemoteQueryDownloadAllAnalysesResultsMessage;
+  | RemoteQueryDownloadAllAnalysesResultsMessage
+  | RemoteQueryExportResultsMessage
+  | CopyRepoListMessage;
 
 export type ToRemoteQueriesMessage =
   | SetRemoteQueryResultMessage
   | SetAnalysesResultsMessage;
 
-export interface RemoteQueryLoadedMessage {
-  t: 'remoteQueryLoaded';
-}
-
 export interface SetRemoteQueryResultMessage {
   t: 'setRemoteQueryResult';
   queryResult: RemoteQueryResult
@@ -429,3 +425,56 @@ export interface RemoteQueryDownloadAllAnalysesResultsMessage {
   analysisSummaries: AnalysisSummary[];
 }
 
+export interface RemoteQueryExportResultsMessage {
+  t: 'remoteQueryExportResults';
+  queryId: string;
+}
+
+export interface CopyRepoListMessage {
+  t: 'copyRepoList';
+  queryId: string;
+}
+
+export interface SetVariantAnalysisMessage {
+  t: 'setVariantAnalysis';
+  variantAnalysis: VariantAnalysis;
+}
+
+export type StopVariantAnalysisMessage = {
+  t: 'stopVariantAnalysis';
+  variantAnalysisId: number;
+}
+
+export type VariantAnalysisState = {
+  variantAnalysisId: number;
+}
+
+export interface SetRepoResultsMessage {
+  t: 'setRepoResults';
+  repoResults: VariantAnalysisScannedRepositoryResult[];
+}
+
+export interface SetRepoStatesMessage {
+  t: 'setRepoStates';
+  repoStates: VariantAnalysisScannedRepositoryState[];
+}
+
+export interface RequestRepositoryResultsMessage {
+  t: 'requestRepositoryResults';
+  repositoryFullName: string;
+}
+
+export interface OpenQueryFileMessage {
+  t: 'openQueryFile';
+}
+
+export type ToVariantAnalysisMessage =
+  | SetVariantAnalysisMessage
+  | SetRepoResultsMessage
+  | SetRepoStatesMessage;
+
+export type FromVariantAnalysisMessage =
+  | ViewLoadedMsg
+  | StopVariantAnalysisMessage
+  | RequestRepositoryResultsMessage
+  | OpenQueryFileMessage;

extensions/ql-vscode/src/pure/legacy-messages.ts

@@ -15,38 +15,7 @@
  */
 
 import * as rpc from 'vscode-jsonrpc';
-
-/**
- * A position within a QL file.
- */
-export interface Position {
-  /**
-   * The one-based index of the start line
-   */
-  line: number;
-  /**
-   * The one-based offset of the start column within
-   * the start line in UTF-16 code-units
-   */
-  column: number;
-  /**
-   * The one-based index of the end line line
-   */
-  endLine: number;
-
-  /**
-   * The one-based offset of the end column within
-   * the end line in UTF-16 code-units
-   */
-  endColumn: number;
-  /**
-   * The path of the file.
-   * If the file name is "Compiler Generated" the
-   * the position is not a real position but
-   * arises from compiler generated code.
-   */
-  fileName: string;
-}
+import * as shared from './messages-shared';
 
 /**
  * A query that should be checked for any errors or warnings
@@ -155,6 +124,10 @@ export interface CompilationOptions {
    * get reported anyway. Useful for universal compilation options.
    */
   computeDefaultStrings: boolean;
+  /**
+   * Emit debug information in compiled query.
+   */
+  emitDebugInfo: boolean;
 }
 
 /**
@@ -254,28 +227,6 @@ export interface DILQuery {
   dilSource: string;
 }
 
-/**
- * The way of compiling the query, as a normal query
- * or a subset of it. Note that precisely one of the two options should be set.
- */
-export interface CompilationTarget {
-  /**
-   * Compile as a normal query
-   */
-  query?: Record<string, never>;
-  /**
-   * Compile as a quick evaluation
-   */
-  quickEval?: QuickEvalOptions;
-}
-
-/**
- * Options for quick evaluation
- */
-export interface QuickEvalOptions {
-  quickEvalPos?: Position;
-}
-
 /**
  * The result of checking a query.
  */
@@ -650,7 +601,7 @@ export interface ClearCacheParams {
 /**
  * Parameters to start a new structured log
  */
- export interface StartLogParams {
+export interface StartLogParams {
   /**
    * The dataset for which we want to start a new structured log
    */
@@ -664,7 +615,7 @@ export interface ClearCacheParams {
 /**
  * Parameters to terminate a structured log
  */
- export interface EndLogParams {
+export interface EndLogParams {
   /**
    * The dataset for which we want to terminated the log
    */
@@ -1008,37 +959,20 @@ export type DeregisterDatabasesResult = {
 };
 
 /**
- * Type for any action that could have progress messages.
+ * A position within a QL file.
  */
-export interface WithProgressId<T> {
-  /**
-   * The main body
-   */
-  body: T;
-  /**
-   * The id used to report progress updates
-   */
-  progressId: number;
-}
+export type Position = shared.Position;
 
-export interface ProgressMessage {
-  /**
-   * The id of the operation that is running
-   */
-  id: number;
-  /**
-   * The current step
-   */
-  step: number;
-  /**
-   * The maximum step. This *should* be constant for a single job.
-   */
-  maxStep: number;
-  /**
-   * The current progress message
-   */
-  message: string;
-}
+/**
+ * The way of compiling the query, as a normal query
+ * or a subset of it. Note that precisely one of the two options should be set.
+ */
+export type CompilationTarget = shared.CompilationTarget;
+
+export type QuickEvalOptions = shared.QuickEvalOptions;
+
+export type WithProgressId<T> = shared.WithProgressId<T>;
+export type ProgressMessage = shared.ProgressMessage;
 
 /**
  * Check a Ql query for errors without compiling it
@@ -1070,12 +1004,12 @@ export const compileUpgradeSequence = new rpc.RequestType<WithProgressId<Compile
 /**
  * Start a new structured log in the evaluator, terminating the previous one if it exists
  */
- export const startLog = new rpc.RequestType<WithProgressId<StartLogParams>, StartLogResult, void, void>('evaluation/startLog');
+export const startLog = new rpc.RequestType<WithProgressId<StartLogParams>, StartLogResult, void, void>('evaluation/startLog');
 
 /**
  * Terminate a structured log in the evaluator. Is a no-op if we aren't logging to the given location
  */
- export const endLog = new rpc.RequestType<WithProgressId<EndLogParams>, EndLogResult, void, void>('evaluation/endLog');
+export const endLog = new rpc.RequestType<WithProgressId<EndLogParams>, EndLogResult, void, void>('evaluation/endLog');
 
 /**
  * Clear the cache of a dataset
@@ -1116,7 +1050,4 @@ export const deregisterDatabases = new rpc.RequestType<
  */
 export const completeQuery = new rpc.RequestType<EvaluationResult, Record<string, any>, void, void>('evaluation/queryCompleted');
 
-/**
- * A notification that the progress has been changed.
- */
-export const progress = new rpc.NotificationType<ProgressMessage, void>('ql/progressUpdated');
+export const progress = shared.progress;

extensions/ql-vscode/src/pure/log-summary-parser.ts

@@ -0,0 +1,34 @@
+import { readJsonlFile } from '../log-insights/jsonl-reader';
+
+// TODO(angelapwen): Only load in necessary information and
+// location in bytes for this log to save memory.
+export interface EvalLogData {
+  predicateName: string;
+  millis: number;
+  resultSize: number;
+  // Key: pipeline identifier; Value: array of pipeline steps
+  ra: Record<string, string[]>;
+}
+
+/**
+ * A pure method that parses a string of evaluator log summaries into
+ * an array of EvalLogData objects.
+ */
+export async function parseViewerData(jsonSummaryPath: string): Promise<EvalLogData[]> {
+  const viewerData: EvalLogData[] = [];
+
+  await readJsonlFile(jsonSummaryPath, async jsonObj => {
+    // Only convert log items that have an RA and millis field
+    if (jsonObj.ra !== undefined && jsonObj.millis !== undefined) {
+      const newLogData: EvalLogData = {
+        predicateName: jsonObj.predicateName,
+        millis: jsonObj.millis,
+        resultSize: jsonObj.resultSize,
+        ra: jsonObj.ra
+      };
+      viewerData.push(newLogData);
+    }
+  });
+
+  return viewerData;
+}

extensions/ql-vscode/src/pure/messages-shared.ts

@@ -0,0 +1,110 @@
+/**
+ * Types for messages exchanged during jsonrpc communication with the
+ * the CodeQL query server.
+ *
+ * This file exists in the queryserver and in the vscode extension, and
+ * should be kept in sync between them.
+ *
+ * A note about the namespaces below, which look like they are
+ * essentially enums, namely Severity, ResultColumnKind, and
+ * QueryResultType. By design, for the sake of extensibility, clients
+ * receiving messages of this protocol are supposed to accept any
+ * number for any of these types. We commit to the given meaning of
+ * the numbers listed in constants in the namespaces, and we commit to
+ * the fact that any unknown QueryResultType value counts as an error.
+ */
+
+import * as rpc from 'vscode-jsonrpc';
+
+/**
+ * A position within a QL file.
+ */
+export interface Position {
+  /**
+   * The one-based index of the start line
+   */
+  line: number;
+  /**
+   * The one-based offset of the start column within
+   * the start line in UTF-16 code-units
+   */
+  column: number;
+  /**
+   * The one-based index of the end line line
+   */
+  endLine: number;
+
+  /**
+   * The one-based offset of the end column within
+   * the end line in UTF-16 code-units
+   */
+  endColumn: number;
+  /**
+   * The path of the file.
+   * If the file name is "Compiler Generated" the
+   * the position is not a real position but
+   * arises from compiler generated code.
+   */
+  fileName: string;
+}
+
+/**
+ * The way of compiling the query, as a normal query
+ * or a subset of it. Note that precisely one of the two options should be set.
+ */
+export interface CompilationTarget {
+  /**
+   * Compile as a normal query
+   */
+  query?: Record<string, never>;
+  /**
+   * Compile as a quick evaluation
+   */
+  quickEval?: QuickEvalOptions;
+}
+
+/**
+ * Options for quick evaluation
+ */
+export interface QuickEvalOptions {
+  quickEvalPos?: Position;
+}
+
+/**
+ * Type for any action that could have progress messages.
+ */
+export interface WithProgressId<T> {
+  /**
+   * The main body
+   */
+  body: T;
+  /**
+   * The id used to report progress updates
+   */
+  progressId: number;
+}
+
+export interface ProgressMessage {
+  /**
+   * The id of the operation that is running
+   */
+  id: number;
+  /**
+   * The current step
+   */
+  step: number;
+  /**
+   * The maximum step. This *should* be constant for a single job.
+   */
+  maxStep: number;
+  /**
+   * The current progress message
+   */
+  message: string;
+}
+
+
+/**
+ * A notification that the progress has been changed.
+ */
+export const progress = new rpc.NotificationType<ProgressMessage, void>('ql/progressUpdated');

extensions/ql-vscode/src/pure/new-messages.ts

@@ -0,0 +1,215 @@
+/**
+ * Types for messages exchanged during jsonrpc communication with the
+ * the CodeQL query server.
+ *
+ * This file exists in the queryserver and in the vscode extension, and
+ * should be kept in sync between them.
+ *
+ * A note about the namespaces below, which look like they are
+ * essentially enums, namely Severity, ResultColumnKind, and
+ * QueryResultType. By design, for the sake of extensibility, clients
+ * receiving messages of this protocol are supposed to accept any
+ * number for any of these types. We commit to the given meaning of
+ * the numbers listed in constants in the namespaces, and we commit to
+ * the fact that any unknown QueryResultType value counts as an error.
+ */
+
+import * as rpc from 'vscode-jsonrpc';
+import * as shared from './messages-shared';
+
+
+
+/**
+ * Parameters to clear the cache
+ */
+export interface ClearCacheParams {
+  /**
+   * The dataset for which we want to clear the cache
+   */
+  db: string;
+  /**
+   * Whether the cache should actually be cleared.
+   */
+  dryRun: boolean;
+}
+
+/**
+ * Parameters for trimming the cache of a dataset
+ */
+export interface TrimCacheParams {
+  /**
+   * The dataset that we want to trim the cache of.
+   */
+  db: string;
+}
+
+/**
+ * The result of trimming or clearing the cache.
+ */
+export interface ClearCacheResult {
+  /**
+   * A user friendly message saying what was or would be
+   * deleted.
+   */
+  deletionMessage: string;
+}
+
+
+export type QueryResultType = number;
+/**
+ * The result of running a query. This namespace is intentionally not
+ * an enum, see "for the sake of extensibility" comment above.
+ */
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace QueryResultType {
+  /**
+   * The query ran successfully
+   */
+  export const SUCCESS = 0;
+  /**
+   * The query failed due to an reason
+   * that isn't listed
+   */
+  export const OTHER_ERROR = 1;
+  /**
+   * The query failed do to compilation erorrs
+   */
+  export const COMPILATION_ERROR = 2;
+  /**
+    * The query failed due to running out of
+    * memory
+    */
+  export const OOM = 3;
+  /**
+   * The query failed because it was cancelled.
+   */
+  export const CANCELLATION = 4;
+  /**
+   * The dbscheme basename was not the same
+   */
+  export const DBSCHEME_MISMATCH_NAME = 5;
+  /**
+   * No upgrade was found
+   */
+  export const DBSCHEME_NO_UPGRADE = 6;
+}
+
+
+export interface RegisterDatabasesParams {
+  databases: string[];
+}
+
+export interface DeregisterDatabasesParams {
+  databases: string[];
+}
+
+export type RegisterDatabasesResult = {
+  registeredDatabases: string[];
+};
+
+export type DeregisterDatabasesResult = {
+  registeredDatabases: string[];
+};
+
+
+export interface RunQueryParams {
+  /**
+   * The path of the query
+   */
+  queryPath: string,
+  /**
+   * The output path
+   */
+  outputPath: string,
+  /**
+   * The database path
+   */
+  db: string,
+  additionalPacks: string[],
+  target: CompilationTarget,
+  externalInputs: Record<string, string>,
+  singletonExternalInputs: Record<string, string>,
+  dilPath?: string,
+  logPath?: string
+}
+
+export interface RunQueryResult {
+  resultType: QueryResultType,
+  message?: string,
+  expectedDbschemeName?: string,
+  evaluationTime: number;
+}
+
+
+
+export interface UpgradeParams {
+  db: string,
+  additionalPacks: string[],
+}
+
+export type UpgradeResult = Record<string, unknown>;
+
+export type ClearPackCacheParams = Record<string, unknown>;
+export type ClearPackCacheResult = Record<string, unknown>;
+
+/**
+ * A position within a QL file.
+ */
+export type Position = shared.Position;
+
+/**
+ * The way of compiling the query, as a normal query
+ * or a subset of it. Note that precisely one of the two options should be set.
+ */
+export type CompilationTarget = shared.CompilationTarget;
+
+export type QuickEvalOptions = shared.QuickEvalOptions;
+
+export type WithProgressId<T> = shared.WithProgressId<T>;
+export type ProgressMessage = shared.ProgressMessage;
+
+/**
+ * Clear the cache of a dataset
+ */
+export const clearCache = new rpc.RequestType<WithProgressId<ClearCacheParams>, ClearCacheResult, void, void>('evaluation/clearCache');
+/**
+ * Trim the cache of a dataset
+ */
+export const trimCache = new rpc.RequestType<WithProgressId<TrimCacheParams>, ClearCacheResult, void, void>('evaluation/trimCache');
+
+/**
+ * Clear the pack cache
+ */
+export const clearPackCache = new rpc.RequestType<WithProgressId<ClearPackCacheParams>, ClearPackCacheResult, void, void>('evaluation/clearPackCache');
+
+/**
+ * Run a query on a database
+ */
+export const runQuery = new rpc.RequestType<WithProgressId<RunQueryParams>, RunQueryResult, void, void>('evaluation/runQuery');
+
+export const registerDatabases = new rpc.RequestType<
+  WithProgressId<RegisterDatabasesParams>,
+  RegisterDatabasesResult,
+  void,
+  void
+>('evaluation/registerDatabases');
+
+export const deregisterDatabases = new rpc.RequestType<
+  WithProgressId<DeregisterDatabasesParams>,
+  DeregisterDatabasesResult,
+  void,
+  void
+>('evaluation/deregisterDatabases');
+
+
+export const upgradeDatabase = new rpc.RequestType<
+  WithProgressId<UpgradeParams>,
+  UpgradeResult,
+  void,
+  void
+>('evaluation/runUpgrade');
+
+/**
+ * A notification that the progress has been changed.
+ */
+export const progress = shared.progress;

extensions/ql-vscode/src/pure/number.ts

@@ -0,0 +1,15 @@
+/*
+ * Contains an assortment of helper constants and functions for working with numbers.
+ */
+
+const numberFormatter = new Intl.NumberFormat('en-US');
+
+/**
+ * Formats a number to be human-readable with decimal places and thousands separators.
+ *
+ * @param value The number to format.
+ * @returns The formatted number. For example, "10,000", "1,000,000", or "1,000,000,000".
+ */
+export function formatDecimal(value: number): string {
+  return numberFormatter.format(value);
+}

extensions/ql-vscode/src/pure/sarif-utils.ts

@@ -1,4 +1,5 @@
 import * as Sarif from 'sarif';
+import { HighlightedRegion } from '../remote-queries/shared/analysis-result';
 import { ResolvableLocationValue } from './bqrs-cli-types';
 
 export interface SarifLink {
@@ -173,3 +174,65 @@ export function parseSarifRegion(
 export function isNoLocation(loc: ParsedSarifLocation): loc is NoLocation {
   return 'hint' in loc;
 }
+
+// Some helpers for highlighting specific regions from a SARIF code snippet
+
+/**
+ * Checks whether a particular line (determined by its line number in the original file)
+ * is part of the highlighted region of a SARIF code snippet.
+ */
+export function shouldHighlightLine(
+  lineNumber: number,
+  highlightedRegion: HighlightedRegion
+): boolean {
+  if (lineNumber < highlightedRegion.startLine) {
+    return false;
+  }
+
+  if (highlightedRegion.endLine == undefined) {
+    return lineNumber == highlightedRegion.startLine;
+  }
+
+  return lineNumber <= highlightedRegion.endLine;
+}
+
+/**
+ * A line of code split into: plain text before the highlighted section, the highlighted
+ * text itself, and plain text after the highlighted section.
+ */
+export interface PartiallyHighlightedLine {
+  plainSection1: string;
+  highlightedSection: string;
+  plainSection2: string;
+}
+
+/**
+ * Splits a line of code into the highlighted and non-highlighted sections.
+ */
+export function parseHighlightedLine(
+  line: string,
+  lineNumber: number,
+  highlightedRegion: HighlightedRegion
+): PartiallyHighlightedLine {
+  const isSingleLineHighlight = highlightedRegion.endLine === undefined;
+  const isFirstHighlightedLine = lineNumber === highlightedRegion.startLine;
+  const isLastHighlightedLine = lineNumber === highlightedRegion.endLine;
+
+  const highlightStartColumn = isSingleLineHighlight
+    ? highlightedRegion.startColumn
+    : isFirstHighlightedLine
+      ? highlightedRegion.startColumn
+      : 0;
+
+  const highlightEndColumn = isSingleLineHighlight
+    ? highlightedRegion.endColumn
+    : isLastHighlightedLine
+      ? highlightedRegion.endColumn
+      : line.length + 1;
+
+  const plainSection1 = line.substring(0, highlightStartColumn - 1);
+  const highlightedSection = line.substring(highlightStartColumn - 1, highlightEndColumn - 1);
+  const plainSection2 = line.substring(highlightEndColumn - 1, line.length);
+
+  return { plainSection1, highlightedSection, plainSection2 };
+}

extensions/ql-vscode/src/pure/time.ts

@@ -0,0 +1,89 @@
+/*
+ * Contains an assortment of helper constants and functions for working with time, dates, and durations.
+ */
+
+export const ONE_SECOND_IN_MS = 1000;
+export const ONE_MINUTE_IN_MS = ONE_SECOND_IN_MS * 60;
+export const ONE_HOUR_IN_MS = ONE_MINUTE_IN_MS * 60;
+export const TWO_HOURS_IN_MS = ONE_HOUR_IN_MS * 2;
+export const THREE_HOURS_IN_MS = ONE_HOUR_IN_MS * 3;
+export const ONE_DAY_IN_MS = ONE_HOUR_IN_MS * 24;
+
+// These are approximations
+export const ONE_MONTH_IN_MS = ONE_DAY_IN_MS * 30;
+export const ONE_YEAR_IN_MS = ONE_DAY_IN_MS * 365;
+
+const durationFormatter = new Intl.RelativeTimeFormat('en', {
+  numeric: 'auto',
+});
+
+/**
+ * Converts a number of milliseconds into a human-readable string with units, indicating a relative time in the past or future.
+ *
+ * @param relativeTimeMillis The duration in milliseconds. A negative number indicates a duration in the past. And a positive number is
+ *  the future.
+ * @returns A humanized duration. For example, "in 2 minutes", "2 minutes ago", "yesterday", or "tomorrow".
+ */
+export function humanizeRelativeTime(relativeTimeMillis?: number) {
+  if (relativeTimeMillis === undefined) {
+    return '';
+  }
+
+  if (Math.abs(relativeTimeMillis) < ONE_HOUR_IN_MS) {
+    return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_MINUTE_IN_MS), 'minute');
+  } else if (Math.abs(relativeTimeMillis) < ONE_DAY_IN_MS) {
+    return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_HOUR_IN_MS), 'hour');
+  } else if (Math.abs(relativeTimeMillis) < ONE_MONTH_IN_MS) {
+    return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_DAY_IN_MS), 'day');
+  } else if (Math.abs(relativeTimeMillis) < ONE_YEAR_IN_MS) {
+    return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_MONTH_IN_MS), 'month');
+  } else {
+    return durationFormatter.format(Math.floor(relativeTimeMillis / ONE_YEAR_IN_MS), 'year');
+  }
+}
+
+/**
+ * Converts a number of milliseconds into a human-readable string with units, indicating an amount of time.
+ * Negative numbers have no meaning and are considered to be "Less than a second".
+ *
+ * @param millis The number of milliseconds to convert.
+ * @returns A humanized duration. For example, "2 seconds", "2 minutes", "2 hours", "2 days", or "2 months".
+ */
+export function humanizeUnit(millis?: number): string {
+  // assume a blank or empty string is a zero
+  // assume anything less than 0 is a zero
+  if (!millis || millis < ONE_SECOND_IN_MS) {
+    return 'Less than a second';
+  }
+  let unit: string;
+  let unitDiff: number;
+  if (millis < ONE_MINUTE_IN_MS) {
+    unit = 'second';
+    unitDiff = Math.floor(millis / ONE_SECOND_IN_MS);
+  } else if (millis < ONE_HOUR_IN_MS) {
+    unit = 'minute';
+    unitDiff = Math.floor(millis / ONE_MINUTE_IN_MS);
+  } else if (millis < ONE_DAY_IN_MS) {
+    unit = 'hour';
+    unitDiff = Math.floor(millis / ONE_HOUR_IN_MS);
+  } else if (millis < ONE_MONTH_IN_MS) {
+    unit = 'day';
+    unitDiff = Math.floor(millis / ONE_DAY_IN_MS);
+  } else if (millis < ONE_YEAR_IN_MS) {
+    unit = 'month';
+    unitDiff = Math.floor(millis / ONE_MONTH_IN_MS);
+  } else {
+    unit = 'year';
+    unitDiff = Math.floor(millis / ONE_YEAR_IN_MS);
+  }
+
+  return createFormatter(unit).format(unitDiff);
+}
+
+function createFormatter(unit: string) {
+  return Intl.NumberFormat('en-US', {
+    style: 'unit',
+    unit,
+    unitDisplay: 'long'
+  });
+}

extensions/ql-vscode/src/pure/zip.ts

@@ -0,0 +1,11 @@
+import * as unzipper from 'unzipper';
+
+/**
+ * Unzips a zip file to a directory.
+ * @param sourcePath The path to the zip file.
+ * @param destinationPath The path to the directory to unzip to.
+ */
+export async function unzipFile(sourcePath: string, destinationPath: string) {
+  const file = await unzipper.Open.file(sourcePath);
+  await file.extract({ path: destinationPath });
+}

extensions/ql-vscode/src/query-history-scrubber.ts

@@ -3,6 +3,7 @@ import * as os from 'os';
 import * as path from 'path';
 import { Disposable, ExtensionContext } from 'vscode';
 import { logger } from './logging';
+import { QueryHistoryManager } from './query-history';
 
 const LAST_SCRUB_TIME_KEY = 'lastScrubTime';
 
@@ -30,12 +31,13 @@ export function registerQueryHistoryScubber(
   throttleTime: number,
   maxQueryTime: number,
   queryDirectory: string,
+  qhm: QueryHistoryManager,
   ctx: ExtensionContext,
 
   // optional counter to keep track of how many times the scrubber has run
   counter?: Counter
 ): Disposable {
-  const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, ctx, counter);
+  const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, qhm, ctx, counter);
 
   return {
     dispose: () => {
@@ -48,6 +50,7 @@ async function scrubQueries(
   throttleTime: number,
   maxQueryTime: number,
   queryDirectory: string,
+  qhm: QueryHistoryManager,
   ctx: ExtensionContext,
   counter?: Counter
 ) {
@@ -89,6 +92,7 @@ async function scrubQueries(
     } finally {
       void logger.log(`Scrubbed ${scrubCount} old queries.`);
     }
+    await qhm.removeDeletedQueries();
   }
 }
 

extensions/ql-vscode/src/query-history.ts

@@ -9,6 +9,7 @@ import {
   ProviderResult,
   Range,
   ThemeIcon,
+  TreeDataProvider,
   TreeItem,
   TreeView,
   Uri,
@@ -25,10 +26,10 @@ import {
 } from './helpers';
 import { logger } from './logging';
 import { URLSearchParams } from 'url';
-import { QueryServerClient } from './queryserver-client';
 import { DisposableObject } from './pure/disposable-object';
 import { commandRunner } from './commandRunner';
-import { assertNever, ONE_HOUR_IN_MS, TWO_HOURS_IN_MS, getErrorMessage, getErrorStack } from './pure/helpers-pure';
+import { ONE_HOUR_IN_MS, TWO_HOURS_IN_MS } from './pure/time';
+import { assertNever, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { CompletedLocalQueryInfo, LocalQueryInfo as LocalQueryInfo, QueryHistoryInfo } from './query-results';
 import { DatabaseManager } from './databases';
 import { registerQueryHistoryScubber } from './query-history-scrubber';
@@ -36,6 +37,18 @@ import { QueryStatus } from './query-status';
 import { slurpQueryHistory, splatQueryHistory } from './query-serialization';
 import * as fs from 'fs-extra';
 import { CliVersionConstraint } from './cli';
+import { HistoryItemLabelProvider } from './history-item-label-provider';
+import { Credentials } from './authentication';
+import { cancelRemoteQuery } from './remote-queries/gh-api/gh-actions-api-client';
+import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
+import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
+import { ResultsView } from './interface';
+import { WebviewReveal } from './interface-utils';
+import { EvalLogViewer } from './eval-log-viewer';
+import EvalLogTreeBuilder from './eval-log-tree-builder';
+import { EvalLogData, parseViewerData } from './pure/log-summary-parser';
+import { QueryWithResults } from './run-queries-shared';
+import { QueryRunner } from './queryRunner';
 
 /**
  * query-history.ts
@@ -103,7 +116,7 @@ const WORKSPACE_QUERY_HISTORY_FILE = 'workspace-query-history.json';
 /**
  * Tree data provider for the query history view.
  */
-export class HistoryTreeDataProvider extends DisposableObject {
+export class HistoryTreeDataProvider extends DisposableObject implements TreeDataProvider<QueryHistoryInfo> {
   private _sortOrder = SortOrder.DateAsc;
 
   private _onDidChangeTreeData = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
@@ -111,6 +124,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
   readonly onDidChangeTreeData: Event<QueryHistoryInfo | undefined> = this
     ._onDidChangeTreeData.event;
 
+  private _onDidChangeCurrentQueryItem = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
+
+  public readonly onDidChangeCurrentQueryItem = this._onDidChangeCurrentQueryItem.event;
+
   private history: QueryHistoryInfo[] = [];
 
   private failedIconPath: string;
@@ -121,7 +138,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
 
   private current: QueryHistoryInfo | undefined;
 
-  constructor(extensionPath: string) {
+  constructor(
+    extensionPath: string,
+    private readonly labelProvider: HistoryItemLabelProvider,
+  ) {
     super();
     this.failedIconPath = path.join(
       extensionPath,
@@ -138,13 +158,13 @@ export class HistoryTreeDataProvider extends DisposableObject {
   }
 
   async getTreeItem(element: QueryHistoryInfo): Promise<TreeItem> {
-    const treeItem = new TreeItem(element.label);
+    const treeItem = new TreeItem(this.labelProvider.getLabel(element));
 
     treeItem.command = {
       title: 'Query History Item',
       command: 'codeQLQueryHistory.itemClicked',
       arguments: [element],
-      tooltip: element.failureReason || element.label
+      tooltip: element.failureReason || this.labelProvider.getLabel(element)
     };
 
     // Populate the icon and the context value. We use the context value to
@@ -169,7 +189,7 @@ export class HistoryTreeDataProvider extends DisposableObject {
         break;
       case QueryStatus.Failed:
         treeItem.iconPath = this.failedIconPath;
-        treeItem.contextValue = 'cancelledResultsItem';
+        treeItem.contextValue = element.t === 'local' ? 'cancelledResultsItem' : 'cancelledRemoteResultsItem';
         break;
       default:
         assertNever(element.status);
@@ -183,8 +203,8 @@ export class HistoryTreeDataProvider extends DisposableObject {
   ): ProviderResult<QueryHistoryInfo[]> {
     return element ? [] : this.history.sort((h1, h2) => {
 
-      const h1Label = h1.label.toLowerCase();
-      const h2Label = h2.label.toLowerCase();
+      const h1Label = this.labelProvider.getLabel(h1).toLowerCase();
+      const h2Label = this.labelProvider.getLabel(h2).toLowerCase();
 
       const h1Date = h1.t === 'local'
         ? h1.initialInfo.start.getTime()
@@ -194,13 +214,12 @@ export class HistoryTreeDataProvider extends DisposableObject {
         ? h2.initialInfo.start.getTime()
         : h2.remoteQuery?.executionStartTime;
 
-      // result count for remote queries is not available here.
       const resultCount1 = h1.t === 'local'
         ? h1.completedQuery?.resultCount ?? -1
-        : -1;
+        : h1.resultCount ?? -1;
       const resultCount2 = h2.t === 'local'
         ? h2.completedQuery?.resultCount ?? -1
-        : -1;
+        : h2.resultCount ?? -1;
 
       switch (this.sortOrder) {
         case SortOrder.NameAsc:
@@ -247,7 +266,10 @@ export class HistoryTreeDataProvider extends DisposableObject {
   }
 
   setCurrentItem(item?: QueryHistoryInfo) {
-    this.current = item;
+    if (item !== this.current) {
+      this.current = item;
+      this._onDidChangeCurrentQueryItem.fire(item);
+    }
   }
 
   remove(item: QueryHistoryInfo) {
@@ -273,7 +295,7 @@ export class HistoryTreeDataProvider extends DisposableObject {
 
   set allHistory(history: QueryHistoryInfo[]) {
     this.history = history;
-    this.current = history[0];
+    this.setCurrentItem(history[0]);
     this.refresh();
   }
 
@@ -300,25 +322,23 @@ export class QueryHistoryManager extends DisposableObject {
   queryHistoryScrubber: Disposable | undefined;
   private queryMetadataStorageLocation;
 
-  private readonly _onDidAddQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
-  readonly onDidAddQueryItem: Event<QueryHistoryInfo> = this
-    ._onDidAddQueryItem.event;
+  private readonly _onDidChangeCurrentQueryItem = super.push(new EventEmitter<QueryHistoryInfo | undefined>());
+  readonly onDidChangeCurrentQueryItem = this._onDidChangeCurrentQueryItem.event;
 
-  private readonly _onDidRemoveQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
-  readonly onDidRemoveQueryItem: Event<QueryHistoryInfo> = this
-    ._onDidRemoveQueryItem.event;
-
-  private readonly _onWillOpenQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
-  readonly onWillOpenQueryItem: Event<QueryHistoryInfo> = this
-    ._onWillOpenQueryItem.event;
+  private readonly _onDidCompleteQuery = super.push(new EventEmitter<LocalQueryInfo>());
+  readonly onDidCompleteQuery = this._onDidCompleteQuery.event;
 
   constructor(
-    private qs: QueryServerClient,
-    private dbm: DatabaseManager,
-    private queryStorageDir: string,
-    ctx: ExtensionContext,
-    private queryHistoryConfigListener: QueryHistoryConfig,
-    private doCompareCallback: (
+    private readonly qs: QueryRunner,
+    private readonly dbm: DatabaseManager,
+    private readonly localQueriesResultsView: ResultsView,
+    private readonly remoteQueriesManager: RemoteQueriesManager,
+    private readonly evalLogViewer: EvalLogViewer,
+    private readonly queryStorageDir: string,
+    private readonly ctx: ExtensionContext,
+    private readonly queryHistoryConfigListener: QueryHistoryConfig,
+    private readonly labelProvider: HistoryItemLabelProvider,
+    private readonly doCompareCallback: (
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo
     ) => Promise<void>
@@ -332,13 +352,19 @@ export class QueryHistoryManager extends DisposableObject {
     this.queryMetadataStorageLocation = path.join((ctx.storageUri || ctx.globalStorageUri).fsPath, WORKSPACE_QUERY_HISTORY_FILE);
 
     this.treeDataProvider = this.push(new HistoryTreeDataProvider(
-      ctx.extensionPath
+      ctx.extensionPath,
+      this.labelProvider
     ));
     this.treeView = this.push(window.createTreeView('codeQLQueryHistory', {
       treeDataProvider: this.treeDataProvider,
       canSelectMany: true,
     }));
 
+    // Forward any change of current history item from the tree data.
+    this.push(this.treeDataProvider.onDidChangeCurrentQueryItem((item) => {
+      this._onDidChangeCurrentQueryItem.fire(item);
+    }));
+
     // Lazily update the tree view selection due to limitations of TreeView API (see
     // `updateTreeViewSelectionIfVisible` doc for details)
     this.push(
@@ -430,6 +456,12 @@ export class QueryHistoryManager extends DisposableObject {
         this.handleShowEvalLogSummary.bind(this)
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.showEvalLogViewer',
+        this.handleShowEvalLogViewer.bind(this)
+      )
+    );
     this.push(
       commandRunner(
         'codeQLQueryHistory.cancel',
@@ -442,6 +474,12 @@ export class QueryHistoryManager extends DisposableObject {
         this.handleShowQueryText.bind(this)
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.exportResults',
+        this.handleExportResults.bind(this)
+      )
+    );
     this.push(
       commandRunner(
         'codeQLQueryHistory.viewCsvResults',
@@ -482,6 +520,12 @@ export class QueryHistoryManager extends DisposableObject {
         }
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.copyRepoList',
+        this.handleCopyRepoList.bind(this)
+      )
+    );
 
     // There are two configuration items that affect the query history:
     // 1. The ttl for query history items.
@@ -490,7 +534,7 @@ export class QueryHistoryManager extends DisposableObject {
     this.push(
       queryHistoryConfigListener.onDidChangeConfiguration(() => {
         this.treeDataProvider.refresh();
-        this.registerQueryHistoryScrubber(queryHistoryConfigListener, ctx);
+        this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
       })
     );
 
@@ -509,13 +553,23 @@ export class QueryHistoryManager extends DisposableObject {
       },
     }));
 
-    this.registerQueryHistoryScrubber(queryHistoryConfigListener, ctx);
+    this.registerQueryHistoryScrubber(queryHistoryConfigListener, this, ctx);
+    this.registerToRemoteQueriesEvents();
+  }
+
+  public completeQuery(info: LocalQueryInfo, results: QueryWithResults): void {
+    info.completeThisQuery(results);
+    this._onDidCompleteQuery.fire(info);
+  }
+
+  private getCredentials() {
+    return Credentials.initialize(this.ctx);
   }
 
   /**
    * Register and create the history scrubber.
    */
-  private registerQueryHistoryScrubber(queryHistoryConfigListener: QueryHistoryConfig, ctx: ExtensionContext) {
+  private registerQueryHistoryScrubber(queryHistoryConfigListener: QueryHistoryConfig, qhm: QueryHistoryManager, ctx: ExtensionContext) {
     this.queryHistoryScrubber?.dispose();
     // Every hour check if we need to re-run the query history scrubber.
     this.queryHistoryScrubber = this.push(
@@ -524,17 +578,61 @@ export class QueryHistoryManager extends DisposableObject {
         TWO_HOURS_IN_MS,
         queryHistoryConfigListener.ttlInMillis,
         this.queryStorageDir,
+        qhm,
         ctx
       )
     );
   }
 
+  private registerToRemoteQueriesEvents() {
+    const queryAddedSubscription = this.remoteQueriesManager.onRemoteQueryAdded(async (event) => {
+      this.addQuery({
+        t: 'remote',
+        status: QueryStatus.InProgress,
+        completed: false,
+        queryId: event.queryId,
+        remoteQuery: event.query,
+      });
+
+      await this.refreshTreeView();
+    });
+
+    const queryRemovedSubscription = this.remoteQueriesManager.onRemoteQueryRemoved(async (event) => {
+      const item = this.treeDataProvider.allHistory.find(i => i.t === 'remote' && i.queryId === event.queryId);
+      if (item) {
+        await this.removeRemoteQuery(item as RemoteQueryHistoryItem);
+      }
+    });
+
+    const queryStatusUpdateSubscription = this.remoteQueriesManager.onRemoteQueryStatusUpdate(async (event) => {
+      const item = this.treeDataProvider.allHistory.find(i => i.t === 'remote' && i.queryId === event.queryId);
+      if (item) {
+        const remoteQueryHistoryItem = item as RemoteQueryHistoryItem;
+        remoteQueryHistoryItem.status = event.status;
+        remoteQueryHistoryItem.failureReason = event.failureReason;
+        remoteQueryHistoryItem.resultCount = event.resultCount;
+        if (event.status === QueryStatus.Completed) {
+          remoteQueryHistoryItem.completed = true;
+        }
+        await this.refreshTreeView();
+      } else {
+        void logger.log('Variant analysis status update event received for unknown variant analysis');
+      }
+    });
+
+    this.push(queryAddedSubscription);
+    this.push(queryRemovedSubscription);
+    this.push(queryStatusUpdateSubscription);
+  }
+
   async readQueryHistory(): Promise<void> {
     void logger.log(`Reading cached query history from '${this.queryMetadataStorageLocation}'.`);
-    const history = await slurpQueryHistory(this.queryMetadataStorageLocation, this.queryHistoryConfigListener);
+    const history = await slurpQueryHistory(this.queryMetadataStorageLocation);
     this.treeDataProvider.allHistory = history;
-    this.treeDataProvider.allHistory.forEach((item) => {
-      this._onDidAddQueryItem.fire(item);
+    this.treeDataProvider.allHistory.forEach(async (item) => {
+      if (item.t === 'remote') {
+        await this.remoteQueriesManager.rehydrateRemoteQuery(item.queryId, item.remoteQuery, item.status);
+      }
     });
   }
 
@@ -578,6 +676,23 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
+  getCurrentQueryHistoryItem(): QueryHistoryInfo | undefined {
+    return this.treeDataProvider.getCurrent();
+  }
+
+  getRemoteQueryById(queryId: string): RemoteQueryHistoryItem | undefined {
+    return this.treeDataProvider.allHistory.find(i => i.t === 'remote' && i.queryId === queryId) as RemoteQueryHistoryItem;
+  }
+
+  async removeDeletedQueries() {
+    await Promise.all(this.treeDataProvider.allHistory.map(async (item) => {
+      if (item.t == 'local' && item.completedQuery && !(await fs.pathExists(item.completedQuery?.query.querySaveDir))) {
+        this.treeDataProvider.remove(item);
+        item.completedQuery?.dispose();
+      }
+    }));
+  }
+
   async handleRemoveHistoryItem(
     singleItem: QueryHistoryInfo,
     multiSelect: QueryHistoryInfo[] = []
@@ -596,26 +711,30 @@ export class QueryHistoryManager extends DisposableObject {
           await item.completedQuery?.query.deleteQuery();
         }
       } else {
-        // Remote queries can be removed locally, but not remotely.
-        // The user must cancel the query on GitHub Actions explicitly.
-        this.treeDataProvider.remove(item);
-        void logger.log(`Deleted ${item.label}.`);
-        if (item.status === QueryStatus.InProgress) {
-          void logger.log('The variant analysis is still running on GitHub Actions. To cancel there, you must go to the workflow run in your browser.');
-        }
-
-        this._onDidRemoveQueryItem.fire(item);
+        await this.removeRemoteQuery(item);
       }
-
     }));
+
     await this.writeQueryHistory();
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
       await this.treeView.reveal(current, { select: true });
-      this._onWillOpenQueryItem.fire(current);
+      await this.openQueryResults(current);
     }
   }
 
+  private async removeRemoteQuery(item: RemoteQueryHistoryItem): Promise<void> {
+    // Remote queries can be removed locally, but not remotely.
+    // The user must cancel the query on GitHub Actions explicitly.
+    this.treeDataProvider.remove(item);
+    void logger.log(`Deleted ${this.labelProvider.getLabel(item)}.`);
+    if (item.status === QueryStatus.InProgress) {
+      void logger.log('The variant analysis is still running on GitHub Actions. To cancel there, you must go to the workflow run in your browser.');
+    }
+
+    await this.remoteQueriesManager.removeRemoteQuery(item.queryId);
+  }
+
   async handleSortByName() {
     if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
       this.treeDataProvider.sortOrder = SortOrder.NameDesc;
@@ -646,20 +765,20 @@ export class QueryHistoryManager extends DisposableObject {
   ): Promise<void> {
     const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
 
-    // TODO will support remote queries
-    if (!this.assertSingleQuery(finalMultiSelect) || finalSingleItem?.t !== 'local') {
+    if (!this.assertSingleQuery(finalMultiSelect)) {
       return;
     }
 
     const response = await window.showInputBox({
-      prompt: 'Label:',
-      placeHolder: '(use default)',
-      value: finalSingleItem.label,
+      placeHolder: `(use default: ${this.queryHistoryConfigListener.format})`,
+      value: finalSingleItem.userSpecifiedLabel ?? '',
+      title: 'Set query label',
+      prompt: 'Set the query history item label. See the description of the codeQL.queryHistory.format setting for more information.',
     });
     // undefined response means the user cancelled the dialog; don't change anything
     if (response !== undefined) {
       // Interpret empty string response as 'go back to using default'
-      finalSingleItem.initialInfo.userSpecifiedLabel = response === '' ? undefined : response;
+      finalSingleItem.userSpecifiedLabel = response === '' ? undefined : response;
       await this.refreshTreeView();
     }
   }
@@ -676,7 +795,7 @@ export class QueryHistoryManager extends DisposableObject {
         throw new Error('Please select a local query.');
       }
 
-      if (!finalSingleItem.completedQuery?.didRunSuccessfully) {
+      if (!finalSingleItem.completedQuery?.sucessful) {
         throw new Error('Please select a query that has completed successfully.');
       }
 
@@ -716,7 +835,7 @@ export class QueryHistoryManager extends DisposableObject {
     } else {
       // show results on single click only if query is completed successfully.
       if (finalSingleItem.status === QueryStatus.Completed) {
-        await this._onWillOpenQueryItem.fire(finalSingleItem);
+        await this.openQueryResults(finalSingleItem);
       }
     }
   }
@@ -741,6 +860,18 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
+  async getQueryHistoryItemDirectory(queryHistoryItem: QueryHistoryInfo): Promise<string> {
+    if (queryHistoryItem.t === 'local') {
+      if (queryHistoryItem.completedQuery) {
+        return queryHistoryItem.completedQuery.query.querySaveDir;
+      }
+    } else if (queryHistoryItem.t === 'remote') {
+      return path.join(this.queryStorageDir, queryHistoryItem.queryId);
+    }
+
+    throw new Error('Unable to get query directory');
+  }
+
   async handleOpenQueryDirectory(
     singleItem: QueryHistoryInfo,
     multiSelect: QueryHistoryInfo[]
@@ -777,14 +908,17 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
-  private warnNoEvalLog() {
-    void showAndLogWarningMessage('No evaluator log is available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG + '?');
+  private warnNoEvalLogs() {
+    void showAndLogWarningMessage(`Evaluator log, summary, and viewer are not available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ' + ${CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG}?`);
   }
 
-  private warnNoEvalLogSummary() {
-    void showAndLogWarningMessage(`No evaluator log summary is available for this run. Perhaps it failed before evaluation, or you are running with a version of CodeQL before ${CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG}?`);
+  private warnInProgressEvalLogSummary() {
+    void showAndLogWarningMessage('The evaluator log summary is still being generated for this run. Please try again later. The summary generation process is tracked in the "CodeQL Extension Log" view.');
   }
 
+  private warnInProgressEvalLogViewer() {
+    void showAndLogWarningMessage('The viewer\'s data is still being generated for this run. Please try again or re-run the query.');
+  }
 
   async handleShowEvalLog(
     singleItem: QueryHistoryInfo,
@@ -800,7 +934,7 @@ export class QueryHistoryManager extends DisposableObject {
     if (finalSingleItem.evalLogLocation) {
       await this.tryOpenExternalFile(finalSingleItem.evalLogLocation);
     } else {
-      this.warnNoEvalLog();
+      this.warnNoEvalLogs();
     }
   }
 
@@ -816,9 +950,42 @@ export class QueryHistoryManager extends DisposableObject {
     }
 
     if (finalSingleItem.evalLogSummaryLocation) {
-        await this.tryOpenExternalFile(finalSingleItem.evalLogSummaryLocation);
+      await this.tryOpenExternalFile(finalSingleItem.evalLogSummaryLocation);
+      return;
+    }
+
+    // Summary log file doesn't exist.
+    if (finalSingleItem.evalLogLocation && await fs.pathExists(finalSingleItem.evalLogLocation)) {
+      // If raw log does exist, then the summary log is still being generated.
+      this.warnInProgressEvalLogSummary();
     } else {
-      this.warnNoEvalLogSummary();
+      this.warnNoEvalLogs();
+    }
+  }
+
+  async handleShowEvalLogViewer(
+    singleItem: QueryHistoryInfo,
+    multiSelect: QueryHistoryInfo[],
+  ) {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+    // Only applicable to an individual local query
+    if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'local') {
+      return;
+    }
+
+    // If the JSON summary file location wasn't saved, display error
+    if (finalSingleItem.jsonEvalLogSummaryLocation == undefined) {
+      this.warnInProgressEvalLogViewer();
+      return;
+    }
+
+    // TODO(angelapwen): Stream the file in.
+    try {
+      const evalLogData: EvalLogData[] = await parseViewerData(finalSingleItem.jsonEvalLogSummaryLocation);
+      const evalLogTreeBuilder = new EvalLogTreeBuilder(finalSingleItem.getQueryName(), evalLogData);
+      this.evalLogViewer.updateRoots(await evalLogTreeBuilder.getRoots());
+    } catch (e) {
+      throw new Error(`Could not read evaluator log summary JSON file to generate viewer data at ${finalSingleItem.jsonEvalLogSummaryLocation}.`);
     }
   }
 
@@ -826,15 +993,22 @@ export class QueryHistoryManager extends DisposableObject {
     singleItem: QueryHistoryInfo,
     multiSelect: QueryHistoryInfo[]
   ) {
-    // Local queries only
-    // In the future, we may support cancelling remote queries, but this is not a short term plan.
     const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
 
-    (finalMultiSelect || [finalSingleItem]).forEach((item) => {
-      if (item.status === QueryStatus.InProgress && item.t === 'local') {
-        item.cancel();
+    const selected = finalMultiSelect || [finalSingleItem];
+    const results = selected.map(async item => {
+      if (item.status === QueryStatus.InProgress) {
+        if (item.t === 'local') {
+          item.cancel();
+        } else if (item.t === 'remote') {
+          void showAndLogInformationMessage('Cancelling variant analysis. This may take a while.');
+          const credentials = await this.getCredentials();
+          await cancelRemoteQuery(credentials, item.remoteQuery);
+        }
       }
     });
+
+    await Promise.all(results);
   }
 
   async handleShowQueryText(
@@ -880,7 +1054,7 @@ export class QueryHistoryManager extends DisposableObject {
         query.resultsPaths.interpretedResultsPath
       );
     } else {
-      const label = finalSingleItem.label;
+      const label = this.labelProvider.getLabel(finalSingleItem);
       void showAndLogInformationMessage(
         `Query ${label} has no interpreted results.`
       );
@@ -902,11 +1076,11 @@ export class QueryHistoryManager extends DisposableObject {
       void this.tryOpenExternalFile(query.csvPath);
       return;
     }
-    await query.exportCsvResults(this.qs, query.csvPath, () => {
+    if (await query.exportCsvResults(this.qs.cliServer, query.csvPath)) {
       void this.tryOpenExternalFile(
         query.csvPath
       );
-    });
+    }
   }
 
   async handleViewCsvAlerts(
@@ -921,7 +1095,7 @@ export class QueryHistoryManager extends DisposableObject {
     }
 
     await this.tryOpenExternalFile(
-      await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs, this.dbm)
+      await finalSingleItem.completedQuery.query.ensureCsvAlerts(this.qs.cliServer, this.dbm)
     );
   }
 
@@ -937,7 +1111,7 @@ export class QueryHistoryManager extends DisposableObject {
     }
 
     await this.tryOpenExternalFile(
-      await finalSingleItem.completedQuery.query.ensureDilPath(this.qs)
+      await finalSingleItem.completedQuery.query.ensureDilPath(this.qs.cliServer)
     );
   }
 
@@ -960,16 +1134,33 @@ export class QueryHistoryManager extends DisposableObject {
     );
   }
 
+  async handleCopyRepoList(
+    singleItem: QueryHistoryInfo,
+    multiSelect: QueryHistoryInfo[],
+  ) {
+    const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
+
+    // Remote queries only
+    if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem || finalSingleItem.t !== 'remote') {
+      return;
+    }
+
+    await commands.executeCommand('codeQL.copyRepoList', finalSingleItem.queryId);
+  }
+
   async getQueryText(item: QueryHistoryInfo): Promise<string> {
     return item.t === 'local'
       ? item.initialInfo.queryText
       : item.remoteQuery.queryText;
   }
 
+  async handleExportResults(): Promise<void> {
+    await commands.executeCommand('codeQL.exportVariantAnalysisResults');
+  }
+
   addQuery(item: QueryHistoryInfo) {
     this.treeDataProvider.pushQuery(item);
     this.updateTreeViewSelectionIfVisible();
-    this._onDidAddQueryItem.fire(item);
   }
 
   /**
@@ -1045,7 +1236,7 @@ the file in the file explorer and dragging it into the workspace.`
       if (!otherQuery.completedQuery) {
         throw new Error('Please select a completed query.');
       }
-      if (!otherQuery.completedQuery.didRunSuccessfully) {
+      if (!otherQuery.completedQuery.sucessful) {
         throw new Error('Please select a successful query.');
       }
       if (otherQuery.initialInfo.databaseInfo.name !== dbName) {
@@ -1065,11 +1256,11 @@ the file in the file explorer and dragging it into the workspace.`
           otherQuery !== singleItem &&
           otherQuery.t === 'local' &&
           otherQuery.completedQuery &&
-          otherQuery.completedQuery.didRunSuccessfully &&
+          otherQuery.completedQuery.sucessful &&
           otherQuery.initialInfo.databaseInfo.name === dbName
       )
       .map((item) => ({
-        label: item.label,
+        label: this.labelProvider.getLabel(item),
         description: (item as CompletedLocalQueryInfo).initialInfo.databaseInfo.name,
         detail: (item as CompletedLocalQueryInfo).completedQuery.statusString,
         query: item as CompletedLocalQueryInfo,
@@ -1170,4 +1361,13 @@ the file in the file explorer and dragging it into the workspace.`
     this.treeDataProvider.refresh();
     await this.writeQueryHistory();
   }
+
+  private async openQueryResults(item: QueryHistoryInfo) {
+    if (item.t === 'local') {
+      await this.localQueriesResultsView.showResults(item as CompletedLocalQueryInfo, WebviewReveal.Forced, false);
+    }
+    else if (item.t === 'remote') {
+      await this.remoteQueriesManager.openRemoteQueryResults(item.queryId);
+    }
+  }
 }

extensions/ql-vscode/src/query-results.ts

@@ -1,7 +1,7 @@
 import { CancellationTokenSource, env } from 'vscode';
 
-import { QueryWithResults, QueryEvaluationInfo } from './run-queries';
-import * as messages from './pure/messages';
+import * as messages from './pure/messages-shared';
+import * as legacyMessages from './pure/legacy-messages';
 import * as cli from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
@@ -14,10 +14,11 @@ import {
   SarifInterpretationData,
   GraphInterpretationData
 } from './pure/interface-types';
-import { QueryHistoryConfig } from './config';
 import { DatabaseInfo } from './pure/interface-types';
 import { QueryStatus } from './query-status';
 import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
+import { QueryEvaluationInfo, QueryWithResults } from './run-queries-shared';
+import { formatLegacyMessage } from './legacy-query-server/run-queries';
 
 /**
  * query-results.ts
@@ -45,7 +46,12 @@ export interface InitialQueryInfo {
 
 export class CompletedQueryInfo implements QueryWithResults {
   readonly query: QueryEvaluationInfo;
-  readonly result: messages.EvaluationResult;
+  readonly message?: string;
+  readonly sucessful?: boolean;
+  /**
+   * The legacy result. This is only set when loading from the query history.
+   */
+  readonly result: legacyMessages.EvaluationResult;
   readonly logFileLocation?: string;
   resultCount: number;
 
@@ -69,16 +75,18 @@ export class CompletedQueryInfo implements QueryWithResults {
   interpretedResultsSortState: InterpretedResultsSortState | undefined;
 
   /**
-   * Note that in the {@link FullQueryInfo.slurp} method, we create a CompletedQueryInfo instance
+   * Note that in the {@link slurpQueryHistory} method, we create a CompletedQueryInfo instance
    * by explicitly setting the prototype in order to avoid calling this constructor.
    */
   constructor(
     evaluation: QueryWithResults,
   ) {
     this.query = evaluation.query;
-    this.result = evaluation.result;
     this.logFileLocation = evaluation.logFileLocation;
+    this.result = evaluation.result;
 
+    this.message = evaluation.message;
+    this.sucessful = evaluation.sucessful;
     // Use the dispose method from the evaluation.
     // The dispose will clean up any additional log locations that this
     // query may have created.
@@ -93,18 +101,12 @@ export class CompletedQueryInfo implements QueryWithResults {
   }
 
   get statusString(): string {
-    switch (this.result.resultType) {
-      case messages.QueryResultType.CANCELLATION:
-        return `cancelled after ${Math.round(this.result.evaluationTime / 1000)} seconds`;
-      case messages.QueryResultType.OOM:
-        return 'out of memory';
-      case messages.QueryResultType.SUCCESS:
-        return `finished in ${Math.round(this.result.evaluationTime / 1000)} seconds`;
-      case messages.QueryResultType.TIMEOUT:
-        return `timed out after ${Math.round(this.result.evaluationTime / 1000)} seconds`;
-      case messages.QueryResultType.OTHER_ERROR:
-      default:
-        return this.result.message ? `failed: ${this.result.message}` : 'failed';
+    if (this.message) {
+      return this.message;
+    } else if (this.result) {
+      return formatLegacyMessage(this.result);
+    } else {
+      throw new Error('No status available');
     }
   }
 
@@ -116,10 +118,6 @@ export class CompletedQueryInfo implements QueryWithResults {
       || this.query.resultsPaths.resultsPath;
   }
 
-  get didRunSuccessfully(): boolean {
-    return this.result.resultType === messages.QueryResultType.SUCCESS;
-  }
-
   async updateSortState(
     server: cli.CodeQLCliServer,
     resultSetName: string,
@@ -218,7 +216,8 @@ export class LocalQueryInfo {
   public completedQuery: CompletedQueryInfo | undefined;
   public evalLogLocation: string | undefined;
   public evalLogSummaryLocation: string | undefined;
-  private config: QueryHistoryConfig | undefined;
+  public jsonEvalLogSummaryLocation: string | undefined;
+  public evalLogSummarySymbolsLocation: string | undefined;
 
   /**
    * Note that in the {@link slurpQueryHistory} method, we create a FullQueryInfo instance
@@ -226,11 +225,8 @@ export class LocalQueryInfo {
    */
   constructor(
     public readonly initialInfo: InitialQueryInfo,
-    config: QueryHistoryConfig,
     private cancellationSource?: CancellationTokenSource // used to cancel in progress queries
-  ) {
-    this.setConfig(config);
-  }
+  ) { /**/ }
 
   cancel() {
     this.cancellationSource?.cancel();
@@ -243,43 +239,12 @@ export class LocalQueryInfo {
     return this.initialInfo.start.toLocaleString(env.language);
   }
 
-  interpolate(template: string): string {
-    const { resultCount = 0, statusString = 'in progress' } = this.completedQuery || {};
-    const replacements: { [k: string]: string } = {
-      t: this.startTime,
-      q: this.getQueryName(),
-      d: this.initialInfo.databaseInfo.name,
-      r: resultCount.toString(),
-      s: statusString,
-      f: this.getQueryFileName(),
-      '%': '%',
-    };
-    return template.replace(/%(.)/g, (match, key) => {
-      const replacement = replacements[key];
-      return replacement !== undefined ? replacement : match;
-    });
+  get userSpecifiedLabel() {
+    return this.initialInfo.userSpecifiedLabel;
   }
 
-  /**
-   * Returns a label for this query that includes interpolated values.
-   */
-  get label(): string {
-    return this.interpolate(
-      this.initialInfo.userSpecifiedLabel ?? this.config?.format ?? ''
-    );
-  }
-
-  /**
-   * Avoids getting the default label for the query.
-   * If there is a custom label for this query, interpolate and use that.
-   * Otherwise, use the name of the query.
-   *
-   * @returns the name of the query, unless there is a custom label for this query.
-   */
-  getShortLabel(): string {
-    return this.initialInfo.userSpecifiedLabel
-      ? this.interpolate(this.initialInfo.userSpecifiedLabel)
-      : this.getQueryName();
+  set userSpecifiedLabel(label: string | undefined) {
+    this.initialInfo.userSpecifiedLabel = label;
   }
 
   /**
@@ -317,7 +282,7 @@ export class LocalQueryInfo {
     return !!this.completedQuery;
   }
 
-  completeThisQuery(info: QueryWithResults) {
+  completeThisQuery(info: QueryWithResults): void {
     this.completedQuery = new CompletedQueryInfo(info);
 
     // dispose of the cancellation token source and also ensure the source is not serialized as JSON
@@ -336,27 +301,10 @@ export class LocalQueryInfo {
       return QueryStatus.Failed;
     } else if (!this.completedQuery) {
       return QueryStatus.InProgress;
-    } else if (this.completedQuery.didRunSuccessfully) {
+    } else if (this.completedQuery.sucessful) {
       return QueryStatus.Completed;
     } else {
       return QueryStatus.Failed;
     }
   }
-
-  /**
-   * The `config` property must not be serialized since it contains a listerner
-   * for global configuration changes. Instead, It should be set when the query
-   * is deserialized.
-   *
-   * @param config the global query history config object
-   */
-  setConfig(config: QueryHistoryConfig) {
-    // avoid serializing config property
-    Object.defineProperty(this, 'config', {
-      enumerable: false,
-      writable: false,
-      configurable: true,
-      value: config
-    });
-  }
 }

extensions/ql-vscode/src/query-serialization.ts

@@ -1,13 +1,13 @@
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
-import { QueryHistoryConfig } from './config';
 import { showAndLogErrorMessage } from './helpers';
 import { asyncFilter, getErrorMessage, getErrorStack } from './pure/helpers-pure';
 import { CompletedQueryInfo, LocalQueryInfo, QueryHistoryInfo } from './query-results';
-import { QueryEvaluationInfo } from './run-queries';
+import { QueryStatus } from './query-status';
+import { QueryEvaluationInfo } from './run-queries-shared';
 
-export async function slurpQueryHistory(fsPath: string, config: QueryHistoryConfig): Promise<QueryHistoryInfo[]> {
+export async function slurpQueryHistory(fsPath: string): Promise<QueryHistoryInfo[]> {
   try {
     if (!(await fs.pathExists(fsPath))) {
       return [];
@@ -29,10 +29,6 @@ export async function slurpQueryHistory(fsPath: string, config: QueryHistoryConf
       if (q.t === 'local') {
         Object.setPrototypeOf(q, LocalQueryInfo.prototype);
 
-        // The config object is a global, se we need to set it explicitly
-        // and ensure it is not serialized to JSON.
-        q.setConfig(config);
-
         // Date instances are serialized as strings. Need to
         // convert them back to Date instances.
         (q.initialInfo as any).start = new Date(q.initialInfo.start);
@@ -44,7 +40,12 @@ export async function slurpQueryHistory(fsPath: string, config: QueryHistoryConf
           q.completedQuery.dispose = () => { /**/ };
         }
       } else if (q.t === 'remote') {
-        // noop
+        // A bug was introduced that didn't set the completed flag in query history
+        // items. The following code makes sure that the flag is set in order to
+        // "patch" older query history items.
+        if (q.status === QueryStatus.Completed) {
+          q.completed = true;
+        }
       }
       return q;
     });

extensions/ql-vscode/src/query-server/query-runner.ts

@@ -0,0 +1,81 @@
+import { CancellationToken } from 'vscode';
+import { ProgressCallback, UserCancellationException } from '../commandRunner';
+import { DatabaseItem } from '../databases';
+import { clearCache, ClearCacheParams, clearPackCache, deregisterDatabases, registerDatabases, upgradeDatabase } from '../pure/new-messages';
+import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
+import { QueryRunner } from '../queryRunner';
+import { QueryWithResults } from '../run-queries-shared';
+import { QueryServerClient } from './queryserver-client';
+import { compileAndRunQueryAgainstDatabase } from './run-queries';
+import * as vscode from 'vscode';
+import { getOnDiskWorkspaceFolders } from '../helpers';
+export class NewQueryRunner extends QueryRunner {
+
+
+  constructor(public readonly qs: QueryServerClient) {
+    super();
+  }
+
+  get cliServer() {
+    return this.qs.cliServer;
+  }
+
+  async restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    await this.qs.restartQueryServer(progress, token);
+  }
+
+  onStart(callBack: (progress: ProgressCallback, token: CancellationToken) => Promise<void>) {
+    this.qs.onDidStartQueryServer(callBack);
+  }
+
+  async clearCacheInDatabase(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
+    if (dbItem.contents === undefined) {
+      throw new Error('Can\'t clear the cache in an invalid database.');
+    }
+
+    const db = dbItem.databaseUri.fsPath;
+    const params: ClearCacheParams = {
+      dryRun: false,
+      db,
+    };
+    await this.qs.sendRequest(clearCache, params, token, progress);
+  }
+  async compileAndRunQueryAgainstDatabase(dbItem: DatabaseItem, initialInfo: InitialQueryInfo, queryStorageDir: string, progress: ProgressCallback, token: CancellationToken, templates?: Record<string, string>, queryInfo?: LocalQueryInfo): Promise<QueryWithResults> {
+    return await compileAndRunQueryAgainstDatabase(this.qs.cliServer, this.qs, dbItem, initialInfo, queryStorageDir, progress, token, templates, queryInfo);
+  }
+
+  async deregisterDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
+    if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: string[] = [dbItem.databaseUri.fsPath];
+      await this.qs.sendRequest(deregisterDatabases, { databases }, token, progress);
+    }
+  }
+  async registerDatabase(progress: ProgressCallback, token: CancellationToken, dbItem: DatabaseItem): Promise<void> {
+    if (dbItem.contents && (await this.qs.cliServer.cliConstraints.supportsDatabaseRegistration())) {
+      const databases: string[] = [dbItem.databaseUri.fsPath];
+      await this.qs.sendRequest(registerDatabases, { databases }, token, progress);
+    }
+  }
+
+  async clearPackCache(): Promise<void> {
+    await this.qs.sendRequest(clearPackCache, {});
+  }
+
+  async upgradeDatabaseExplicit(dbItem: DatabaseItem, progress: ProgressCallback, token: CancellationToken): Promise<void> {
+
+    const yesItem = { title: 'Yes', isCloseAffordance: false };
+    const noItem = { title: 'No', isCloseAffordance: true };
+    const dialogOptions: vscode.MessageItem[] = [yesItem, noItem];
+
+
+
+    const message = `Should the database ${dbItem.databaseUri.fsPath} be destructively upgraded?\n\nThis should not be necessary to run queries 
+    as we will non-destructively update it anyway.`;
+    const chosenItem = await vscode.window.showInformationMessage(message, { modal: true }, ...dialogOptions);
+
+    if (chosenItem !== yesItem) {
+      throw new UserCancellationException('User cancelled the database upgrade.');
+    }
+    await this.qs.sendRequest(upgradeDatabase, { db: dbItem.databaseUri.fsPath, additionalPacks: getOnDiskWorkspaceFolders() }, token, progress);
+  }
+}

extensions/ql-vscode/src/query-server/queryserver-client.ts

@@ -0,0 +1,205 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+
+import { DisposableObject } from '../pure/disposable-object';
+import { CancellationToken, commands } from 'vscode';
+import { createMessageConnection, RequestType } from 'vscode-jsonrpc';
+import * as cli from '../cli';
+import { QueryServerConfig } from '../config';
+import { Logger, ProgressReporter } from '../logging';
+import { progress, ProgressMessage, WithProgressId } from '../pure/new-messages';
+import * as messages from '../pure/new-messages';
+import { ProgressCallback, ProgressTask } from '../commandRunner';
+import { findQueryLogFile } from '../run-queries-shared';
+import { ServerProcess } from '../json-rpc-server';
+
+type ServerOpts = {
+  logger: Logger;
+  contextStoragePath: string;
+}
+
+
+type WithProgressReporting = (task: (progress: ProgressReporter, token: CancellationToken) => Thenable<void>) => Thenable<void>;
+
+/**
+ * Client that manages a query server process.
+ * The server process is started upon initialization and tracked during its lifetime.
+ * The server process is disposed when the client is disposed, or if the client asks
+ * to restart it (which disposes the existing process and starts a new one).
+ */
+export class QueryServerClient extends DisposableObject {
+
+  serverProcess?: ServerProcess;
+  progressCallbacks: { [key: number]: ((res: ProgressMessage) => void) | undefined };
+  nextCallback: number;
+  nextProgress: number;
+  withProgressReporting: WithProgressReporting;
+
+  private readonly queryServerStartListeners = [] as ProgressTask<void>[];
+
+  // Can't use standard vscode EventEmitter here since they do not cause the calling
+  // function to fail if one of the event handlers fail. This is something that
+  // we need here.
+  readonly onDidStartQueryServer = (e: ProgressTask<void>) => {
+    this.queryServerStartListeners.push(e);
+  }
+
+  public activeQueryLogFile: string | undefined;
+
+  constructor(
+    readonly config: QueryServerConfig,
+    readonly cliServer: cli.CodeQLCliServer,
+    readonly opts: ServerOpts,
+    withProgressReporting: WithProgressReporting
+  ) {
+    super();
+    // When the query server configuration changes, restart the query server.
+    if (config.onDidChangeConfiguration !== undefined) {
+      this.push(config.onDidChangeConfiguration(() =>
+        commands.executeCommand('codeQL.restartQueryServer')));
+    }
+    this.withProgressReporting = withProgressReporting;
+    this.nextCallback = 0;
+    this.nextProgress = 0;
+    this.progressCallbacks = {};
+  }
+
+  get logger(): Logger {
+    return this.opts.logger;
+  }
+
+  /** Stops the query server by disposing of the current server process. */
+  private stopQueryServer(): void {
+    if (this.serverProcess !== undefined) {
+      this.disposeAndStopTracking(this.serverProcess);
+    } else {
+      void this.logger.log('No server process to be stopped.');
+    }
+  }
+
+  /** Restarts the query server by disposing of the current server process and then starting a new one. */
+  async restartQueryServer(
+    progress: ProgressCallback,
+    token: CancellationToken
+  ): Promise<void> {
+    this.stopQueryServer();
+    await this.startQueryServer();
+
+    // Ensure we await all responses from event handlers so that
+    // errors can be properly reported to the user.
+    await Promise.all(this.queryServerStartListeners.map(handler => handler(
+      progress,
+      token
+    )));
+  }
+
+  showLog(): void {
+    this.logger.show();
+  }
+
+  /** Starts a new query server process, sending progress messages to the status bar. */
+  async startQueryServer(): Promise<void> {
+    // Use an arrow function to preserve the value of `this`.
+    return this.withProgressReporting((progress, _) => this.startQueryServerImpl(progress));
+  }
+
+  /** Starts a new query server process, sending progress messages to the given reporter. */
+  private async startQueryServerImpl(progressReporter: ProgressReporter): Promise<void> {
+    void this.logger.log('Starting NEW query server.');
+
+    const ramArgs = await this.cliServer.resolveRam(this.config.queryMemoryMb, progressReporter);
+    const args = ['--threads', this.config.numThreads.toString()].concat(ramArgs);
+
+    if (this.config.saveCache) {
+      args.push('--save-cache');
+    }
+
+    if (this.config.cacheSize > 0) {
+      args.push('--max-disk-cache');
+      args.push(this.config.cacheSize.toString());
+    }
+
+    const structuredLogFile = `${this.opts.contextStoragePath}/structured-evaluator-log.json`;
+    await fs.ensureFile(structuredLogFile);
+
+    args.push('--evaluator-log');
+    args.push(structuredLogFile);
+
+    // We hard-code the verbosity level to 5 and minify to false.
+    // This will be the behavior of the per-query structured logging in the CLI after 2.8.3.
+    args.push('--evaluator-log-level');
+    args.push('5');
+
+
+    if (this.config.debug) {
+      args.push('--debug', '--tuple-counting');
+    }
+
+    if (cli.shouldDebugQueryServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9010,server=y,suspend=y,quiet=y');
+    }
+
+    const child = cli.spawnServer(
+      this.config.codeQlPath,
+      'CodeQL query server',
+      ['execute', 'query-server2'],
+      args,
+      this.logger,
+      data => this.logger.log(data.toString(), {
+        trailingNewline: false,
+        additionalLogLocation: this.activeQueryLogFile
+      }),
+      undefined, // no listener for stdout
+      progressReporter
+    );
+    progressReporter.report({ message: 'Connecting to CodeQL query server' });
+    const connection = createMessageConnection(child.stdout, child.stdin);
+    connection.onNotification(progress, res => {
+      const callback = this.progressCallbacks[res.id];
+      if (callback) {
+        callback(res);
+      }
+    });
+    this.serverProcess = new ServerProcess(child, connection, 'Query Server 2', this.logger);
+    // Ensure the server process is disposed together with this client.
+    this.track(this.serverProcess);
+    connection.listen();
+    progressReporter.report({ message: 'Connected to CodeQL query server v2' });
+    this.nextCallback = 0;
+    this.nextProgress = 0;
+    this.progressCallbacks = {};
+  }
+
+  get serverProcessPid(): number {
+    return this.serverProcess!.child.pid || 0;
+  }
+
+  async sendRequest<P, R, E, RO>(type: RequestType<WithProgressId<P>, R, E, RO>, parameter: P, token?: CancellationToken, progress?: (res: ProgressMessage) => void): Promise<R> {
+    const id = this.nextProgress++;
+    this.progressCallbacks[id] = progress;
+
+    this.updateActiveQuery(type.method, parameter);
+    try {
+      if (this.serverProcess === undefined) {
+        throw new Error('No query server process found.');
+      }
+      return await this.serverProcess.connection.sendRequest(type, { body: parameter, progressId: id }, token);
+    } finally {
+      delete this.progressCallbacks[id];
+    }
+  }
+
+  /**
+   * Updates the active query every time there is a new request to compile.
+   * The active query is used to specify the side log.
+   *
+   * This isn't ideal because in situations where there are queries running
+   * in parallel, each query's log messages are interleaved. Fixing this
+   * properly will require a change in the query server.
+   */
+  private updateActiveQuery(method: string, parameter: any): void {
+    if (method === messages.runQuery.method) {
+      this.activeQueryLogFile = findQueryLogFile(path.dirname(path.dirname((parameter as messages.RunQueryParams).outputPath)));
+    }
+  }
+}

extensions/ql-vscode/src/query-server/run-queries.ts

@@ -0,0 +1,143 @@
+import * as path from 'path';
+import {
+  CancellationToken
+} from 'vscode';
+import * as cli from '../cli';
+import { ProgressCallback } from '../commandRunner';
+import { DatabaseItem } from '../databases';
+import {
+  getOnDiskWorkspaceFolders,
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+  tryGetQueryMetadata
+} from '../helpers';
+import { logger } from '../logging';
+import * as messages from '../pure/new-messages';
+import * as legacyMessages from '../pure/legacy-messages';
+import { InitialQueryInfo, LocalQueryInfo } from '../query-results';
+import { QueryEvaluationInfo, QueryWithResults } from '../run-queries-shared';
+import * as qsClient from './queryserver-client';
+
+
+/**
+ * run-queries.ts
+ * --------------
+ *
+ * Compiling and running QL queries.
+ */
+
+
+/**
+ * A collection of evaluation-time information about a query,
+ * including the query itself, and where we have decided to put
+ * temporary files associated with it, such as the compiled query
+ * output and results.
+ */
+
+export async function compileAndRunQueryAgainstDatabase(
+  cliServer: cli.CodeQLCliServer,
+  qs: qsClient.QueryServerClient,
+  dbItem: DatabaseItem,
+  initialInfo: InitialQueryInfo,
+  queryStorageDir: string,
+  progress: ProgressCallback,
+  token: CancellationToken,
+  templates?: Record<string, string>,
+  queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
+): Promise<QueryWithResults> {
+  if (!dbItem.contents || !dbItem.contents.dbSchemeUri) {
+    throw new Error(`Database ${dbItem.databaseUri} does not have a CodeQL database scheme.`);
+  }
+
+  // Read the query metadata if possible, to use in the UI.
+  const metadata = await tryGetQueryMetadata(cliServer, initialInfo.queryPath);
+
+  const hasMetadataFile = (await dbItem.hasMetadataFile());
+  const query = new QueryEvaluationInfo(
+    path.join(queryStorageDir, initialInfo.id),
+    dbItem.databaseUri.fsPath,
+    hasMetadataFile,
+    initialInfo.quickEvalPosition,
+    metadata,
+  );
+
+  if (!dbItem.contents || dbItem.error) {
+    throw new Error('Can\'t run query on invalid database.');
+  }
+  const target = query.quickEvalPosition ? {
+    quickEval: { quickEvalPos: query.quickEvalPosition }
+  } : { query: {} };
+
+  const diskWorkspaceFolders = getOnDiskWorkspaceFolders();
+  const db = dbItem.databaseUri.fsPath;
+  const logPath = queryInfo ? query.evalLogPath : undefined;
+  const queryToRun: messages.RunQueryParams = {
+    db,
+    additionalPacks: diskWorkspaceFolders,
+    externalInputs: {},
+    singletonExternalInputs: templates || {},
+    outputPath: query.resultsPaths.resultsPath,
+    queryPath: initialInfo.queryPath,
+    logPath,
+    target,
+  };
+  await query.createTimestampFile();
+  let result: messages.RunQueryResult | undefined;
+  try {
+    result = await qs.sendRequest(messages.runQuery, queryToRun, token, progress);
+    if (qs.config.customLogDirectory) {
+      void showAndLogWarningMessage(
+        `Custom log directories are no longer supported. The "codeQL.runningQueries.customLogDirectory" setting is deprecated. Unset the setting to stop seeing this message. Query logs saved to ${query.logPath}.`
+      );
+    }
+  } finally {
+    if (queryInfo) {
+      if (await query.hasEvalLog()) {
+        await query.addQueryLogs(queryInfo, qs.cliServer, qs.logger);
+      } else {
+        void showAndLogWarningMessage(`Failed to write structured evaluator log to ${query.evalLogPath}.`);
+      }
+    }
+  }
+
+  if (result.resultType !== messages.QueryResultType.SUCCESS) {
+    const message = result.message || 'Failed to run query';
+    void logger.log(message);
+    void showAndLogErrorMessage(message);
+  }
+  let message;
+  switch (result.resultType) {
+    case messages.QueryResultType.CANCELLATION:
+      message = `cancelled after ${Math.round(result.evaluationTime / 1000)} seconds`;
+      break;
+    case messages.QueryResultType.OOM:
+      message = 'out of memory';
+      break;
+    case messages.QueryResultType.SUCCESS:
+      message = `finished in ${Math.round(result.evaluationTime / 1000)} seconds`;
+      break;
+    case messages.QueryResultType.COMPILATION_ERROR:
+      message = `compilation failed: ${result.message}`;
+      break;
+    case messages.QueryResultType.OTHER_ERROR:
+    default:
+      message = result.message ? `failed: ${result.message}` : 'failed';
+      break;
+  }
+  const sucessful = result.resultType === messages.QueryResultType.SUCCESS;
+  return {
+    query,
+    result: {
+      evaluationTime: result.evaluationTime,
+      queryId: 0,
+      resultType: sucessful ? legacyMessages.QueryResultType.SUCCESS : legacyMessages.QueryResultType.OTHER_ERROR,
+      runId: 0,
+      message
+    },
+    message,
+    sucessful,
+    dispose: () => {
+      qs.logger.removeAdditionalLogLocation(undefined);
+    }
+  };
+}

extensions/ql-vscode/src/queryRunner.ts

@@ -0,0 +1,50 @@
+import { CancellationToken } from 'vscode';
+import { CodeQLCliServer } from './cli';
+import { ProgressCallback } from './commandRunner';
+import { DatabaseItem } from './databases';
+import { InitialQueryInfo, LocalQueryInfo } from './query-results';
+import { QueryWithResults } from './run-queries-shared';
+
+
+
+export abstract class QueryRunner {
+  abstract restartQueryServer(progress: ProgressCallback, token: CancellationToken): Promise<void>;
+
+  abstract cliServer: CodeQLCliServer;
+
+  abstract onStart(arg0: (progress: ProgressCallback, token: CancellationToken) => Promise<void>): void;
+  abstract clearCacheInDatabase(
+    dbItem: DatabaseItem,
+    progress: ProgressCallback,
+    token: CancellationToken): Promise<void>;
+
+  abstract compileAndRunQueryAgainstDatabase(
+    dbItem: DatabaseItem,
+    initialInfo: InitialQueryInfo,
+    queryStorageDir: string,
+    progress: ProgressCallback,
+    token: CancellationToken,
+    templates?: Record<string, string>,
+    queryInfo?: LocalQueryInfo, // May be omitted for queries not initiated by the user. If omitted we won't create a structured log for the query.
+  ): Promise<QueryWithResults>;
+
+  abstract deregisterDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    dbItem: DatabaseItem,
+  ): Promise<void>;
+
+  abstract registerDatabase(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    dbItem: DatabaseItem,
+  ): Promise<void>;
+
+  abstract upgradeDatabaseExplicit(
+    dbItem: DatabaseItem,
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void>
+
+  abstract clearPackCache(): Promise<void>
+}

extensions/ql-vscode/src/quick-query.ts

@@ -121,15 +121,22 @@ export async function displayQuickQuery(
       const quickQueryQlpackYaml: any = {
         name: 'vscode/quick-query',
         version: '1.0.0',
-        libraryPathDependencies: [qlpack]
+        dependencies: {
+          [qlpack]: '*'
+        }
       };
-      await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.safeDump(quickQueryQlpackYaml), 'utf8');
+      await fs.writeFile(qlPackFile, QLPACK_FILE_HEADER + yaml.dump(quickQueryQlpackYaml), 'utf8');
     }
 
     if (shouldRewrite || !(await fs.pathExists(qlFile))) {
       await fs.writeFile(qlFile, getInitialQueryContents(dbItem.language, dbscheme), 'utf8');
     }
 
+    if (shouldRewrite) {
+      await cliServer.clearCache();
+      await cliServer.packInstall(queriesDir, true);
+    }
+
     await Window.showTextDocument(await workspace.openTextDocument(qlFile));
   } catch (e) {
     if (e instanceof ResponseError && e.code == ErrorCodes.RequestCancelled) {
@@ -144,6 +151,6 @@ async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
   if (!(await fs.pathExists(qlPackFile))) {
     return true;
   }
-  const qlPackContents: any = yaml.safeLoad(await fs.readFile(qlPackFile, 'utf8'));
-  return qlPackContents.libraryPathDependencies?.[0] !== newDependency;
+  const qlPackContents: any = yaml.load(await fs.readFile(qlPackFile, 'utf8'));
+  return !qlPackContents.dependencies?.[newDependency];
 }

extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts

@@ -5,7 +5,7 @@ import { CancellationToken, ExtensionContext } from 'vscode';
 
 import { Credentials } from '../authentication';
 import { Logger } from '../logging';
-import { downloadArtifactFromLink } from './gh-actions-api-client';
+import { downloadArtifactFromLink } from './gh-api/gh-actions-api-client';
 import { AnalysisSummary } from './shared/remote-query-result';
 import { AnalysisResults, AnalysisAlert, AnalysisRawResults } from './shared/analysis-result';
 import { UserCancellationException } from '../commandRunner';
@@ -116,7 +116,10 @@ export class AnalysesResultsManager {
     const analysisResults: AnalysisResults = {
       nwo: analysis.nwo,
       status: 'InProgress',
-      interpretedResults: []
+      interpretedResults: [],
+      resultCount: analysis.resultCount,
+      starCount: analysis.starCount,
+      lastUpdated: analysis.lastUpdated,
     };
     const queryId = analysis.downloadLink.queryId;
     const resultsForQuery = this.internalGetAnalysesResults(queryId);
@@ -145,7 +148,7 @@ export class AnalysesResultsManager {
         status: 'Completed'
       };
     } else if (fileExtension === '.bqrs') {
-      const queryResults = await this.readBqrsResults(artifactPath, fileLinkPrefix);
+      const queryResults = await this.readBqrsResults(artifactPath, fileLinkPrefix, analysis.sourceLocationPrefix);
       newAnaysisResults = {
         ...analysisResults,
         rawResults: queryResults,
@@ -177,8 +180,8 @@ export class AnalysesResultsManager {
     return await fs.pathExists(createDownloadPath(this.storagePath, analysis.downloadLink));
   }
 
-  private async readBqrsResults(filePath: string, fileLinkPrefix: string): Promise<AnalysisRawResults> {
-    return await extractRawResults(this.cliServer, this.logger, filePath, fileLinkPrefix);
+  private async readBqrsResults(filePath: string, fileLinkPrefix: string, sourceLocationPrefix: string): Promise<AnalysisRawResults> {
+    return await extractRawResults(this.cliServer, this.logger, filePath, fileLinkPrefix, sourceLocationPrefix);
   }
 
   private async readSarifResults(filePath: string, fileLinkPrefix: string): Promise<AnalysisAlert[]> {

extensions/ql-vscode/src/remote-queries/bqrs-processing.ts

@@ -9,6 +9,7 @@ export async function extractRawResults(
   logger: Logger,
   filePath: string,
   fileLinkPrefix: string,
+  sourceLocationPrefix: string
 ): Promise<AnalysisRawResults> {
   const bqrsInfo = await cliServer.bqrsInfo(filePath);
   const resultSets = bqrsInfo['result-sets'];
@@ -31,5 +32,5 @@ export async function extractRawResults(
 
   const capped = !!chunk.next;
 
-  return { schema, resultSet, fileLinkPrefix, capped };
+  return { schema, resultSet, fileLinkPrefix, sourceLocationPrefix, capped };
 }

extensions/ql-vscode/src/remote-queries/export-results.ts

@@ -0,0 +1,158 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+
+import { window, commands, Uri, ExtensionContext, QuickPickItem, workspace, ViewColumn } from 'vscode';
+import { Credentials } from '../authentication';
+import { UserCancellationException } from '../commandRunner';
+import {
+  showInformationMessageWithAction,
+  pluralize
+} from '../helpers';
+import { logger } from '../logging';
+import { QueryHistoryManager } from '../query-history';
+import { createGist } from './gh-api/gh-actions-api-client';
+import { RemoteQueriesManager } from './remote-queries-manager';
+import { generateMarkdown } from './remote-queries-markdown-generation';
+import { RemoteQuery } from './remote-query';
+import { AnalysisResults, sumAnalysesResults } from './shared/analysis-result';
+import { RemoteQueryHistoryItem } from './remote-query-history-item';
+
+/**
+ * Exports the results of the given or currently-selected remote query.
+ * The user is prompted to select the export format.
+ */
+export async function exportRemoteQueryResults(
+  queryHistoryManager: QueryHistoryManager,
+  remoteQueriesManager: RemoteQueriesManager,
+  ctx: ExtensionContext,
+  queryId?: string,
+): Promise<void> {
+  let queryHistoryItem: RemoteQueryHistoryItem;
+  if (queryId) {
+    const query = queryHistoryManager.getRemoteQueryById(queryId);
+    if (!query) {
+      void logger.log(`Could not find query with id ${queryId}`);
+      throw new Error('There was an error when trying to retrieve variant analysis information');
+    }
+    queryHistoryItem = query;
+  } else {
+    const query = queryHistoryManager.getCurrentQueryHistoryItem();
+    if (!query || query.t !== 'remote') {
+      throw new Error('No variant analysis results currently open. To open results, click an item in the query history view.');
+    }
+    queryHistoryItem = query;
+  }
+
+  if (!queryHistoryItem.completed) {
+    throw new Error('Variant analysis results are not yet available.');
+  }
+
+  void logger.log(`Exporting variant analysis results for query: ${queryHistoryItem.queryId}`);
+  const query = queryHistoryItem.remoteQuery;
+  const analysesResults = remoteQueriesManager.getAnalysesResults(queryHistoryItem.queryId);
+
+  const gistOption = {
+    label: '$(ports-open-browser-icon) Create Gist (GitHub)',
+  };
+  const localMarkdownOption = {
+    label: '$(markdown) Save as markdown',
+  };
+  const exportFormat = await determineExportFormat(gistOption, localMarkdownOption);
+
+  if (exportFormat === gistOption) {
+    await exportResultsToGist(ctx, query, analysesResults);
+  } else if (exportFormat === localMarkdownOption) {
+    const queryDirectoryPath = await queryHistoryManager.getQueryHistoryItemDirectory(
+      queryHistoryItem
+    );
+    await exportResultsToLocalMarkdown(queryDirectoryPath, query, analysesResults);
+  }
+}
+
+/**
+ * Determines the format in which to export the results, from the given export options.
+ */
+async function determineExportFormat(
+  ...options: { label: string }[]
+): Promise<QuickPickItem> {
+  const exportFormat = await window.showQuickPick(
+    options,
+    {
+      placeHolder: 'Select export format',
+      canPickMany: false,
+      ignoreFocusOut: true,
+    }
+  );
+  if (!exportFormat || !exportFormat.label) {
+    throw new UserCancellationException('No export format selected', true);
+  }
+  return exportFormat;
+}
+
+/**
+ * Converts the results of a remote query to markdown and uploads the files as a secret gist.
+ */
+export async function exportResultsToGist(
+  ctx: ExtensionContext,
+  query: RemoteQuery,
+  analysesResults: AnalysisResults[]
+): Promise<void> {
+  const credentials = await Credentials.initialize(ctx);
+  const description = buildGistDescription(query, analysesResults);
+  const markdownFiles = generateMarkdown(query, analysesResults, 'gist');
+  // Convert markdownFiles to the appropriate format for uploading to gist
+  const gistFiles = markdownFiles.reduce((acc, cur) => {
+    acc[`${cur.fileName}.md`] = { content: cur.content.join('\n') };
+    return acc;
+  }, {} as { [key: string]: { content: string } });
+
+  const gistUrl = await createGist(credentials, description, gistFiles);
+  if (gistUrl) {
+    const shouldOpenGist = await showInformationMessageWithAction(
+      'Variant analysis results exported to gist.',
+      'Open gist'
+    );
+    if (shouldOpenGist) {
+      await commands.executeCommand('vscode.open', Uri.parse(gistUrl));
+    }
+  }
+}
+
+/**
+ * Builds Gist description
+ * Ex: Empty Block (Go) x results (y repositories)
+ */
+const buildGistDescription = (query: RemoteQuery, analysesResults: AnalysisResults[]) => {
+  const resultCount = sumAnalysesResults(analysesResults);
+  const resultLabel = pluralize(resultCount, 'result', 'results');
+  const repositoryLabel = query.repositoryCount ? `(${pluralize(query.repositoryCount, 'repository', 'repositories')})` : '';
+  return `${query.queryName} (${query.language}) ${resultLabel} ${repositoryLabel}`;
+};
+
+/**
+ * Converts the results of a remote query to markdown and saves the files locally
+ * in the query directory (where query results and metadata are also saved).
+ */
+async function exportResultsToLocalMarkdown(
+  queryDirectoryPath: string,
+  query: RemoteQuery,
+  analysesResults: AnalysisResults[]
+) {
+  const markdownFiles = generateMarkdown(query, analysesResults, 'local');
+  const exportedResultsPath = path.join(queryDirectoryPath, 'exported-results');
+  await fs.ensureDir(exportedResultsPath);
+  for (const markdownFile of markdownFiles) {
+    const filePath = path.join(exportedResultsPath, `${markdownFile.fileName}.md`);
+    await fs.writeFile(filePath, markdownFile.content.join('\n'), 'utf8');
+  }
+  const shouldOpenExportedResults = await showInformationMessageWithAction(
+    `Variant analysis results exported to \"${exportedResultsPath}\".`,
+    'Open exported results'
+  );
+  if (shouldOpenExportedResults) {
+    const summaryFilePath = path.join(exportedResultsPath, '_summary.md');
+    const summaryFile = await workspace.openTextDocument(summaryFilePath);
+    await window.showTextDocument(summaryFile, ViewColumn.One);
+    await commands.executeCommand('revealFileInOS', Uri.file(summaryFilePath));
+  }
+}

extensions/ql-vscode/src/remote-queries/gh-api/gh-actions-api-client.ts

@@ -1,13 +1,16 @@
-import * as unzipper from 'unzipper';
 import * as path from 'path';
 import * as fs from 'fs-extra';
-import { showAndLogWarningMessage, tmpDir } from '../helpers';
-import { Credentials } from '../authentication';
-import { logger } from '../logging';
-import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
-import { DownloadLink, createDownloadPath } from './download-link';
-import { RemoteQuery } from './remote-query';
-import { RemoteQueryFailureIndexItem, RemoteQueryResultIndex, RemoteQuerySuccessIndexItem } from './remote-query-result-index';
+import { showAndLogErrorMessage, showAndLogWarningMessage, tmpDir } from '../../helpers';
+import { Credentials } from '../../authentication';
+import { logger } from '../../logging';
+import { RemoteQueryWorkflowResult } from '../remote-query-workflow-result';
+import { DownloadLink, createDownloadPath } from '../download-link';
+import { RemoteQuery } from '../remote-query';
+import { RemoteQueryFailureIndexItem, RemoteQueryResultIndex, RemoteQuerySuccessIndexItem } from '../remote-query-result-index';
+import { getErrorMessage } from '../../pure/helpers-pure';
+import { unzipFile } from '../../pure/zip';
+
+export const RESULT_INDEX_ARTIFACT_NAME = 'result-index';
 
 interface ApiSuccessIndexItem {
   nwo: string;
@@ -16,6 +19,7 @@ interface ApiSuccessIndexItem {
   results_count: number;
   bqrs_file_size: number;
   sarif_file_size?: number;
+  source_location_prefix: string;
 }
 
 interface ApiFailureIndexItem {
@@ -42,7 +46,10 @@ export async function getRemoteQueryIndex(
   const artifactsUrlPath = `/repos/${owner}/${repoName}/actions/artifacts`;
 
   const artifactList = await listWorkflowRunArtifacts(credentials, owner, repoName, workflowRunId);
-  const resultIndexArtifactId = getArtifactIDfromName('result-index', workflowUri, artifactList);
+  const resultIndexArtifactId = tryGetArtifactIDfromName(RESULT_INDEX_ARTIFACT_NAME, artifactList);
+  if (!resultIndexArtifactId) {
+    return undefined;
+  }
   const resultIndex = await getResultIndex(credentials, owner, repoName, resultIndexArtifactId);
 
   const successes = resultIndex?.successes.map(item => {
@@ -55,7 +62,8 @@ export async function getRemoteQueryIndex(
       sha: item.sha,
       resultCount: item.results_count,
       bqrsFileSize: item.bqrs_file_size,
-      sarifFileSize: item.sarif_file_size
+      sarifFileSize: item.sarif_file_size,
+      sourceLocationPrefix: item.source_location_prefix
     } as RemoteQuerySuccessIndexItem;
   });
 
@@ -74,6 +82,18 @@ export async function getRemoteQueryIndex(
   };
 }
 
+export async function cancelRemoteQuery(
+  credentials: Credentials,
+  remoteQuery: RemoteQuery
+): Promise<void> {
+  const octokit = await credentials.getOctokit();
+  const { actionsWorkflowRunId, controllerRepository: { owner, name } } = remoteQuery;
+  const response = await octokit.request(`POST /repos/${owner}/${name}/actions/runs/${actionsWorkflowRunId}/cancel`);
+  if (response.status >= 300) {
+    throw new Error(`Error cancelling variant analysis: ${response.status} ${response?.data?.message || ''}`);
+  }
+}
+
 export async function downloadArtifactFromLink(
   credentials: Credentials,
   storagePath: string,
@@ -90,14 +110,33 @@ export async function downloadArtifactFromLink(
     const response = await octokit.request(`GET ${downloadLink.urlPath}/zip`, {});
 
     const zipFilePath = createDownloadPath(storagePath, downloadLink, 'zip');
-    await saveFile(`${zipFilePath}`, response.data as ArrayBuffer);
 
-    // Extract the zipped artifact.
-    await unzipFile(zipFilePath, extractedPath);
+    await unzipBuffer(response.data as ArrayBuffer, zipFilePath, extractedPath);
   }
   return path.join(extractedPath, downloadLink.innerFilePath || '');
 }
 
+/**
+ * Checks whether a specific artifact is present in the list of artifacts of a workflow run.
+ * @param credentials Credentials for authenticating to the GitHub API.
+ * @param owner
+ * @param repo
+ * @param workflowRunId The ID of the workflow run to get the artifact for.
+ * @param artifactName The artifact name, as a string.
+ * @returns A boolean indicating if the artifact is available.
+ */
+export async function isArtifactAvailable(
+  credentials: Credentials,
+  owner: string,
+  repo: string,
+  workflowRunId: number,
+  artifactName: string,
+): Promise<boolean> {
+  const artifactList = await listWorkflowRunArtifacts(credentials, owner, repo, workflowRunId);
+
+  return tryGetArtifactIDfromName(artifactName, artifactList) !== undefined;
+}
+
 /**
  * Downloads the result index artifact and extracts the result index items.
  * @param credentials Credentials for authenticating to the GitHub API.
@@ -211,15 +250,29 @@ function getArtifactIDfromName(
   workflowUri: string,
   artifacts: Array<{ id: number, name: string }>
 ): number {
-  const artifact = artifacts.find(a => a.name === artifactName);
+  const artifactId = tryGetArtifactIDfromName(artifactName, artifacts);
 
-  if (!artifact) {
+  if (!artifactId) {
     const errorMessage =
       `Could not find artifact with name ${artifactName} in workflow ${workflowUri}.
       Please check whether the workflow run has successfully completed.`;
     throw Error(errorMessage);
   }
 
+  return artifactId;
+}
+
+/**
+ * @param artifactName The artifact name, as a string.
+ * @param artifacts An array of artifact details (from the "list workflow run artifacts" API response).
+ * @returns The artifact ID corresponding to the given artifact name, if it exists.
+ */
+function tryGetArtifactIDfromName(
+  artifactName: string,
+  artifacts: Array<{ id: number, name: string }>
+): number | undefined {
+  const artifact = artifacts.find(a => a.name === artifactName);
+
   return artifact?.id;
 }
 
@@ -245,20 +298,16 @@ async function downloadArtifact(
     archive_format: 'zip',
   });
   const artifactPath = path.join(tmpDir.name, `${artifactId}`);
-  await saveFile(`${artifactPath}.zip`, response.data as ArrayBuffer);
-  await unzipFile(`${artifactPath}.zip`, artifactPath);
+  await unzipBuffer(response.data as ArrayBuffer, `${artifactPath}.zip`, artifactPath);
   return artifactPath;
 }
 
-async function saveFile(filePath: string, data: ArrayBuffer): Promise<void> {
+async function unzipBuffer(data: ArrayBuffer, filePath: string, destinationPath: string): Promise<void> {
   void logger.log(`Saving file to ${filePath}`);
   await fs.writeFile(filePath, Buffer.from(data));
-}
 
-async function unzipFile(sourcePath: string, destinationPath: string) {
   void logger.log(`Unzipping file to ${destinationPath}`);
-  const file = await unzipper.Open.file(sourcePath);
-  await file.extract({ path: destinationPath });
+  await unzipFile(filePath, destinationPath);
 }
 
 function getWorkflowError(conclusion: string | null): string {
@@ -282,3 +331,100 @@ function getWorkflowError(conclusion: string | null): string {
 
   return `Unexpected variant analysis execution conclusion: ${conclusion}`;
 }
+
+/**
+ * Creates a gist with the given description and files.
+ * Returns the URL of the created gist.
+ */
+export async function createGist(
+  credentials: Credentials,
+  description: string,
+  files: { [key: string]: { content: string } }
+): Promise<string | undefined> {
+  const octokit = await credentials.getOctokit();
+  const response = await octokit.request('POST /gists', {
+    description,
+    files,
+    public: false,
+  });
+  if (response.status >= 300) {
+    throw new Error(`Error exporting variant analysis results: ${response.status} ${response?.data || ''}`);
+  }
+  return response.data.html_url;
+}
+
+const repositoriesMetadataQuery = `query Stars($repos: String!, $pageSize: Int!, $cursor: String) {
+  search(
+    query: $repos
+    type: REPOSITORY
+    first: $pageSize
+    after: $cursor
+  ) {
+    edges {
+      node {
+        ... on Repository {
+          name
+          owner {
+            login
+          }
+          stargazerCount
+          updatedAt
+        }
+      }
+      cursor
+    }
+  }
+}`;
+
+type RepositoriesMetadataQueryResponse = {
+  search: {
+    edges: {
+      cursor: string;
+      node: {
+        name: string;
+        owner: {
+          login: string;
+        };
+        stargazerCount: number;
+        updatedAt: string; // Actually a ISO Date string
+      }
+    }[]
+  }
+};
+
+export type RepositoriesMetadata = Record<string, { starCount: number, lastUpdated: number }>
+
+export async function getRepositoriesMetadata(credentials: Credentials, nwos: string[], pageSize = 100): Promise<RepositoriesMetadata> {
+  const octokit = await credentials.getOctokit();
+  const repos = `repo:${nwos.join(' repo:')} fork:true`;
+  let cursor = null;
+  const metadata: RepositoriesMetadata = {};
+  try {
+    do {
+      const response: RepositoriesMetadataQueryResponse = await octokit.graphql({
+        query: repositoriesMetadataQuery,
+        repos,
+        pageSize,
+        cursor
+      });
+      cursor = response.search.edges.length === pageSize ? response.search.edges[pageSize - 1].cursor : null;
+
+      for (const edge of response.search.edges) {
+        const node = edge.node;
+        const owner = node.owner.login;
+        const name = node.name;
+        const starCount = node.stargazerCount;
+        // lastUpdated is always negative since it happened in the past.
+        const lastUpdated = new Date(node.updatedAt).getTime() - Date.now();
+        metadata[`${owner}/${name}`] = {
+          starCount, lastUpdated
+        };
+      }
+
+    } while (cursor);
+  } catch (e) {
+    void showAndLogErrorMessage(`Error retrieving repository metadata for variant analysis: ${getErrorMessage(e)}`);
+  }
+
+  return metadata;
+}

extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts

@@ -0,0 +1,96 @@
+import { Credentials } from '../../authentication';
+import { OctokitResponse } from '@octokit/types/dist-types';
+import { VariantAnalysisSubmission } from '../shared/variant-analysis';
+import {
+  VariantAnalysis,
+  VariantAnalysisRepoTask,
+  VariantAnalysisSubmissionRequest
+} from './variant-analysis';
+import { Repository } from './repository';
+
+export async function submitVariantAnalysis(
+  credentials: Credentials,
+  submissionDetails: VariantAnalysisSubmission
+): Promise<VariantAnalysis> {
+  const octokit = await credentials.getOctokit();
+
+  const { actionRepoRef, query, databases, controllerRepoId } = submissionDetails;
+
+  const data: VariantAnalysisSubmissionRequest = {
+    action_repo_ref: actionRepoRef,
+    language: query.language,
+    query_pack: query.pack,
+    repositories: databases.repositories,
+    repository_lists: databases.repositoryLists,
+    repository_owners: databases.repositoryOwners,
+  };
+
+  const response: OctokitResponse<VariantAnalysis> = await octokit.request(
+    'POST /repositories/:controllerRepoId/code-scanning/codeql/variant-analyses',
+    {
+      controllerRepoId,
+      data
+    }
+  );
+
+  return response.data;
+}
+
+export async function getVariantAnalysis(
+  credentials: Credentials,
+  controllerRepoId: number,
+  variantAnalysisId: number
+): Promise<VariantAnalysis> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<VariantAnalysis> = await octokit.request(
+    'GET /repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId',
+    {
+      controllerRepoId,
+      variantAnalysisId
+    }
+  );
+
+  return response.data;
+}
+
+export async function getVariantAnalysisRepo(
+  credentials: Credentials,
+  controllerRepoId: number,
+  variantAnalysisId: number,
+  repoId: number
+): Promise<VariantAnalysisRepoTask> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<VariantAnalysisRepoTask> = await octokit.request(
+    'GET /repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId/repositories/:repoId',
+    {
+      controllerRepoId,
+      variantAnalysisId,
+      repoId
+    }
+  );
+
+  return response.data;
+}
+
+export async function getVariantAnalysisRepoResult(
+  credentials: Credentials,
+  downloadUrl: string,
+): Promise<ArrayBuffer> {
+  const octokit = await credentials.getOctokit();
+  const response = await octokit.request(`GET ${downloadUrl}`);
+
+  return response.data;
+}
+
+export async function getRepositoryFromNwo(
+  credentials: Credentials,
+  owner: string,
+  repo: string
+): Promise<Repository> {
+  const octokit = await credentials.getOctokit();
+
+  const response = await octokit.rest.repos.get({ owner, repo });
+  return response.data as Repository;
+}

extensions/ql-vscode/src/remote-queries/gh-api/repository.ts

@@ -0,0 +1,13 @@
+/**
+ * Defines basic information about a repository.
+ * 
+ * Different parts of the API may return different subsets of information
+ * about a repository, but this model represents the very basic information
+ * that will always be available.
+ */
+export interface Repository {
+  id: number,
+  name: string,
+  full_name: string,
+  private: boolean,
+}

extensions/ql-vscode/src/remote-queries/gh-api/variant-analysis.ts

@@ -0,0 +1,83 @@
+import { Repository } from './repository';
+
+export interface VariantAnalysisSubmissionRequest {
+  action_repo_ref: string,
+  language: VariantAnalysisQueryLanguage,
+  query_pack: string,
+  repositories?: string[],
+  repository_lists?: string[],
+  repository_owners?: string[]
+}
+
+export type VariantAnalysisQueryLanguage =
+  | 'csharp'
+  | 'cpp'
+  | 'go'
+  | 'java'
+  | 'javascript'
+  | 'python'
+  | 'ruby';
+
+export interface VariantAnalysis {
+  id: number,
+  controller_repo: Repository,
+  actor_id: number,
+  query_language: VariantAnalysisQueryLanguage,
+  query_pack_url: string,
+  status: VariantAnalysisStatus,
+  actions_workflow_run_id?: number,
+  failure_reason?: VariantAnalysisFailureReason,
+  scanned_repositories?: VariantAnalysisScannedRepository[],
+  skipped_repositories?: VariantAnalysisSkippedRepositories
+}
+
+export type VariantAnalysisStatus =
+  | 'in_progress'
+  | 'completed';
+
+export type VariantAnalysisFailureReason =
+  | 'no_repos_queried'
+  | 'internal_error';
+
+export type VariantAnalysisRepoStatus =
+  | 'pending'
+  | 'in_progress'
+  | 'succeeded'
+  | 'failed'
+  | 'canceled'
+  | 'timed_out';
+
+export interface VariantAnalysisScannedRepository {
+  repository: Repository,
+  analysis_status: VariantAnalysisRepoStatus,
+  result_count?: number,
+  artifact_size_in_bytes?: number,
+  failure_message?: string
+}
+
+export interface VariantAnalysisSkippedRepositoryGroup {
+  repository_count: number,
+  repositories: Repository[]
+}
+
+export interface VariantAnalysisNotFoundRepositoryGroup {
+  repository_count: number,
+  repository_full_names: string[]
+}
+export interface VariantAnalysisRepoTask {
+  repository: Repository,
+  analysis_status: VariantAnalysisRepoStatus,
+  artifact_size_in_bytes?: number,
+  result_count?: number,
+  failure_message?: string,
+  database_commit_sha?: string,
+  source_location_prefix?: string,
+  artifact_url?: string
+}
+
+export interface VariantAnalysisSkippedRepositories {
+  access_mismatch_repos?: VariantAnalysisSkippedRepositoryGroup,
+  not_found_repo_nwos?: VariantAnalysisNotFoundRepositoryGroup,
+  no_codeql_db_repos?: VariantAnalysisSkippedRepositoryGroup,
+  over_limit_repos?: VariantAnalysisSkippedRepositoryGroup
+}

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -1,89 +1,117 @@
-import { CancellationToken, commands, ExtensionContext, Uri, window } from 'vscode';
+import { CancellationToken, commands, EventEmitter, ExtensionContext, Uri, env, window } from 'vscode';
 import { nanoid } from 'nanoid';
 import * as path from 'path';
 import * as fs from 'fs-extra';
+import * as os from 'os';
 
 import { Credentials } from '../authentication';
 import { CodeQLCliServer } from '../cli';
 import { ProgressCallback } from '../commandRunner';
-import { createTimestampFile, showAndLogErrorMessage, showInformationMessageWithAction } from '../helpers';
+import { createTimestampFile, showAndLogErrorMessage, showAndLogInformationMessage, showInformationMessageWithAction } from '../helpers';
 import { Logger } from '../logging';
 import { runRemoteQuery } from './run-remote-query';
-import { RemoteQueriesInterfaceManager } from './remote-queries-interface';
+import { RemoteQueriesView } from './remote-queries-view';
 import { RemoteQuery } from './remote-query';
 import { RemoteQueriesMonitor } from './remote-queries-monitor';
-import { getRemoteQueryIndex } from './gh-actions-api-client';
+import { getRemoteQueryIndex, getRepositoriesMetadata, RepositoriesMetadata } from './gh-api/gh-actions-api-client';
 import { RemoteQueryResultIndex } from './remote-query-result-index';
-import { RemoteQueryResult } from './remote-query-result';
+import { RemoteQueryResult, sumAnalysisSummariesResults } from './remote-query-result';
 import { DownloadLink } from './download-link';
 import { AnalysesResultsManager } from './analyses-results-manager';
 import { assertNever } from '../pure/helpers-pure';
-import { RemoteQueryHistoryItem } from './remote-query-history-item';
-import { QueryHistoryManager } from '../query-history';
 import { QueryStatus } from '../query-status';
 import { DisposableObject } from '../pure/disposable-object';
-import { QueryHistoryInfo } from '../query-results';
+import { AnalysisResults } from './shared/analysis-result';
+import { VariantAnalysisManager } from './variant-analysis-manager';
 
 const autoDownloadMaxSize = 300 * 1024;
 const autoDownloadMaxCount = 100;
 
 const noop = () => { /* do nothing */ };
+
+export interface NewQueryEvent {
+  queryId: string;
+  query: RemoteQuery
+}
+
+export interface RemovedQueryEvent {
+  queryId: string;
+}
+
+export interface UpdatedQueryStatusEvent {
+  queryId: string;
+  status: QueryStatus;
+  failureReason?: string;
+  repositoryCount?: number;
+  resultCount?: number;
+}
+
 export class RemoteQueriesManager extends DisposableObject {
+  public readonly onRemoteQueryAdded;
+  public readonly onRemoteQueryRemoved;
+  public readonly onRemoteQueryStatusUpdate;
+
+  private readonly remoteQueryAddedEventEmitter;
+  private readonly remoteQueryRemovedEventEmitter;
+  private readonly remoteQueryStatusUpdateEventEmitter;
+
   private readonly remoteQueriesMonitor: RemoteQueriesMonitor;
   private readonly analysesResultsManager: AnalysesResultsManager;
-  private readonly interfaceManager: RemoteQueriesInterfaceManager;
+  private readonly variantAnalysisManager: VariantAnalysisManager;
+  private readonly view: RemoteQueriesView;
 
   constructor(
     private readonly ctx: ExtensionContext,
     private readonly cliServer: CodeQLCliServer,
-    private readonly qhm: QueryHistoryManager,
     private readonly storagePath: string,
     logger: Logger,
+    variantAnalysisManager: VariantAnalysisManager,
   ) {
     super();
     this.analysesResultsManager = new AnalysesResultsManager(ctx, cliServer, storagePath, logger);
-    this.interfaceManager = new RemoteQueriesInterfaceManager(ctx, logger, this.analysesResultsManager);
+    this.view = new RemoteQueriesView(ctx, logger, this.analysesResultsManager);
     this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
+    this.variantAnalysisManager = variantAnalysisManager;
 
-    // Handle events from the query history manager
-    this.push(this.qhm.onDidAddQueryItem(this.handleAddQueryItem.bind(this)));
-    this.push(this.qhm.onDidRemoveQueryItem(this.handleRemoveQueryItem.bind(this)));
-    this.push(this.qhm.onWillOpenQueryItem(this.handleOpenQueryItem.bind(this)));
+    this.remoteQueryAddedEventEmitter = this.push(new EventEmitter<NewQueryEvent>());
+    this.remoteQueryRemovedEventEmitter = this.push(new EventEmitter<RemovedQueryEvent>());
+    this.remoteQueryStatusUpdateEventEmitter = this.push(new EventEmitter<UpdatedQueryStatusEvent>());
+    this.onRemoteQueryAdded = this.remoteQueryAddedEventEmitter.event;
+    this.onRemoteQueryRemoved = this.remoteQueryRemovedEventEmitter.event;
+    this.onRemoteQueryStatusUpdate = this.remoteQueryStatusUpdateEventEmitter.event;
+
+    this.push(this.view);
   }
 
-  private async handleAddQueryItem(queryItem: QueryHistoryInfo) {
-    if (queryItem?.t === 'remote') {
-      if (!(await this.queryHistoryItemExists(queryItem))) {
-        // In this case, the query was deleted from disk, most likely because it was purged
-        // by another workspace. We should remove it from the history manager.
-        await this.qhm.handleRemoveHistoryItem(queryItem);
-      } else if (queryItem.status === QueryStatus.InProgress) {
-        // In this case, last time we checked, the query was still in progress.
-        // We need to setup the monitor to check for completion.
-        await commands.executeCommand('codeQL.monitorRemoteQuery', queryItem);
-      }
+  public async rehydrateRemoteQuery(queryId: string, query: RemoteQuery, status: QueryStatus) {
+    if (!(await this.queryRecordExists(queryId))) {
+      // In this case, the query was deleted from disk, most likely because it was purged
+      // by another workspace.
+      this.remoteQueryRemovedEventEmitter.fire({ queryId });
+    } else if (status === QueryStatus.InProgress) {
+      // In this case, last time we checked, the query was still in progress.
+      // We need to setup the monitor to check for completion.
+      await commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
     }
   }
 
-  private async handleRemoveQueryItem(queryItem: QueryHistoryInfo) {
-    if (queryItem?.t === 'remote') {
-      this.analysesResultsManager.removeAnalysesResults(queryItem.queryId);
-      await this.removeStorageDirectory(queryItem);
-    }
+  public async removeRemoteQuery(queryId: string) {
+    this.analysesResultsManager.removeAnalysesResults(queryId);
+    await this.removeStorageDirectory(queryId);
   }
 
-  private async handleOpenQueryItem(queryItem: QueryHistoryInfo) {
-    if (queryItem?.t === 'remote') {
-      try {
-        const remoteQueryResult = await this.retrieveJsonFile(queryItem, 'query-result.json') as RemoteQueryResult;
-        // open results in the background
-        void this.openResults(queryItem.remoteQuery, remoteQueryResult).then(
-          noop,
-          err => void showAndLogErrorMessage(err)
-        );
-      } catch (e) {
-        void showAndLogErrorMessage(`Could not open query results. ${e}`);
-      }
+  public async openRemoteQueryResults(queryId: string) {
+    try {
+      const remoteQuery = await this.retrieveJsonFile(queryId, 'query.json') as RemoteQuery;
+      const remoteQueryResult = await this.retrieveJsonFile(queryId, 'query-result.json') as RemoteQueryResult;
+
+      // Open results in the background
+      void this.openResults(remoteQuery, remoteQueryResult).then(
+        noop,
+        err => void showAndLogErrorMessage(err)
+      );
+    } catch (e) {
+      void showAndLogErrorMessage(`Could not open query results. ${e}`);
     }
   }
 
@@ -99,69 +127,48 @@ export class RemoteQueriesManager extends DisposableObject {
       credentials, uri || window.activeTextEditor?.document.uri,
       false,
       progress,
-      token);
+      token,
+      this.variantAnalysisManager);
 
     if (querySubmission?.query) {
       const query = querySubmission.query;
-      const queryId = this.createQueryId(query.queryName);
+      const queryId = this.createQueryId();
 
-      const queryHistoryItem: RemoteQueryHistoryItem = {
-        t: 'remote',
-        status: QueryStatus.InProgress,
-        completed: false,
-        queryId,
-        label: query.queryName,
-        remoteQuery: query,
-      };
-      await this.prepareStorageDirectory(queryHistoryItem);
-      await this.storeJsonFile(queryHistoryItem, 'query.json', query);
+      await this.prepareStorageDirectory(queryId);
+      await this.storeJsonFile(queryId, 'query.json', query);
 
-      this.qhm.addQuery(queryHistoryItem);
-      await this.qhm.refreshTreeView();
+      this.remoteQueryAddedEventEmitter.fire({ queryId, query });
+      void commands.executeCommand('codeQL.monitorRemoteQuery', queryId, query);
     }
   }
 
   public async monitorRemoteQuery(
-    queryItem: RemoteQueryHistoryItem,
+    queryId: string,
+    remoteQuery: RemoteQuery,
     cancellationToken: CancellationToken
   ): Promise<void> {
     const credentials = await Credentials.initialize(this.ctx);
 
-    const queryWorkflowResult = await this.remoteQueriesMonitor.monitorQuery(queryItem.remoteQuery, cancellationToken);
+    const queryWorkflowResult = await this.remoteQueriesMonitor.monitorQuery(remoteQuery, cancellationToken);
 
     const executionEndTime = Date.now();
 
     if (queryWorkflowResult.status === 'CompletedSuccessfully') {
-      const resultIndex = await getRemoteQueryIndex(credentials, queryItem.remoteQuery);
-      queryItem.completed = true;
-      if (resultIndex) {
-        queryItem.status = QueryStatus.Completed;
-        const queryResult = this.mapQueryResult(executionEndTime, resultIndex, queryItem.queryId);
-
-        await this.storeJsonFile(queryItem, 'query-result.json', queryResult);
-
-        // Kick off auto-download of results in the background.
-        void commands.executeCommand('codeQL.autoDownloadRemoteQueryResults', queryResult);
-
-        // Ask if the user wants to open the results in the background.
-        void this.askToOpenResults(queryItem.remoteQuery, queryResult).then(
-          noop,
-          err => {
-            void showAndLogErrorMessage(err);
-          }
-        );
-      } else {
-        void showAndLogErrorMessage(`There was an issue retrieving the result for the query ${queryItem.label}`);
-        queryItem.status = QueryStatus.Failed;
-      }
+      await this.downloadAvailableResults(queryId, remoteQuery, credentials, executionEndTime);
     } else if (queryWorkflowResult.status === 'CompletedUnsuccessfully') {
-      queryItem.failureReason = queryWorkflowResult.error;
-      queryItem.status = QueryStatus.Failed;
-      void showAndLogErrorMessage(`Variant analysis execution failed. Error: ${queryWorkflowResult.error}`);
+      if (queryWorkflowResult.error?.includes('cancelled')) {
+        // Workflow was cancelled on the server
+        this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed, failureReason: 'Cancelled' });
+        await this.downloadAvailableResults(queryId, remoteQuery, credentials, executionEndTime);
+        void showAndLogInformationMessage('Variant analysis was cancelled');
+      } else {
+        this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed, failureReason: queryWorkflowResult.error });
+        void showAndLogErrorMessage(`Variant analysis execution failed. Error: ${queryWorkflowResult.error}`);
+      }
     } else if (queryWorkflowResult.status === 'Cancelled') {
-      queryItem.failureReason = 'Cancelled';
-      queryItem.status = QueryStatus.Failed;
-      void showAndLogErrorMessage('Variant analysis monitoring was cancelled');
+      this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed, failureReason: 'Cancelled' });
+      await this.downloadAvailableResults(queryId, remoteQuery, credentials, executionEndTime);
+      void showAndLogInformationMessage('Variant analysis was cancelled');
     } else if (queryWorkflowResult.status === 'InProgress') {
       // Should not get here. Only including this to ensure `assertNever` uses proper type checking.
       void showAndLogErrorMessage(`Unexpected status: ${queryWorkflowResult.status}`);
@@ -169,7 +176,6 @@ export class RemoteQueriesManager extends DisposableObject {
       // Ensure all cases are covered
       assertNever(queryWorkflowResult.status);
     }
-    await this.qhm.refreshTreeView();
   }
 
   public async autoDownloadRemoteQueryResults(
@@ -183,6 +189,7 @@ export class RemoteQueriesManager extends DisposableObject {
         nwo: a.nwo,
         databaseSha: a.databaseSha,
         resultCount: a.resultCount,
+        sourceLocationPrefix: a.sourceLocationPrefix,
         downloadLink: a.downloadLink,
         fileSize: String(a.fileSizeInBytes)
       }));
@@ -190,21 +197,46 @@ export class RemoteQueriesManager extends DisposableObject {
     await this.analysesResultsManager.loadAnalysesResults(
       analysesToDownload,
       token,
-      results => this.interfaceManager.setAnalysisResults(results));
+      results => this.view.setAnalysisResults(results, queryResult.queryId));
   }
 
-  private mapQueryResult(executionEndTime: number, resultIndex: RemoteQueryResultIndex, queryId: string): RemoteQueryResult {
+  public async copyRemoteQueryRepoListToClipboard(queryId: string) {
+    const queryResult = await this.getRemoteQueryResult(queryId);
+    const repos = queryResult.analysisSummaries
+      .filter(a => a.resultCount > 0)
+      .map(a => a.nwo);
 
+    if (repos.length > 0) {
+      const text = [
+        '"new-repo-list": [',
+        ...repos.slice(0, -1).map(repo => `    "${repo}",`),
+        `    "${repos[repos.length - 1]}"`,
+        ']'
+      ];
+
+      await env.clipboard.writeText(text.join(os.EOL));
+    }
+  }
+
+  private mapQueryResult(
+    executionEndTime: number,
+    resultIndex: RemoteQueryResultIndex,
+    queryId: string,
+    metadata: RepositoriesMetadata
+  ): RemoteQueryResult {
     const analysisSummaries = resultIndex.successes.map(item => ({
       nwo: item.nwo,
       databaseSha: item.sha || 'HEAD',
       resultCount: item.resultCount,
+      sourceLocationPrefix: item.sourceLocationPrefix,
       fileSizeInBytes: item.sarifFileSize ? item.sarifFileSize : item.bqrsFileSize,
+      starCount: metadata[item.nwo]?.starCount,
+      lastUpdated: metadata[item.nwo]?.lastUpdated,
       downloadLink: {
         id: item.artifactId.toString(),
         urlPath: `${resultIndex.artifactsUrlPath}/${item.artifactId}`,
         innerFilePath: item.sarifFileSize ? 'results.sarif' : 'results.bqrs',
-        queryId,
+        queryId
       } as DownloadLink
     }));
     const analysisFailures = resultIndex.failures.map(item => ({
@@ -221,11 +253,11 @@ export class RemoteQueriesManager extends DisposableObject {
   }
 
   public async openResults(query: RemoteQuery, queryResult: RemoteQueryResult) {
-    await this.interfaceManager.showResults(query, queryResult);
+    await this.view.showResults(query, queryResult);
   }
 
   private async askToOpenResults(query: RemoteQuery, queryResult: RemoteQueryResult): Promise<void> {
-    const totalResultCount = queryResult.analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
+    const totalResultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
     const totalRepoCount = queryResult.analysisSummaries.length;
     const message = `Query "${query.queryName}" run on ${totalRepoCount} repositories and returned ${totalResultCount} results`;
 
@@ -237,12 +269,10 @@ export class RemoteQueriesManager extends DisposableObject {
 
   /**
    * Generates a unique id for this query, suitable for determining the storage location for the downloaded query artifacts.
-   * @param queryName
-   * @returns
+   * @returns A unique id for this query.
    */
-  private createQueryId(queryName: string): string {
-    return `${queryName}-${nanoid()}`;
-
+  private createQueryId(): string {
+    return nanoid();
   }
 
   /**
@@ -251,29 +281,87 @@ export class RemoteQueriesManager extends DisposableObject {
    * used by the query history manager to determine when the directory
    * should be deleted.
    *
-   * @param queryName The name of the query that was run.
    */
-  private async prepareStorageDirectory(queryHistoryItem: RemoteQueryHistoryItem): Promise<void> {
-    await createTimestampFile(path.join(this.storagePath, queryHistoryItem.queryId));
+  private async prepareStorageDirectory(queryId: string): Promise<void> {
+    await createTimestampFile(path.join(this.storagePath, queryId));
   }
 
-  private async storeJsonFile<T>(queryHistoryItem: RemoteQueryHistoryItem, fileName: string, obj: T): Promise<void> {
-    const filePath = path.join(this.storagePath, queryHistoryItem.queryId, fileName);
+  private async getRemoteQueryResult(queryId: string): Promise<RemoteQueryResult> {
+    return await this.retrieveJsonFile<RemoteQueryResult>(queryId, 'query-result.json');
+  }
+
+  private async storeJsonFile<T>(queryId: string, fileName: string, obj: T): Promise<void> {
+    const filePath = path.join(this.storagePath, queryId, fileName);
     await fs.writeFile(filePath, JSON.stringify(obj, null, 2), 'utf8');
   }
 
-  private async retrieveJsonFile<T>(queryHistoryItem: RemoteQueryHistoryItem, fileName: string): Promise<T> {
-    const filePath = path.join(this.storagePath, queryHistoryItem.queryId, fileName);
+  private async retrieveJsonFile<T>(queryId: string, fileName: string): Promise<T> {
+    const filePath = path.join(this.storagePath, queryId, fileName);
     return JSON.parse(await fs.readFile(filePath, 'utf8'));
   }
 
-  private async removeStorageDirectory(queryItem: RemoteQueryHistoryItem): Promise<void> {
-    const filePath = path.join(this.storagePath, queryItem.queryId);
+  private async removeStorageDirectory(queryId: string): Promise<void> {
+    const filePath = path.join(this.storagePath, queryId);
     await fs.remove(filePath);
   }
 
-  private async queryHistoryItemExists(queryItem: RemoteQueryHistoryItem): Promise<boolean> {
-    const filePath = path.join(this.storagePath, queryItem.queryId);
+  private async queryRecordExists(queryId: string): Promise<boolean> {
+    const filePath = path.join(this.storagePath, queryId);
     return await fs.pathExists(filePath);
   }
+
+  /**
+   * Checks whether there's a result index artifact available for the given query.
+   * If so, set the query status to `Completed` and auto-download the results.
+   */
+  private async downloadAvailableResults(
+    queryId: string,
+    remoteQuery: RemoteQuery,
+    credentials: Credentials,
+    executionEndTime: number
+  ): Promise<void> {
+    const resultIndex = await getRemoteQueryIndex(credentials, remoteQuery);
+    if (resultIndex) {
+      const metadata = await this.getRepositoriesMetadata(resultIndex, credentials);
+      const queryResult = this.mapQueryResult(executionEndTime, resultIndex, queryId, metadata);
+      const resultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
+      this.remoteQueryStatusUpdateEventEmitter.fire({
+        queryId,
+        status: QueryStatus.Completed,
+        repositoryCount: queryResult.analysisSummaries.length,
+        resultCount
+      });
+
+      await this.storeJsonFile(queryId, 'query-result.json', queryResult);
+
+      // Kick off auto-download of results in the background.
+      void commands.executeCommand('codeQL.autoDownloadRemoteQueryResults', queryResult);
+
+      // Ask if the user wants to open the results in the background.
+      void this.askToOpenResults(remoteQuery, queryResult).then(
+        noop,
+        err => {
+          void showAndLogErrorMessage(err);
+        }
+      );
+    } else {
+      const controllerRepo = `${remoteQuery.controllerRepository.owner}/${remoteQuery.controllerRepository.name}`;
+      const workflowRunUrl = `https://github.com/${controllerRepo}/actions/runs/${remoteQuery.actionsWorkflowRunId}`;
+      void showAndLogErrorMessage(
+        `There was an issue retrieving the result for the query [${remoteQuery.queryName}](${workflowRunUrl}).`
+      );
+      this.remoteQueryStatusUpdateEventEmitter.fire({ queryId, status: QueryStatus.Failed });
+    }
+  }
+
+  private async getRepositoriesMetadata(resultIndex: RemoteQueryResultIndex, credentials: Credentials) {
+    const nwos = resultIndex.successes.map(s => s.nwo);
+    return await getRepositoriesMetadata(credentials, nwos);
+  }
+
+  // Pulled from the analysis results manager, so that we can get access to
+  // analyses results from the "export results" command.
+  public getAnalysesResults(queryId: string): AnalysisResults[] {
+    return [...this.analysesResultsManager.getAnalysesResults(queryId)];
+  }
 }

extensions/ql-vscode/src/remote-queries/remote-queries-markdown-generation.ts

@@ -0,0 +1,344 @@
+import { CellValue } from '../pure/bqrs-cli-types';
+import { tryGetRemoteLocation } from '../pure/bqrs-utils';
+import { createRemoteFileRef } from '../pure/location-link-utils';
+import { parseHighlightedLine, shouldHighlightLine } from '../pure/sarif-utils';
+import { convertNonPrintableChars } from '../text-utils';
+import { RemoteQuery } from './remote-query';
+import { AnalysisAlert, AnalysisRawResults, AnalysisResults, CodeSnippet, FileLink, getAnalysisResultCount, HighlightedRegion } from './shared/analysis-result';
+
+export type MarkdownLinkType = 'local' | 'gist';
+
+export interface MarkdownFile {
+  fileName: string;
+  content: string[]; // Each array item is a line of the markdown file.
+}
+
+/**
+ * Generates markdown files with variant analysis results.
+ */
+export function generateMarkdown(
+  query: RemoteQuery,
+  analysesResults: AnalysisResults[],
+  linkType: MarkdownLinkType
+): MarkdownFile[] {
+  const resultsFiles: MarkdownFile[] = [];
+  // Generate summary file with links to individual files
+  const summaryFile: MarkdownFile = generateMarkdownSummary(query);
+  for (const analysisResult of analysesResults) {
+    const resultsCount = getAnalysisResultCount(analysisResult);
+    if (resultsCount === 0) {
+      continue;
+    }
+
+    // Append nwo and results count to the summary table
+    const nwo = analysisResult.nwo;
+    const fileName = createFileName(nwo);
+    const link = createRelativeLink(fileName, linkType);
+    summaryFile.content.push(`| ${nwo} | [${resultsCount} result(s)](${link}) |`);
+
+    // Generate individual markdown file for each repository
+    const resultsFileContent = [
+      `### ${analysisResult.nwo}`,
+      ''
+    ];
+    for (const interpretedResult of analysisResult.interpretedResults) {
+      const individualResult = generateMarkdownForInterpretedResult(interpretedResult, query.language);
+      resultsFileContent.push(...individualResult);
+    }
+    if (analysisResult.rawResults) {
+      const rawResultTable = generateMarkdownForRawResults(analysisResult.rawResults);
+      resultsFileContent.push(...rawResultTable);
+    }
+    resultsFiles.push({
+      fileName: fileName,
+      content: resultsFileContent,
+    });
+  }
+  return [summaryFile, ...resultsFiles];
+}
+
+export function generateMarkdownSummary(query: RemoteQuery): MarkdownFile {
+  const lines: string[] = [];
+  // Title
+  lines.push(
+    `### Results for "${query.queryName}"`,
+    ''
+  );
+
+  // Expandable section containing query text
+  const queryCodeBlock = [
+    '```ql',
+    ...query.queryText.split('\n'),
+    '```',
+  ];
+  lines.push(
+    ...buildExpandableMarkdownSection('Query', queryCodeBlock)
+  );
+
+  // Padding between sections
+  lines.push(
+    '<br />',
+    '',
+  );
+
+  // Summary table
+  lines.push(
+    '### Summary',
+    '',
+    '| Repository | Results |',
+    '| --- | --- |',
+  );
+  // nwo and result count will be appended to this table
+  return {
+    fileName: '_summary',
+    content: lines
+  };
+}
+
+function generateMarkdownForInterpretedResult(interpretedResult: AnalysisAlert, language: string): string[] {
+  const lines: string[] = [];
+  lines.push(createMarkdownRemoteFileRef(
+    interpretedResult.fileLink,
+    interpretedResult.highlightedRegion?.startLine,
+    interpretedResult.highlightedRegion?.endLine
+  ));
+  lines.push('');
+  const codeSnippet = interpretedResult.codeSnippet;
+  const highlightedRegion = interpretedResult.highlightedRegion;
+  if (codeSnippet) {
+    lines.push(
+      ...generateMarkdownForCodeSnippet(codeSnippet, language, highlightedRegion),
+    );
+  }
+  const alertMessage = generateMarkdownForAlertMessage(interpretedResult);
+  lines.push(alertMessage, '');
+
+  // If available, show paths
+  const hasPathResults = interpretedResult.codeFlows.length > 0;
+  if (hasPathResults) {
+    const pathLines = generateMarkdownForPathResults(interpretedResult, language);
+    lines.push(...pathLines);
+  }
+
+  // Padding between results
+  lines.push(
+    '----------------------------------------',
+    '',
+  );
+  return lines;
+}
+
+function generateMarkdownForCodeSnippet(
+  codeSnippet: CodeSnippet,
+  language: string,
+  highlightedRegion?: HighlightedRegion
+): string[] {
+  const lines: string[] = [];
+  const snippetStartLine = codeSnippet.startLine || 0;
+  const codeLines = codeSnippet.text
+    .split('\n')
+    .map((line, index) =>
+      highlightAndEscapeCodeLines(line, index + snippetStartLine, highlightedRegion)
+    );
+
+  // Make sure there are no extra newlines before or after the <code> block:
+  const codeLinesWrapped = [...codeLines];
+  codeLinesWrapped[0] = `<pre><code class="${language}">${codeLinesWrapped[0]}`;
+  codeLinesWrapped[codeLinesWrapped.length - 1] = `${codeLinesWrapped[codeLinesWrapped.length - 1]}</code></pre>`;
+
+  lines.push(
+    ...codeLinesWrapped,
+    '',
+  );
+  return lines;
+}
+
+function highlightAndEscapeCodeLines(
+  line: string,
+  lineNumber: number,
+  highlightedRegion?: HighlightedRegion
+): string {
+  if (!highlightedRegion || !shouldHighlightLine(lineNumber, highlightedRegion)) {
+    return escapeHtmlCharacters(line);
+  }
+  const partiallyHighlightedLine = parseHighlightedLine(
+    line,
+    lineNumber,
+    highlightedRegion
+  );
+
+  const plainSection1 = escapeHtmlCharacters(partiallyHighlightedLine.plainSection1);
+  const highlightedSection = escapeHtmlCharacters(partiallyHighlightedLine.highlightedSection);
+  const plainSection2 = escapeHtmlCharacters(partiallyHighlightedLine.plainSection2);
+
+  return `${plainSection1}<strong>${highlightedSection}</strong>${plainSection2}`;
+}
+
+function generateMarkdownForAlertMessage(
+  interpretedResult: AnalysisAlert
+): string {
+  let alertMessage = '';
+  for (const token of interpretedResult.message.tokens) {
+    if (token.t === 'text') {
+      alertMessage += token.text;
+    } else if (token.t === 'location') {
+      alertMessage += createMarkdownRemoteFileRef(
+        token.location.fileLink,
+        token.location.highlightedRegion?.startLine,
+        token.location.highlightedRegion?.endLine,
+        token.text
+      );
+    }
+  }
+  // Italicize the alert message
+  return `*${alertMessage}*`;
+}
+
+function generateMarkdownForPathResults(
+  interpretedResult: AnalysisAlert,
+  language: string
+): string[] {
+  const lines: string[] = [];
+  lines.push('#### Paths', '');
+  for (const codeFlow of interpretedResult.codeFlows) {
+    const pathLines: string[] = [];
+    const stepCount = codeFlow.threadFlows.length;
+    const title = `Path with ${stepCount} steps`;
+    for (let i = 0; i < stepCount; i++) {
+      const threadFlow = codeFlow.threadFlows[i];
+      const link = createMarkdownRemoteFileRef(
+        threadFlow.fileLink,
+        threadFlow.highlightedRegion?.startLine,
+        threadFlow.highlightedRegion?.endLine
+      );
+      const codeSnippet = generateMarkdownForCodeSnippet(
+        threadFlow.codeSnippet,
+        language,
+        threadFlow.highlightedRegion
+      );
+      // Indent the snippet to fit with the numbered list.
+      const codeSnippetIndented = codeSnippet.map((line) => `    ${line}`);
+      pathLines.push(`${i + 1}. ${link}`, ...codeSnippetIndented);
+    }
+    lines.push(
+      ...buildExpandableMarkdownSection(title, pathLines)
+    );
+  }
+  return lines;
+}
+
+function generateMarkdownForRawResults(
+  analysisRawResults: AnalysisRawResults
+): string[] {
+  const tableRows: string[] = [];
+  const columnCount = analysisRawResults.schema.columns.length;
+  // Table headers are the column names if they exist, and empty otherwise
+  const headers = analysisRawResults.schema.columns.map(
+    (column) => column.name || ''
+  );
+  const tableHeader = `| ${headers.join(' | ')} |`;
+
+  tableRows.push(tableHeader);
+  tableRows.push('|' + ' --- |'.repeat(columnCount));
+
+  for (const row of analysisRawResults.resultSet.rows) {
+    const cells = row.map((cell) =>
+      generateMarkdownForRawTableCell(cell, analysisRawResults.fileLinkPrefix, analysisRawResults.sourceLocationPrefix)
+    );
+    tableRows.push(`| ${cells.join(' | ')} |`);
+  }
+  return tableRows;
+}
+
+function generateMarkdownForRawTableCell(
+  value: CellValue,
+  fileLinkPrefix: string,
+  sourceLocationPrefix: string
+) {
+  let cellValue: string;
+  switch (typeof value) {
+    case 'string':
+    case 'number':
+    case 'boolean':
+      cellValue = `\`${convertNonPrintableChars(value.toString())}\``;
+      break;
+    case 'object':
+      {
+        const url = tryGetRemoteLocation(value.url, fileLinkPrefix, sourceLocationPrefix);
+        if (url) {
+          cellValue = `[\`${convertNonPrintableChars(value.label)}\`](${url})`;
+        } else {
+          cellValue = `\`${convertNonPrintableChars(value.label)}\``;
+        }
+      }
+      break;
+  }
+  // `|` characters break the table, so we need to escape them
+  return cellValue.replaceAll('|', '\\|');
+}
+
+
+/**
+ * Creates a markdown link to a remote file.
+ * If the "link text" is not provided, we use the file path.
+ */
+export function createMarkdownRemoteFileRef(
+  fileLink: FileLink,
+  startLine?: number,
+  endLine?: number,
+  linkText?: string,
+): string {
+  const markdownLink = `[${linkText || fileLink.filePath}](${createRemoteFileRef(fileLink, startLine, endLine)})`;
+  return markdownLink;
+}
+
+/**
+ * Builds an expandable markdown section of the form:
+ * <details> 
+ * <summary>title</summary>
+ * 
+ * contents
+ * 
+ * </details>
+ */
+function buildExpandableMarkdownSection(title: string, contents: string[]): string[] {
+  const expandableLines: string[] = [];
+  expandableLines.push(
+    '<details>',
+    `<summary>${title}</summary>`,
+    '',
+    ...contents,
+    '',
+    '</details>',
+    ''
+  );
+  return expandableLines;
+}
+
+function createRelativeLink(fileName: string, linkType: MarkdownLinkType): string {
+  switch (linkType) {
+    case 'local':
+      return `./${fileName}.md`;
+
+    case 'gist':
+      // Creates an anchor link to a file in the gist. This is of the form:
+      // '#file-<name>-<file-extension>'
+      return `#file-${fileName}-md`;
+  }
+}
+
+/**
+ * Creates the name of the markdown file for a given repository nwo.
+ * This name doesn't include the file extension.
+ */
+function createFileName(nwo: string) {
+  const [owner, repo] = nwo.split('/');
+  return `${owner}-${repo}`;
+}
+
+/**
+ * Escape characters that could be interpreted as HTML instead of raw code.
+ */
+function escapeHtmlCharacters(text: string): string {
+  return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}

extensions/ql-vscode/src/remote-queries/remote-queries-monitor.ts

@@ -1,7 +1,7 @@
 import * as vscode from 'vscode';
 import { Credentials } from '../authentication';
 import { Logger } from '../logging';
-import { getWorkflowStatus } from './gh-actions-api-client';
+import { getWorkflowStatus, isArtifactAvailable, RESULT_INDEX_ARTIFACT_NAME } from './gh-api/gh-actions-api-client';
 import { RemoteQuery } from './remote-query';
 import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
 
@@ -42,7 +42,25 @@ export class RemoteQueriesMonitor {
         remoteQuery.controllerRepository.name,
         remoteQuery.actionsWorkflowRunId);
 
-      if (workflowStatus.status !== 'InProgress') {
+      // Even if the workflow indicates it has completed, artifacts
+      // might still take a while to become available. So we need to
+      // check for the artifact before we can declare the workflow
+      // as having completed.
+      if (workflowStatus.status === 'CompletedSuccessfully') {
+        const resultIndexAvailable = await isArtifactAvailable(
+          credentials,
+          remoteQuery.controllerRepository.owner,
+          remoteQuery.controllerRepository.name,
+          remoteQuery.actionsWorkflowRunId,
+          RESULT_INDEX_ARTIFACT_NAME
+        );
+
+        if (resultIndexAvailable) {
+          return workflowStatus;
+        }
+
+        // We don't have a result-index yet, so we'll keep monitoring.
+      } else if (workflowStatus.status !== 'InProgress') {
         return workflowStatus;
       }
 

extensions/ql-vscode/src/remote-queries/remote-queries-view.ts

@@ -1,10 +1,10 @@
 import {
-  WebviewPanel,
   ExtensionContext,
   window as Window,
   ViewColumn,
   Uri,
-  workspace
+  workspace,
+  commands,
 } from 'vscode';
 import * as path from 'path';
 
@@ -15,30 +15,36 @@ import {
   RemoteQueryDownloadAllAnalysesResultsMessage
 } from '../pure/interface-types';
 import { Logger } from '../logging';
-import { getHtmlForWebview } from '../interface-utils';
 import { assertNever } from '../pure/helpers-pure';
-import { AnalysisSummary, RemoteQueryResult } from './remote-query-result';
+import {
+  AnalysisSummary,
+  RemoteQueryResult,
+  sumAnalysisSummariesResults
+} from './remote-query-result';
 import { RemoteQuery } from './remote-query';
-import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
-import { AnalysisSummary as AnalysisResultViewModel } from './shared/remote-query-result';
+import {
+  AnalysisSummary as AnalysisResultViewModel,
+  RemoteQueryResult as RemoteQueryResultViewModel
+} from './shared/remote-query-result';
 import { showAndLogWarningMessage } from '../helpers';
 import { URLSearchParams } from 'url';
 import { SHOW_QUERY_TEXT_MSG } from '../query-history';
 import { AnalysesResultsManager } from './analyses-results-manager';
 import { AnalysisResults } from './shared/analysis-result';
+import { humanizeUnit } from '../pure/time';
+import { AbstractWebview, WebviewPanelConfig } from '../abstract-webview';
 
-export class RemoteQueriesInterfaceManager {
-  private panel: WebviewPanel | undefined;
-  private panelLoaded = false;
-  private panelLoadedCallBacks: (() => void)[] = [];
+export class RemoteQueriesView extends AbstractWebview<ToRemoteQueriesMessage, FromRemoteQueriesMessage> {
+  private currentQueryId: string | undefined;
 
   constructor(
-    private readonly ctx: ExtensionContext,
+    ctx: ExtensionContext,
     private readonly logger: Logger,
     private readonly analysesResultsManager: AnalysesResultsManager
   ) {
+    super(ctx);
     this.panelLoadedCallBacks.push(() => {
-      void logger.log('Remote queries view loaded');
+      void logger.log('Variant analysis results view loaded');
     });
   }
 
@@ -47,6 +53,8 @@ export class RemoteQueriesInterfaceManager {
 
     await this.waitForPanelLoaded();
     const model = this.buildViewModel(query, queryResult);
+    this.currentQueryId = queryResult.queryId;
+
     await this.postMessage({
       t: 'setRemoteQueryResult',
       queryResult: model
@@ -55,7 +63,7 @@ export class RemoteQueriesInterfaceManager {
     // Ensure all pre-downloaded artifacts are loaded into memory
     await this.analysesResultsManager.loadDownloadedAnalyses(model.analysisSummaries);
 
-    await this.setAnalysisResults(this.analysesResultsManager.getAnalysesResults(queryResult.queryId));
+    await this.setAnalysisResults(this.analysesResultsManager.getAnalysesResults(queryResult.queryId), queryResult.queryId);
   }
 
   /**
@@ -68,13 +76,14 @@ export class RemoteQueriesInterfaceManager {
    */
   private buildViewModel(query: RemoteQuery, queryResult: RemoteQueryResult): RemoteQueryResultViewModel {
     const queryFileName = path.basename(query.queryFilePath);
-    const totalResultCount = queryResult.analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
+    const totalResultCount = sumAnalysisSummariesResults(queryResult.analysisSummaries);
     const executionDuration = this.getDuration(queryResult.executionEndTime, query.executionStartTime);
     const analysisSummaries = this.buildAnalysisSummaries(queryResult.analysisSummaries);
     const totalRepositoryCount = queryResult.analysisSummaries.length;
     const affectedRepositories = queryResult.analysisSummaries.filter(r => r.resultCount > 0);
 
     return {
+      queryId: queryResult.queryId,
       queryTitle: query.queryName,
       queryFileName: queryFileName,
       queryFilePath: query.queryFilePath,
@@ -91,68 +100,56 @@ export class RemoteQueriesInterfaceManager {
     };
   }
 
-  getPanel(): WebviewPanel {
-    if (this.panel == undefined) {
-      const { ctx } = this;
-      const panel = (this.panel = Window.createWebviewPanel(
-        'remoteQueriesView',
-        'CodeQL Query Results',
-        { viewColumn: ViewColumn.Active, preserveFocus: true },
-        {
-          enableScripts: true,
-          enableFindWidget: true,
-          retainContextWhenHidden: true,
-          localResourceRoots: [
-            Uri.file(this.analysesResultsManager.storagePath),
-            Uri.file(path.join(this.ctx.extensionPath, 'out')),
-          ],
-        }
-      ));
-      this.panel.onDidDispose(
-        () => {
-          this.panel = undefined;
-        },
-        null,
-        ctx.subscriptions
-      );
-
-      const scriptPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/remoteQueriesView.js')
-      );
-
-      const baseStylesheetUriOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/remote-queries/view/baseStyles.css')
-      );
-
-      const stylesheetPathOnDisk = Uri.file(
-        ctx.asAbsolutePath('out/remote-queries/view/remoteQueries.css')
-      );
-
-      panel.webview.html = getHtmlForWebview(
-        panel.webview,
-        scriptPathOnDisk,
-        [baseStylesheetUriOnDisk, stylesheetPathOnDisk],
-        true
-      );
-      ctx.subscriptions.push(
-        panel.webview.onDidReceiveMessage(
-          async (e) => this.handleMsgFromView(e),
-          undefined,
-          ctx.subscriptions
-        )
-      );
-    }
-    return this.panel;
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: 'remoteQueriesView',
+      title: 'CodeQL Query Results',
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: 'remote-queries',
+      additionalOptions: {
+        localResourceRoots: [
+          Uri.file(this.analysesResultsManager.storagePath)
+        ]
+      }
+    };
   }
 
-  private waitForPanelLoaded(): Promise<void> {
-    return new Promise((resolve) => {
-      if (this.panelLoaded) {
-        resolve();
-      } else {
-        this.panelLoadedCallBacks.push(resolve);
-      }
-    });
+  protected onPanelDispose(): void {
+    this.currentQueryId = undefined;
+  }
+
+  protected async onMessage(msg: FromRemoteQueriesMessage): Promise<void> {
+    switch (msg.t) {
+      case 'viewLoaded':
+        this.onWebViewLoaded();
+        break;
+      case 'remoteQueryError':
+        void this.logger.log(
+          `Variant analysis error: ${msg.error}`
+        );
+        break;
+      case 'openFile':
+        await this.openFile(msg.filePath);
+        break;
+      case 'openVirtualFile':
+        await this.openVirtualFile(msg.queryText);
+        break;
+      case 'copyRepoList':
+        await commands.executeCommand('codeQL.copyRepoList', msg.queryId);
+        break;
+      case 'remoteQueryDownloadAnalysisResults':
+        await this.downloadAnalysisResults(msg);
+        break;
+      case 'remoteQueryDownloadAllAnalysesResults':
+        await this.downloadAllAnalysesResults(msg);
+        break;
+      case 'remoteQueryExportResults':
+        await commands.executeCommand('codeQL.exportVariantAnalysisResults', msg.queryId);
+        break;
+      default:
+        assertNever(msg);
+    }
   }
 
   private async openFile(filePath: string) {
@@ -180,82 +177,33 @@ export class RemoteQueriesInterfaceManager {
     }
   }
 
-  private async handleMsgFromView(
-    msg: FromRemoteQueriesMessage
-  ): Promise<void> {
-    switch (msg.t) {
-      case 'remoteQueryLoaded':
-        this.panelLoaded = true;
-        this.panelLoadedCallBacks.forEach((cb) => cb());
-        this.panelLoadedCallBacks = [];
-        break;
-      case 'remoteQueryError':
-        void this.logger.log(
-          `Variant analysis error: ${msg.error}`
-        );
-        break;
-      case 'openFile':
-        await this.openFile(msg.filePath);
-        break;
-      case 'openVirtualFile':
-        await this.openVirtualFile(msg.queryText);
-        break;
-      case 'remoteQueryDownloadAnalysisResults':
-        await this.downloadAnalysisResults(msg);
-        break;
-      case 'remoteQueryDownloadAllAnalysesResults':
-        await this.downloadAllAnalysesResults(msg);
-        break;
-      default:
-        assertNever(msg);
-    }
-  }
-
   private async downloadAnalysisResults(msg: RemoteQueryDownloadAnalysisResultsMessage): Promise<void> {
+    const queryId = this.currentQueryId;
     await this.analysesResultsManager.downloadAnalysisResults(
       msg.analysisSummary,
-      results => this.setAnalysisResults(results));
+      results => this.setAnalysisResults(results, queryId));
   }
 
   private async downloadAllAnalysesResults(msg: RemoteQueryDownloadAllAnalysesResultsMessage): Promise<void> {
+    const queryId = this.currentQueryId;
     await this.analysesResultsManager.loadAnalysesResults(
       msg.analysisSummaries,
       undefined,
-      results => this.setAnalysisResults(results));
+      results => this.setAnalysisResults(results, queryId));
   }
 
-  public async setAnalysisResults(analysesResults: AnalysisResults[]): Promise<void> {
-    if (this.panel?.active) {
+  public async setAnalysisResults(analysesResults: AnalysisResults[], queryId: string | undefined): Promise<void> {
+    if (this.panel?.active && this.currentQueryId === queryId) {
       await this.postMessage({
         t: 'setAnalysesResults',
-        analysesResults: analysesResults
+        analysesResults
       });
     }
   }
 
-  private postMessage(msg: ToRemoteQueriesMessage): Thenable<boolean> {
-    return this.getPanel().webview.postMessage(msg);
-  }
-
   private getDuration(startTime: number, endTime: number): string {
     const diffInMs = startTime - endTime;
-    return this.formatDuration(diffInMs);
-  }
-
-  private formatDuration(ms: number): string {
-    const seconds = ms / 1000;
-    const minutes = seconds / 60;
-    const hours = minutes / 60;
-    const days = hours / 24;
-    if (days > 1) {
-      return `${days.toFixed(2)} days`;
-    } else if (hours > 1) {
-      return `${hours.toFixed(2)} hours`;
-    } else if (minutes > 1) {
-      return `${minutes.toFixed(2)} minutes`;
-    } else {
-      return `${seconds.toFixed(2)} seconds`;
-    }
+    return humanizeUnit(diffInMs);
   }
 
   private formatDate = (millis: number): string => {
@@ -296,7 +244,10 @@ export class RemoteQueriesInterfaceManager {
       databaseSha: analysisResult.databaseSha || 'HEAD',
       resultCount: analysisResult.resultCount,
       downloadLink: analysisResult.downloadLink,
-      fileSize: this.formatFileSize(analysisResult.fileSizeInBytes)
+      sourceLocationPrefix: analysisResult.sourceLocationPrefix,
+      fileSize: this.formatFileSize(analysisResult.fileSizeInBytes),
+      starCount: analysisResult.starCount,
+      lastUpdated: analysisResult.lastUpdated
     }));
   }
 }

extensions/ql-vscode/src/remote-queries/remote-query-history-item.ts

@@ -7,9 +7,10 @@ import { RemoteQuery } from './remote-query';
 export interface RemoteQueryHistoryItem {
   readonly t: 'remote';
   failureReason?: string;
+  resultCount?: number;
   status: QueryStatus;
   completed: boolean;
   readonly queryId: string,
-  label: string; // TODO, the query label should have interpolation like local queries
   remoteQuery: RemoteQuery;
+  userSpecifiedLabel?: string;
 }

extensions/ql-vscode/src/remote-queries/remote-query-result-index.ts

@@ -12,6 +12,7 @@ export interface RemoteQuerySuccessIndexItem {
   resultCount: number;
   bqrsFileSize: number;
   sarifFileSize?: number;
+  sourceLocationPrefix: string;
 }
 
 export interface RemoteQueryFailureIndexItem {

extensions/ql-vscode/src/remote-queries/remote-query-result.ts

@@ -2,16 +2,26 @@ import { DownloadLink } from './download-link';
 import { AnalysisFailure } from './shared/analysis-failure';
 
 export interface RemoteQueryResult {
-  executionEndTime: number; // Can't use a Date here since it needs to be serialized and desserialized.
-  analysisSummaries: AnalysisSummary[];
-  analysisFailures: AnalysisFailure[];
-  queryId: string;
+  executionEndTime: number, // Can't use a Date here since it needs to be serialized and desserialized.
+  analysisSummaries: AnalysisSummary[],
+  analysisFailures: AnalysisFailure[],
+  queryId: string,
 }
 
 export interface AnalysisSummary {
   nwo: string,
   databaseSha: string,
   resultCount: number,
+  sourceLocationPrefix: string,
   downloadLink: DownloadLink,
-  fileSizeInBytes: number
+  fileSizeInBytes: number,
+  starCount?: number,
+  lastUpdated?: number,
 }
+
+/**
+ * Sums up the number of results for all repos queried via a remote query.
+ */
+export const sumAnalysisSummariesResults = (analysisSummaries: AnalysisSummary[]): number => {
+  return analysisSummaries.reduce((acc, cur) => acc + cur.resultCount, 0);
+};

extensions/ql-vscode/src/remote-queries/remote-query-submission-result.ts

@@ -1,6 +1,8 @@
 import { RemoteQuery } from './remote-query';
+import { VariantAnalysis } from './shared/variant-analysis';
 
 export interface RemoteQuerySubmissionResult {
   queryDirPath?: string;
   query?: RemoteQuery;
+  variantAnalysis?: VariantAnalysis;
 }

extensions/ql-vscode/src/remote-queries/remote-query.ts

@@ -8,4 +8,5 @@ export interface RemoteQuery {
   controllerRepository: Repository;
   executionStartTime: number; // Use number here since it needs to be serialized and desserialized.
   actionsWorkflowRunId: number;
+  repositoryCount: number;
 }

extensions/ql-vscode/src/remote-queries/repository-selection.ts

@@ -1,54 +1,210 @@
+import * as fs from 'fs-extra';
 import { QuickPickItem, window } from 'vscode';
-import { showAndLogErrorMessage } from '../helpers';
-import { getRemoteRepositoryLists } from '../config';
 import { logger } from '../logging';
-import { REPO_REGEX } from '../pure/helpers-pure';
+import { getRemoteRepositoryLists, getRemoteRepositoryListsPath } from '../config';
+import { OWNER_REGEX, REPO_REGEX } from '../pure/helpers-pure';
+import { UserCancellationException } from '../commandRunner';
+
+export interface RepositorySelection {
+  repositories?: string[];
+  repositoryLists?: string[];
+  owners?: string[];
+}
 
 interface RepoListQuickPickItem extends QuickPickItem {
-  repoList: string[];
+  repositories?: string[];
+  repositoryList?: string;
+  useCustomRepo?: boolean;
+  useAllReposOfOwner?: boolean;
+}
+
+interface RepoList {
+  label: string;
+  repositories: string[];
 }
 
 /**
- * Gets the repositories to run the query against.
+ * Gets the repositories or repository lists to run the query against.
+ * @returns The user selection.
  */
-export async function getRepositories(): Promise<string[] | undefined> {
-  const repoLists = getRemoteRepositoryLists();
-  if (repoLists && Object.keys(repoLists).length) {
-    const quickPickItems = Object.entries(repoLists).map<RepoListQuickPickItem>(([key, value]) => (
-      {
-        label: key,       // the name of the repository list
-        repoList: value,  // the actual array of repositories
-      }
-    ));
-    const quickpick = await window.showQuickPick<RepoListQuickPickItem>(
-      quickPickItems,
-      {
-        placeHolder: 'Select a repository list. You can define repository lists in the `codeQL.variantAnalysis.repositoryLists` setting.',
-        ignoreFocusOut: true,
-      });
-    if (quickpick?.repoList.length) {
-      void logger.log(`Selected repositories: ${quickpick.repoList.join(', ')}`);
-      return quickpick.repoList;
-    } else {
-      void showAndLogErrorMessage('No repositories selected.');
-      return;
+export async function getRepositorySelection(): Promise<RepositorySelection> {
+  const quickPickItems = [
+    createCustomRepoQuickPickItem(),
+    createAllReposOfOwnerQuickPickItem(),
+    ...createSystemDefinedRepoListsQuickPickItems(),
+    ...(await createUserDefinedRepoListsQuickPickItems()),
+  ];
+
+  const options = {
+    placeHolder: 'Select a repository list. You can define repository lists in the `codeQL.variantAnalysis.repositoryLists` setting.',
+    ignoreFocusOut: true,
+  };
+
+  const quickpick = await window.showQuickPick<RepoListQuickPickItem>(
+    quickPickItems,
+    options);
+
+  if (quickpick?.repositories?.length) {
+    void logger.log(`Selected repositories: ${quickpick.repositories.join(', ')}`);
+    return { repositories: quickpick.repositories };
+  } else if (quickpick?.repositoryList) {
+    void logger.log(`Selected repository list: ${quickpick.repositoryList}`);
+    return { repositoryLists: [quickpick.repositoryList] };
+  } else if (quickpick?.useCustomRepo) {
+    const customRepo = await getCustomRepo();
+    if (customRepo === undefined) {
+      // The user cancelled, do nothing.
+      throw new UserCancellationException('No repositories selected', true);
     }
+    if (!customRepo || !REPO_REGEX.test(customRepo)) {
+      throw new UserCancellationException('Invalid repository format. Please enter a valid repository in the format <owner>/<repo> (e.g. github/codeql)');
+    }
+    void logger.log(`Entered repository: ${customRepo}`);
+    return { repositories: [customRepo] };
+  } else if (quickpick?.useAllReposOfOwner) {
+    const owner = await getOwner();
+    if (owner === undefined) {
+      // The user cancelled, do nothing.
+      throw new UserCancellationException('No repositories selected', true);
+    }
+    if (!owner || !OWNER_REGEX.test(owner)) {
+      throw new Error(`Invalid user or organization: ${owner}`);
+    }
+    void logger.log(`Entered owner: ${owner}`);
+    return { owners: [owner] };
   } else {
-    void logger.log('No repository lists defined. Displaying text input box.');
-    const remoteRepo = await window.showInputBox({
-      title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
-      placeHolder: '<owner>/<repo>',
-      prompt: 'Tip: you can save frequently used repositories in the `codeQL.variantAnalysis.repositoryLists` setting',
-      ignoreFocusOut: true,
-    });
-    if (!remoteRepo) {
-      void showAndLogErrorMessage('No repositories entered.');
-      return;
-    } else if (!REPO_REGEX.test(remoteRepo)) { // Check if user entered invalid input
-      void showAndLogErrorMessage('Invalid repository format. Must be in the format <owner>/<repo> (e.g. github/codeql)');
-      return;
-    }
-    void logger.log(`Entered repository: ${remoteRepo}`);
-    return [remoteRepo];
+    // We don't need to display a warning pop-up in this case, since the user just escaped out of the operation.
+    // We set 'true' to make this a silent exception.
+    throw new UserCancellationException('No repositories selected', true);
   }
 }
+
+/**
+ * Checks if the selection is valid or not.
+ * @param repoSelection The selection to check.
+ * @returns A boolean flag indicating if the selection is valid or not.
+ */
+export function isValidSelection(repoSelection: RepositorySelection): boolean {
+  const repositories = repoSelection.repositories || [];
+  const repositoryLists = repoSelection.repositoryLists || [];
+  const owners = repoSelection.owners || [];
+
+  return (repositories.length > 0 || repositoryLists.length > 0 || owners.length > 0);
+}
+
+function createSystemDefinedRepoListsQuickPickItems(): RepoListQuickPickItem[] {
+  const topNs = [10, 100, 1000];
+
+  return topNs.map(n => ({
+    label: '$(star) Top ' + n,
+    repositoryList: `top_${n}`,
+    alwaysShow: true
+  } as RepoListQuickPickItem));
+}
+
+async function readExternalRepoLists(): Promise<RepoList[]> {
+  const repoLists: RepoList[] = [];
+
+  const path = getRemoteRepositoryListsPath();
+  if (!path) {
+    return repoLists;
+  }
+
+  await validateExternalRepoListsFile(path);
+  const json = await readExternalRepoListsJson(path);
+
+  for (const [repoListName, repositories] of Object.entries(json)) {
+    if (!Array.isArray(repositories)) {
+      throw Error('Invalid repository lists file. It should contain an array of repositories for each list.');
+    }
+
+    repoLists.push({
+      label: repoListName,
+      repositories
+    });
+  }
+
+  return repoLists;
+}
+
+async function validateExternalRepoListsFile(path: string): Promise<void> {
+  const pathExists = await fs.pathExists(path);
+  if (!pathExists) {
+    throw Error(`External repository lists file does not exist at ${path}`);
+  }
+
+  const pathStat = await fs.stat(path);
+  if (pathStat.isDirectory()) {
+    throw Error('External repository lists path should not point to a directory');
+  }
+}
+
+async function readExternalRepoListsJson(path: string): Promise<Record<string, unknown>> {
+  let json;
+
+  try {
+    const fileContents = await fs.readFile(path, 'utf8');
+    json = await JSON.parse(fileContents);
+  } catch (error) {
+    throw Error('Invalid repository lists file. It should contain valid JSON.');
+  }
+
+  if (Array.isArray(json)) {
+    throw Error('Invalid repository lists file. It should be an object mapping names to a list of repositories.');
+  }
+
+  return json;
+}
+
+function readRepoListsFromSettings(): RepoList[] {
+  const repoLists = getRemoteRepositoryLists();
+  if (!repoLists) {
+    return [];
+  }
+
+  return Object.entries(repoLists).map<RepoList>(([label, repositories]) => (
+    {
+      label,
+      repositories
+    }
+  ));
+}
+
+async function createUserDefinedRepoListsQuickPickItems(): Promise<RepoListQuickPickItem[]> {
+  const repoListsFromSetings = readRepoListsFromSettings();
+  const repoListsFromExternalFile = await readExternalRepoLists();
+
+  return [...repoListsFromSetings, ...repoListsFromExternalFile];
+}
+
+function createCustomRepoQuickPickItem(): RepoListQuickPickItem {
+  return {
+    label: '$(edit) Enter a GitHub repository',
+    useCustomRepo: true,
+    alwaysShow: true,
+  };
+}
+
+function createAllReposOfOwnerQuickPickItem(): RepoListQuickPickItem {
+  return {
+    label: '$(edit) Enter a GitHub user or organization',
+    useAllReposOfOwner: true,
+    alwaysShow: true
+  };
+}
+
+async function getCustomRepo(): Promise<string | undefined> {
+  return await window.showInputBox({
+    title: 'Enter a GitHub repository in the format <owner>/<repo> (e.g. github/codeql)',
+    placeHolder: '<owner>/<repo>',
+    prompt: 'Tip: you can save frequently used repositories in the `codeQL.variantAnalysis.repositoryLists` setting',
+    ignoreFocusOut: true,
+  });
+}
+
+async function getOwner(): Promise<string | undefined> {
+  return await window.showInputBox({
+    title: 'Enter a GitHub user or organization',
+    ignoreFocusOut: true
+  });
+}

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -1,7 +1,8 @@
-import { CancellationToken, Uri, window } from 'vscode';
+import { CancellationToken, commands, Uri, window } from 'vscode';
 import * as path from 'path';
 import * as yaml from 'js-yaml';
 import * as fs from 'fs-extra';
+import * as os from 'os';
 import * as tmp from 'tmp-promise';
 import {
   askForLanguage,
@@ -10,19 +11,25 @@ import {
   showAndLogErrorMessage,
   showAndLogInformationMessage,
   tryGetQueryMetadata,
-  tmpDir
+  pluralize,
+  tmpDir,
 } from '../helpers';
 import { Credentials } from '../authentication';
 import * as cli from '../cli';
 import { logger } from '../logging';
-import { getActionBranch, getRemoteControllerRepo, setRemoteControllerRepo } from '../config';
+import { getActionBranch, getRemoteControllerRepo, isVariantAnalysisLiveResultsEnabled, setRemoteControllerRepo } from '../config';
 import { ProgressCallback, UserCancellationException } from '../commandRunner';
-import { OctokitResponse } from '@octokit/types/dist-types';
+import { OctokitResponse, RequestError } from '@octokit/types/dist-types';
 import { RemoteQuery } from './remote-query';
 import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
 import { QueryMetadata } from '../pure/interface-types';
 import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
-import { getRepositories } from './repository-selection';
+import * as ghApiClient from './gh-api/gh-api-client';
+import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
+import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
+import { Repository } from './shared/repository';
+import { processVariantAnalysis } from './variant-analysis-processor';
+import { VariantAnalysisManager } from './variant-analysis-manager';
 
 export interface QlPack {
   name: string;
@@ -33,7 +40,15 @@ export interface QlPack {
 }
 
 interface QueriesResponse {
-  workflow_run_id: number
+  workflow_run_id: number,
+  errors?: {
+    invalid_repositories?: string[],
+    repositories_without_database?: string[],
+    private_repositories?: string[],
+    cutoff_repositories?: string[],
+    cutoff_repositories_count?: number,
+  },
+  repositories_queried: string[],
 }
 
 /**
@@ -104,7 +119,7 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri
         [`codeql/${language}-all`]: '*',
       }
     };
-    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.safeDump(syntheticQueryPack));
+    await fs.writeFile(path.join(queryPackDir, 'qlpack.yml'), yaml.dump(syntheticQueryPack));
   }
   if (!language) {
     throw new UserCancellationException('Could not determine language.');
@@ -133,7 +148,7 @@ async function findPackRoot(queryFile: string): Promise<string> {
   while (!(await fs.pathExists(path.join(dir, 'qlpack.yml')))) {
     dir = path.dirname(dir);
     if (isFileSystemRoot(dir)) {
-      // there is no qlpack.yml in this direcory or any parent directory.
+      // there is no qlpack.yml in this directory or any parent directory.
       // just use the query file's directory as the pack root.
       return path.dirname(queryFile);
     }
@@ -168,10 +183,11 @@ export async function runRemoteQuery(
   uri: Uri | undefined,
   dryRun: boolean,
   progress: ProgressCallback,
-  token: CancellationToken
+  token: CancellationToken,
+  variantAnalysisManager: VariantAnalysisManager
 ): Promise<void | RemoteQuerySubmissionResult> {
   if (!(await cliServer.cliConstraints.supportsRemoteQueries())) {
-    throw new Error(`Remote queries are not supported by this version of CodeQL. Please upgrade to v${cli.CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
+    throw new Error(`Variant analysis is not supported by this version of CodeQL. Please upgrade to v${cli.CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES
       } or later.`);
   }
 
@@ -189,8 +205,8 @@ export async function runRemoteQuery(
       message: 'Determining query target language'
     });
 
-    const repositories = await getRepositories();
-    if (!repositories || repositories.length === 0) {
+    const repoSelection = await getRepositorySelection();
+    if (!isValidSelection(repoSelection)) {
       throw new UserCancellationException('No repositories to query.');
     }
 
@@ -200,31 +216,7 @@ export async function runRemoteQuery(
       message: 'Determining controller repo'
     });
 
-    // Get the controller repo from the config, if it exists.
-    // If it doesn't exist, prompt the user to enter it, and save that value to the config.
-    let controllerRepo: string | undefined;
-    controllerRepo = getRemoteControllerRepo();
-    if (!controllerRepo || !REPO_REGEX.test(controllerRepo)) {
-      void logger.log(controllerRepo ? 'Invalid controller repository name.' : 'No controller repository defined.');
-      controllerRepo = await window.showInputBox({
-        title: 'Controller repository in which to display progress and results of remote queries',
-        placeHolder: '<owner>/<repo>',
-        prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
-        ignoreFocusOut: true,
-      });
-      if (!controllerRepo) {
-        void showAndLogErrorMessage('No controller repository entered.');
-        return;
-      } else if (!REPO_REGEX.test(controllerRepo)) { // Check if user entered invalid input
-        void showAndLogErrorMessage('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
-        return;
-      }
-      void logger.log(`Setting the controller repository as: ${controllerRepo}`);
-      await setRemoteControllerRepo(controllerRepo);
-    }
-
-    void logger.log(`Using controller repository: ${controllerRepo}`);
-    const [owner, repo] = controllerRepo.split('/');
+    const controllerRepo = await getControllerRepo(credentials);
 
     progress({
       maxStep: 4,
@@ -249,28 +241,74 @@ export async function runRemoteQuery(
     });
 
     const actionBranch = getActionBranch();
-    const workflowRunId = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repositories, owner, repo, base64Pack, dryRun);
     const queryStartTime = Date.now();
     const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
 
-    if (dryRun) {
-      return { queryDirPath: remoteQueryDir.path };
-    } else {
-      if (!workflowRunId) {
-        return;
+    if (isVariantAnalysisLiveResultsEnabled()) {
+      const queryName = getQueryName(queryMetadata, queryFile);
+      const variantAnalysisLanguage = parseVariantAnalysisQueryLanguage(language);
+      if (variantAnalysisLanguage === undefined) {
+        throw new UserCancellationException(`Found unsupported language: ${language}`);
       }
 
-      const remoteQuery = await buildRemoteQueryEntity(
-        queryFile,
-        queryMetadata,
-        owner,
-        repo,
-        queryStartTime,
-        workflowRunId,
-        language);
+      const variantAnalysisSubmission: VariantAnalysisSubmission = {
+        startTime: queryStartTime,
+        actionRepoRef: actionBranch,
+        controllerRepoId: controllerRepo.id,
+        query: {
+          name: queryName,
+          filePath: queryFile,
+          pack: base64Pack,
+          language: variantAnalysisLanguage,
+        },
+        databases: {
+          repositories: repoSelection.repositories,
+          repositoryLists: repoSelection.repositoryLists,
+          repositoryOwners: repoSelection.owners
+        }
+      };
 
-      // don't return the path because it has been deleted
-      return { query: remoteQuery };
+      const variantAnalysisResponse = await ghApiClient.submitVariantAnalysis(
+        credentials,
+        variantAnalysisSubmission
+      );
+
+      const processedVariantAnalysis = processVariantAnalysis(variantAnalysisSubmission, variantAnalysisResponse);
+
+      variantAnalysisManager.onVariantAnalysisSubmitted(processedVariantAnalysis);
+
+      void logger.log(`Variant analysis:\n${JSON.stringify(processedVariantAnalysis, null, 2)}`);
+
+      void showAndLogInformationMessage(`Variant analysis ${processedVariantAnalysis.query.name} submitted for processing`);
+
+      void commands.executeCommand('codeQL.openVariantAnalysisView', processedVariantAnalysis.id);
+      void commands.executeCommand('codeQL.monitorVariantAnalysis', processedVariantAnalysis);
+
+      return { variantAnalysis: processedVariantAnalysis };
+    } else {
+      const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, controllerRepo, base64Pack, dryRun);
+
+      if (dryRun) {
+        return { queryDirPath: remoteQueryDir.path };
+      } else {
+        if (!apiResponse) {
+          return;
+        }
+
+        const workflowRunId = apiResponse.workflow_run_id;
+        const repositoryCount = apiResponse.repositories_queried.length;
+        const remoteQuery = await buildRemoteQueryEntity(
+          queryFile,
+          queryMetadata,
+          controllerRepo,
+          queryStartTime,
+          workflowRunId,
+          language,
+          repositoryCount);
+
+        // don't return the path because it has been deleted
+        return { query: remoteQuery };
+      }
     }
 
   } finally {
@@ -287,41 +325,101 @@ async function runRemoteQueriesApiRequest(
   credentials: Credentials,
   ref: string,
   language: string,
-  repositories: string[],
-  owner: string,
-  repo: string,
+  repoSelection: RepositorySelection,
+  controllerRepo: Repository,
   queryPackBase64: string,
   dryRun = false
-): Promise<void | number> {
+): Promise<void | QueriesResponse> {
+  const data = {
+    ref,
+    language,
+    repositories: repoSelection.repositories ?? undefined,
+    repository_lists: repoSelection.repositoryLists ?? undefined,
+    repository_owners: repoSelection.owners ?? undefined,
+    query_pack: queryPackBase64,
+  };
+
   if (dryRun) {
     void showAndLogInformationMessage('[DRY RUN] Would have sent request. See extension log for the payload.');
-    void logger.log(JSON.stringify({ ref, language, repositories, owner, repo, queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes' }));
+    void logger.log(JSON.stringify({
+      controllerRepo,
+      data: {
+        ...data,
+        queryPackBase64: queryPackBase64.substring(0, 100) + '... ' + queryPackBase64.length + ' bytes'
+      }
+    }));
     return;
   }
 
   try {
     const octokit = await credentials.getOctokit();
     const response: OctokitResponse<QueriesResponse, number> = await octokit.request(
-      'POST /repos/:owner/:repo/code-scanning/codeql/queries',
+      'POST /repositories/:controllerRepoId/code-scanning/codeql/queries',
       {
-        owner,
-        repo,
-        data: {
-          ref,
-          language,
-          repositories,
-          query_pack: queryPackBase64,
-        }
+        controllerRepoId: controllerRepo.id,
+        data
       }
     );
-    const workflowRunId = response.data.workflow_run_id;
-    void showAndLogInformationMessage(`Successfully scheduled runs. [Click here to see the progress](https://github.com/${owner}/${repo}/actions/runs/${workflowRunId}).`);
-    return workflowRunId;
-  } catch (error) {
-    void showAndLogErrorMessage(getErrorMessage(error));
+    const { popupMessage, logMessage } = parseResponse(controllerRepo, response.data);
+    void showAndLogInformationMessage(popupMessage, { fullMessage: logMessage });
+    return response.data;
+  } catch (error: any) {
+    if (error.status === 404) {
+      void showAndLogErrorMessage(`Controller repository was not found. Please make sure it's a valid repo name.${eol}`);
+    } else {
+      void showAndLogErrorMessage(getErrorMessage(error));
+    }
   }
 }
 
+const eol = os.EOL;
+const eol2 = os.EOL + os.EOL;
+
+// exported for testing only
+export function parseResponse(controllerRepo: Repository, response: QueriesResponse) {
+  const repositoriesQueried = response.repositories_queried;
+  const repositoryCount = repositoriesQueried.length;
+
+  const popupMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. [Click here to see the progress](https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}).`
+    + (response.errors ? `${eol2}Some repositories could not be scheduled. See extension log for details.` : '');
+
+  let logMessage = `Successfully scheduled runs on ${pluralize(repositoryCount, 'repository', 'repositories')}. See https://github.com/${controllerRepo.fullName}/actions/runs/${response.workflow_run_id}.`;
+  logMessage += `${eol2}Repositories queried:${eol}${repositoriesQueried.join(', ')}`;
+  if (response.errors) {
+    const { invalid_repositories, repositories_without_database, private_repositories, cutoff_repositories, cutoff_repositories_count } = response.errors;
+    logMessage += `${eol2}Some repositories could not be scheduled.`;
+    if (invalid_repositories?.length) {
+      logMessage += `${eol2}${pluralize(invalid_repositories.length, 'repository', 'repositories')} invalid and could not be found:${eol}${invalid_repositories.join(', ')}`;
+    }
+    if (repositories_without_database?.length) {
+      logMessage += `${eol2}${pluralize(repositories_without_database.length, 'repository', 'repositories')} did not have a CodeQL database available:${eol}${repositories_without_database.join(', ')}`;
+      logMessage += `${eol}For each public repository that has not yet been added to the database service, we will try to create a database next time the store is updated.`;
+    }
+    if (private_repositories?.length) {
+      logMessage += `${eol2}${pluralize(private_repositories.length, 'repository', 'repositories')} not public:${eol}${private_repositories.join(', ')}`;
+      logMessage += `${eol}When using a public controller repository, only public repositories can be queried.`;
+    }
+    if (cutoff_repositories_count) {
+      logMessage += `${eol2}${pluralize(cutoff_repositories_count, 'repository', 'repositories')} over the limit for a single request`;
+      if (cutoff_repositories) {
+        logMessage += `:${eol}${cutoff_repositories.join(', ')}`;
+        if (cutoff_repositories_count !== cutoff_repositories.length) {
+          const moreRepositories = cutoff_repositories_count - cutoff_repositories.length;
+          logMessage += `${eol}...${eol}And another ${pluralize(moreRepositories, 'repository', 'repositories')}.`;
+        }
+      } else {
+        logMessage += '.';
+      }
+      logMessage += `${eol}Repositories were selected based on how recently they had been updated.`;
+    }
+  }
+
+  return {
+    popupMessage,
+    logMessage
+  };
+}
+
 /**
  * Updates the default suite of the query pack. This is used to ensure
  * only the specified query is run.
@@ -333,7 +431,7 @@ async function runRemoteQueriesApiRequest(
  */
 async function ensureNameAndSuite(queryPackDir: string, packRelativePath: string): Promise<void> {
   const packPath = path.join(queryPackDir, 'qlpack.yml');
-  const qlpack = yaml.safeLoad(await fs.readFile(packPath, 'utf8')) as QlPack;
+  const qlpack = yaml.load(await fs.readFile(packPath, 'utf8')) as QlPack;
   delete qlpack.defaultSuiteFile;
 
   qlpack.name = QUERY_PACK_NAME;
@@ -343,22 +441,21 @@ async function ensureNameAndSuite(queryPackDir: string, packRelativePath: string
   }, {
     query: packRelativePath.replace(/\\/g, '/')
   }];
-  await fs.writeFile(packPath, yaml.safeDump(qlpack));
+  await fs.writeFile(packPath, yaml.dump(qlpack));
 }
 
 async function buildRemoteQueryEntity(
   queryFilePath: string,
   queryMetadata: QueryMetadata | undefined,
-  controllerRepoOwner: string,
-  controllerRepoName: string,
+  controllerRepo: Repository,
   queryStartTime: number,
   workflowRunId: number,
-  language: string
+  language: string,
+  repositoryCount: number
 ): Promise<RemoteQuery> {
-  // The query name is either the name as specified in the query metadata, or the file name.
-  const queryName = queryMetadata?.name ?? path.basename(queryFilePath);
-
+  const queryName = getQueryName(queryMetadata, queryFilePath);
   const queryText = await fs.readFile(queryFilePath, 'utf8');
+  const [owner, name] = controllerRepo.fullName.split('/');
 
   return {
     queryName,
@@ -366,10 +463,59 @@ async function buildRemoteQueryEntity(
     queryText,
     language,
     controllerRepository: {
-      owner: controllerRepoOwner,
-      name: controllerRepoName,
+      owner,
+      name,
     },
     executionStartTime: queryStartTime,
-    actionsWorkflowRunId: workflowRunId
+    actionsWorkflowRunId: workflowRunId,
+    repositoryCount,
   };
 }
+
+function getQueryName(queryMetadata: QueryMetadata | undefined, queryFilePath: string): string {
+  // The query name is either the name as specified in the query metadata, or the file name.
+  return queryMetadata?.name ?? path.basename(queryFilePath);
+}
+
+export async function getControllerRepo(credentials: Credentials): Promise<Repository> {
+  // Get the controller repo from the config, if it exists.
+  // If it doesn't exist, prompt the user to enter it, and save that value to the config.
+  let controllerRepoNwo: string | undefined;
+  controllerRepoNwo = getRemoteControllerRepo();
+  if (!controllerRepoNwo || !REPO_REGEX.test(controllerRepoNwo)) {
+    void logger.log(controllerRepoNwo ? 'Invalid controller repository name.' : 'No controller repository defined.');
+    controllerRepoNwo = await window.showInputBox({
+      title: 'Controller repository in which to run the GitHub Actions workflow for this variant analysis',
+      placeHolder: '<owner>/<repo>',
+      prompt: 'Enter the name of a GitHub repository in the format <owner>/<repo>',
+      ignoreFocusOut: true,
+    });
+    if (!controllerRepoNwo) {
+      throw new UserCancellationException('No controller repository entered.');
+    } else if (!REPO_REGEX.test(controllerRepoNwo)) { // Check if user entered invalid input
+      throw new UserCancellationException('Invalid repository format. Must be a valid GitHub repository in the format <owner>/<repo>.');
+    }
+    void logger.log(`Setting the controller repository as: ${controllerRepoNwo}`);
+    await setRemoteControllerRepo(controllerRepoNwo);
+  }
+
+  void logger.log(`Using controller repository: ${controllerRepoNwo}`);
+  const [owner, repo] = controllerRepoNwo.split('/');
+
+  try {
+    const controllerRepo = await ghApiClient.getRepositoryFromNwo(credentials, owner, repo);
+    void logger.log(`Controller repository ID: ${controllerRepo.id}`);
+    return {
+      id: controllerRepo.id,
+      fullName: controllerRepo.full_name,
+      private: controllerRepo.private,
+    };
+
+  } catch (e: any) {
+    if ((e as RequestError).status === 404) {
+      throw new Error(`Controller repository "${owner}/${repo}" not found`);
+    } else {
+      throw new Error(`Error getting controller repository "${owner}/${repo}": ${e.message}`);
+    }
+  }
+}

extensions/ql-vscode/src/remote-queries/sarif-processing.ts

@@ -86,7 +86,7 @@ function getShortDescription(
     return rule.shortDescription.text;
   }
 
-  return message.tokens.map(token => token.text).join();
+  return message.tokens.map(token => token.text).join('');
 }
 
 export function tryGetSeverity(

extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts

@@ -7,12 +7,16 @@ export interface AnalysisResults {
   status: AnalysisResultStatus;
   interpretedResults: AnalysisAlert[];
   rawResults?: AnalysisRawResults;
+  resultCount: number,
+  starCount?: number,
+  lastUpdated?: number,
 }
 
 export interface AnalysisRawResults {
   schema: ResultSetSchema;
   resultSet: RawResultSet;
   fileLinkPrefix: string;
+  sourceLocationPrefix: string;
   capped: boolean;
 }
 
@@ -78,3 +82,17 @@ export interface AnalysisMessageLocationToken {
 }
 
 export type ResultSeverity = 'Recommendation' | 'Warning' | 'Error';
+
+/**
+ * Returns the number of (raw + interpreted) results for an analysis.
+ */
+export const getAnalysisResultCount = (analysisResults: AnalysisResults): number => {
+  const rawResultCount = analysisResults.rawResults?.resultSet.rows.length || 0;
+  return analysisResults.interpretedResults.length + rawResultCount;
+};
+
+/**
+ * Returns the total number of results for an analysis by adding all individual repo results.
+ */
+export const sumAnalysesResults = (analysesResults: AnalysisResults[]) =>
+  analysesResults.reduce((acc, curr) => acc + getAnalysisResultCount(curr), 0);