Update changelog

Fix docker command

.github/codeql/queries/ProgressBar.qll

@@ -0,0 +1,16 @@
+import javascript
+
+class WithProgressCall extends CallExpr {
+  WithProgressCall() { this.getCalleeName() = "withProgress" }
+
+  predicate usesToken() { exists(this.getTokenParameter()) }
+
+  Parameter getTokenParameter() { result = this.getArgument(0).(Function).getParameter(1) }
+
+  Property getCancellableProperty() { result = this.getArgument(1).(ObjectExpr).getPropertyByName("cancellable") }
+
+  predicate isCancellable() {
+    this.getCancellableProperty().getInit().(BooleanLiteral).getBoolValue() =
+      true
+  }
+}

.github/codeql/queries/progress-not-cancellable.ql

@@ -0,0 +1,20 @@
+/**
+ * @name Using token for non-cancellable progress bar
+ * @kind problem
+ * @problem.severity warning
+ * @id vscode-codeql/progress-not-cancellable
+ * @description If we call `withProgress` without `cancellable: true` then the
+ * token that is given to us should be ignored because it won't ever be cancelled.
+ * This makes the code more confusing as it tries to account for cases that can't
+ * happen. The fix is to either not use the token or make the progress bar cancellable.
+ */
+
+import javascript
+import ProgressBar
+
+from WithProgressCall t
+where not t.isCancellable() and t.usesToken()
+select t,
+  "The $@ should not be used when the progress bar is not cancellable. Either stop using the $@ or mark the progress bar as cancellable.",
+  t.getTokenParameter(), t.getTokenParameter().getName(), t.getTokenParameter(),
+  t.getTokenParameter().getName()

.github/codeql/queries/token-not-used.ql

@@ -0,0 +1,18 @@
+/**
+ * @name Don't ignore the token for a cancellable progress bar
+ * @kind problem
+ * @problem.severity warning
+ * @id vscode-codeql/token-not-used
+ * @description If we call `withProgress` with `cancellable: true` but then
+ * ignore the token that is given to us, it will lead to a poor user experience
+ * because the progress bar will appear to be canceled but it will not actually
+ * affect the background process. Either check the token and respect when it
+ * has been cancelled, or mark the progress bar as not cancellable.
+ */
+
+import javascript
+import ProgressBar
+
+from WithProgressCall t
+where t.isCancellable() and not t.usesToken()
+select t, "This progress bar is $@ but the token is not used. Either use the token or mark the progress bar as not cancellable.", t.getCancellableProperty(), "cancellable"

.github/dependabot.yml

@@ -13,6 +13,23 @@ updates:
       # are unrelated to the Node version, so we allow those.
       - dependency-name: "@types/node"
         update-types: ["version-update:semver-major", "version-update:semver-minor"]
+    groups:
+      octokit:
+        patterns:
+          - "@octokit/*"
+        update-types:
+          - "minor"
+          - "patch"
+      storybook:
+        patterns:
+          - "@storybook/*"
+          - "storybook"
+      testing-library:
+        patterns:
+          - "@testing-library/*"
+      typescript-eslint:
+        patterns:
+          - "@typescript-eslint/*"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
@@ -20,3 +37,10 @@ updates:
       day: "thursday" # Thursday is arbitrary
     labels:
       - "Update dependencies"
+  - package-ecosystem: docker
+    directory: "extensions/ql-vscode/test/e2e"
+    schedule:
+      interval: "weekly"
+      day: "thursday" # Thursday is arbitrary
+    labels:
+      - "Update dependencies"

.github/pull_request_template.md

@@ -5,8 +5,4 @@
 
 Replace this with a description of the changes your pull request makes.
 
-## Checklist
-
-- [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
-- [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] _[Maintainers only]_ If this pull request makes user-facing changes that require documentation changes, open a corresponding docs pull request in the [github/codeql](https://github.com/github/codeql/tree/main/docs/codeql/codeql-for-visual-studio-code) repo and add the `ready-for-doc-review` label there.
+Remember to update the [changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) if there have been user-facing changes!

.github/workflows/cli-test.yml

@@ -11,6 +11,7 @@ on:
       - extensions/ql-vscode/src/language-support/**
       - extensions/ql-vscode/src/query-server/**
       - extensions/ql-vscode/supported_cli_versions.json
+      - extensions/ql-vscode/src/variant-analysis/run-remote-query.ts
 
 jobs:
   find-nightly:

.github/workflows/dependency-review.yml

@@ -13,4 +13,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@v4
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4

.github/workflows/e2e-tests.yml

@@ -0,0 +1,46 @@
+name: Run E2E Playwright tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  e2e-test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: extensions/ql-vscode/.nvmrc
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: npm ci
+
+      - name: Start containers
+        working-directory: extensions/ql-vscode/test/e2e
+        run: docker compose -f "docker-compose.yml" up -d --build
+
+      - name: Install Playwright Browsers
+        working-directory: extensions/ql-vscode
+        run: npx playwright install --with-deps
+      - name: Run Playwright tests
+        working-directory: extensions/ql-vscode/test/e2e
+        run: npx playwright test
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: extensions/ql-vscode/playwright-report/
+          retention-days: 30
+      - name: Stop containers
+        working-directory: extensions/ql-vscode/test/e2e
+        if: always()
+        run: docker compose -f "docker-compose.yml" down -v

.github/workflows/main.yml

@@ -144,6 +144,19 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
     steps:
+      # Enable 8.3 filename creation. This is not required to run the extension but it is required for the unit tests to pass.
+      # This feature is currently enabled by default in Windows 11 for the C: drive and therefore we must maintain support for it.
+      # This setting needs to be enabled before files are created, i.e. before we checkout the repository.
+      - name: Enable 8.3 filenames
+        shell: pwsh
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          $shortNameEnableProcess = Start-Process -FilePath fsutil.exe -ArgumentList ('8dot3name', 'set', '0') -Wait -PassThru
+          $shortNameEnableExitCode = $shortNameEnableProcess.ExitCode
+          if ($shortNameEnableExitCode -ne 0) {
+              exit $shortNameEnableExitCode
+          }
+
       - name: Checkout
         uses: actions/checkout@v4
         with:

.github/workflows/release.yml

@@ -134,7 +134,7 @@ jobs:
 
       - name: Publish to Registry
         run: |
-          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix
+          npx @vscode/vsce publish -p $VSCE_TOKEN --packagePath *.vsix
 
   open-vsx-publish:
     name: Publish to Open VSX Registry

.gitignore

@@ -19,3 +19,7 @@ artifacts/
 # CodeQL metadata
 .cache/
 .codeql/
+
+# E2E Reports
+**/playwright-report/**
+**/test-results/**

CONTRIBUTING.md

@@ -28,6 +28,7 @@ Here are a few things you can do that will increase the likelihood of your pull
   - [Integration tests that do require the VS Code API are located here](extensions/ql-vscode/src/vscode-tests).
 - Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
 - Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+- Update the [changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) if you are making user-facing changes.
 
 ## Setting up a local build
 

README.md

@@ -21,13 +21,6 @@ To see what has changed in the last few versions of the extension, see the [Chan
 
 This project will track new feature development in CodeQL and, whenever appropriate, bring that functionality to the Visual Studio Code experience.
 
-## Dependencies
-
-This extension depends on the following two extensions for required functionality. They will be installed automatically when you install VS Code CodeQL.
-
-- [Test Adapter Converter](https://marketplace.visualstudio.com/items?itemName=ms-vscode.test-adapter-converter)
-- [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer)
-
 ## Contributing
 
 This project welcomes contributions. See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to build, install, and contribute.

docs/node-version.md

@@ -1,5 +1,8 @@
 # Node version
 
+> [!NOTE]
+> It should not be necessary to update the Node.js version manually. The [update-node-version.yml](https://github.com/github/vscode-codeql/blob/main/.github/workflows/update-node-version.yml) workflow runs daily and will open a pull request if the Node.js version needs to be updated.
+
 The CodeQL for VS Code extension defines the version of Node.js that it is intended to run with. This Node.js version is used when running most CI and unit tests.
 
 When running in production (i.e. as an extension for a VS Code application) it will use the Node.js version provided by VS Code. This can mean a different Node.js version is used by different users with different versions of VS Code.

docs/releasing.md

@@ -1,5 +1,6 @@
 # Releasing (write access required)
 
+1. Make sure the needed authentication keys are valid. Most likely the Azure DevOps PAT needs to be regenerated. See below.
 1. Determine the new version number. We default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
     - Making substantial new features available to all users. This can include lifting a feature flag.
     - Breakage in compatibility with recent versions of the CLI.
@@ -60,7 +61,7 @@
 
 ## Secrets and authentication for publishing
 
-Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token.
 
 To regenerate the Open VSX token:
 
@@ -69,4 +70,4 @@ To regenerate the Open VSX token:
 1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
 1. Update the secret in the `publish-open-vsx` environment in the project settings.
 
-To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.
+To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 7 days and must be regenerated.

docs/test-plan.md

@@ -53,18 +53,20 @@ choose to go through some of the Optional Test Cases.
 
 #### Test case 3: Running a non-problem query and viewing results
 
-1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+1. Open the [cpp HubClasses query](https://github.com/github/codeql/blob/main/cpp/ql/src/Architecture/General%20Class-Level%20Information/HubClasses.ql).
 2. Select the `google/brotli` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
+   - Check that the `#select` result set is shown
    - Check that the results table is rendered
    - Check that result locations can be clicked on
 
 #### Test case 4: Can use AST viewer
 
 1. Click on any code location from a previous query to open a source file from a database
-2. Open the AST viewing panel and click "View AST"
-3. Once the AST is computed:
+2. Select the highlighted code in the source file
+3. Open the AST viewing panel and click "View AST"
+4. Once the AST is computed:
    - Check that it can be navigated
 
 ### MRVA
@@ -143,6 +145,48 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
+#### Test Case 6: Using model packs in MRVA
+
+1. Create a model pack with mock data
+   1. Create a new directory `test-model-pack`
+   2. Create a `qlpack.yml` file in that directory with the following contents:
+
+      ```yaml
+      name: github/test-model-pack
+      version: 0.0.0
+      library: true
+      extensionTargets:
+        codeql/python-all: '*'
+      dataExtensions:
+        - extension.yml
+      ```
+
+   3. Create an `extension.yml` in the same directory with the following contents:
+
+      ```yaml
+      extensions:
+      - addsTo:
+          pack: codeql/python-all
+          extensible: sinkModel
+        data:
+         - ["vscode-codeql","Member[initialize].Argument[0]","code-injection"]
+      ```
+
+2. In a Python query pack, create the following query (e.g. `sinks.ql`):
+
+   ```ql
+   import python
+   import semmle.python.frameworks.data.internal.ApiGraphModelsExtensions
+
+   from string path, string kind
+   where sinkModel("vscode-codeql", path, kind, _)
+   select path, kind
+   ```
+
+3. Run a MRVA against a Python repository (e.g. `psf/requests`) with this query.
+4. Check that the results view contains 1 result with the values corresponding to the `extension.yml` file:
+   ![Model packs results table for `psf/requests`](images/model-pack-results-table.png)
+
 ### CodeQL Model Editor
 
 #### Test Case 1: Opening the model editor
@@ -151,7 +195,7 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
    - Check that the editor loads and shows methods to model.
    - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
-   - Check that the "Open source" link works.
+   - Check that the "Open source" link works (if you have the database source).
    - Check that the 'View' button works and the Method Usage panel highlight the correct method and usage
    - Check that the Method Modeling panel shows the correct method and modeling state
 
@@ -166,17 +210,7 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 4. Open the ".model.yml" file corresponding to the library that was changed.
    - Check that the file contains entries for the methods that were modeled.
 
-#### Test Case 3: Model with AI
-
-Note that this test requires the feature flag: `codeQL.model.llmGeneration`
-
-A package that the AI normally gives models for is `javax.servlet-api` from the `jhy/jsoup` repository.
-
-1. Click "Model with AI".
-   - Check that rows change to "Thinking".
-   - Check that results come back and rows get filled out.
-
-#### Test Case 4: Model as dependency
+#### Test Case 3: Model as dependency
 
 Note that this test requires the feature flag: `codeQL.model.flowGeneration`
 

extensions/ql-vscode/.babelrc.json

@@ -1,16 +0,0 @@
-{
-  "sourceType": "unambiguous",
-  "presets": [
-    [
-      "@babel/preset-env",
-      {
-        "targets": {
-          "chrome": 100
-        }
-      }
-    ],
-    "@babel/preset-typescript",
-    "@babel/preset-react"
-  ],
-  "plugins": []
-}

extensions/ql-vscode/.eslintrc.js

@@ -28,6 +28,7 @@ const baseConfig = {
     "plugin:@typescript-eslint/recommended",
     "plugin:import/recommended",
     "plugin:import/typescript",
+    "plugin:deprecation/recommended",
   ],
   rules: {
     "@typescript-eslint/await-thenable": "error",
@@ -39,7 +40,7 @@ const baseConfig = {
         ignoreRestSiblings: false,
       },
     ],
-    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-explicit-any": "error",
     "@typescript-eslint/no-floating-promises": ["error", { ignoreVoid: true }],
     "@typescript-eslint/no-invalid-this": "off",
     "@typescript-eslint/no-shadow": "off",
@@ -120,15 +121,6 @@ module.exports = {
         },
       },
     },
-    {
-      files: ["test/**/*"],
-      parserOptions: {
-        project: resolve(__dirname, "test/tsconfig.json"),
-      },
-      env: {
-        jest: true,
-      },
-    },
     {
       files: ["test/vscode-tests/**/*"],
       parserOptions: {
@@ -155,6 +147,18 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ["test/**/*"],
+      parserOptions: {
+        project: resolve(__dirname, "test/tsconfig.json"),
+      },
+      env: {
+        jest: true,
+      },
+      rules: {
+        "@typescript-eslint/no-explicit-any": "off",
+      },
+    },
     {
       files: [
         ".eslintrc.js",

extensions/ql-vscode/.nvmrc

@@ -1 +1 @@
-v18.15.0
+v20.14.0

extensions/ql-vscode/.storybook/main.ts

@@ -1,7 +1,7 @@
-import type { StorybookConfig } from "@storybook/react-webpack5";
+import type { StorybookConfig } from "@storybook/react-vite";
 
 const config: StorybookConfig = {
-  stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
+  stories: ["../src/**/*.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
   addons: [
     "@storybook/addon-links",
     "@storybook/addon-essentials",
@@ -10,7 +10,7 @@ const config: StorybookConfig = {
     "./vscode-theme-addon/preset.ts",
   ],
   framework: {
-    name: "@storybook/react-webpack5",
+    name: "@storybook/react-vite",
     options: {},
   },
   docs: {

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -5,10 +5,10 @@ import { useCallback } from "react";
 import { useGlobals } from "@storybook/manager-api";
 import {
   IconButton,
-  Icons,
   TooltipLinkList,
   WithTooltip,
 } from "@storybook/components";
+import { DashboardIcon } from "@storybook/icons";
 
 import { themeNames, VSCodeTheme } from "./theme";
 
@@ -53,7 +53,7 @@ export const ThemeSelector: FunctionComponent = () => {
         title="Change the theme of the preview"
         active={vscodeTheme !== VSCodeTheme.Dark}
       >
-        <Icons icon="dashboard" />
+        <DashboardIcon />
       </IconButton>
     </WithTooltip>
   );

extensions/ql-vscode/.storybook/vscode-theme-addon/manager.tsx

@@ -1,5 +1,6 @@
 import * as React from "react";
-import { addons, types } from "@storybook/manager-api";
+import { addons } from "@storybook/manager-api";
+import { Addon_TypesEnum } from "storybook/internal/types";
 import { ThemeSelector } from "./ThemeSelector";
 
 const ADDON_ID = "vscode-theme-addon";
@@ -7,7 +8,7 @@ const ADDON_ID = "vscode-theme-addon";
 addons.register(ADDON_ID, () => {
   addons.add(ADDON_ID, {
     title: "VSCode Themes",
-    type: types.TOOL,
+    type: Addon_TypesEnum.TOOL,
     match: ({ viewMode }) => !!(viewMode && viewMode.match(/^(story|docs)$/)),
     render: () => <ThemeSelector />,
   });

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -1,3 +1,5 @@
+/// <reference types="vite/client" />
+
 import { useEffect } from "react";
 import type {
   PartialStoryFn as StoryFunction,
@@ -6,31 +8,20 @@ import type {
 
 import { VSCodeTheme } from "./theme";
 
+import darkThemeStyle from "../../src/stories/vscode-theme-dark.css?url";
+import lightThemeStyle from "../../src/stories/vscode-theme-light.css?url";
+import lightHighContrastThemeStyle from "../../src/stories/vscode-theme-light-high-contrast.css?url";
+import darkHighContrastThemeStyle from "../../src/stories/vscode-theme-dark-high-contrast.css?url";
+import githubLightDefaultThemeStyle from "../../src/stories/vscode-theme-github-light-default.css?url";
+import githubDarkDefaultThemeStyle from "../../src/stories/vscode-theme-github-dark-default.css?url";
+
 const themeFiles: { [key in VSCodeTheme]: string } = {
-  [VSCodeTheme.Dark]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-dark.css")
-      .default,
-  [VSCodeTheme.Light]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-light.css")
-      .default,
-  [VSCodeTheme.LightHighContrast]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-light-high-contrast.css")
-      .default,
-  [VSCodeTheme.DarkHighContrast]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-dark-high-contrast.css")
-      .default,
-  [VSCodeTheme.GitHubLightDefault]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-github-light-default.css")
-      .default,
-  [VSCodeTheme.GitHubDarkDefault]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-github-dark-default.css")
-      .default,
+  [VSCodeTheme.Dark]: darkThemeStyle,
+  [VSCodeTheme.Light]: lightThemeStyle,
+  [VSCodeTheme.LightHighContrast]: lightHighContrastThemeStyle,
+  [VSCodeTheme.DarkHighContrast]: darkHighContrastThemeStyle,
+  [VSCodeTheme.GitHubLightDefault]: githubLightDefaultThemeStyle,
+  [VSCodeTheme.GitHubDarkDefault]: githubDarkDefaultThemeStyle,
 };
 
 export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,53 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.14.0 - 7 August 2024
+
+- Add Python support to the CodeQL Model Editor. [#3676](https://github.com/github/vscode-codeql/pull/3676)
+- Update variant analysis view to display the length of the shortest path for path queries. [#3671](https://github.com/github/vscode-codeql/pull/3671)
+- Remove support for CodeQL CLI versions older than 2.15.5. [#3681](https://github.com/github/vscode-codeql/pull/3681)
+
+## 1.13.1 - 29 May 2024
+
+- Fix a bug when re-importing test databases that erroneously showed old source code. [#3616](https://github.com/github/vscode-codeql/pull/3616)
+- Update the progress window with details on potentially long-running post-processing steps after running a query. [#3622](https://github.com/github/vscode-codeql/pull/3622)
+
+## 1.13.0 - 1 May 2024
+
+- Add Ruby support to the CodeQL Model Editor. [#3584](https://github.com/github/vscode-codeql/pull/3584)
+- Remove support for CodeQL CLI versions older than 2.14.6. [#3562](https://github.com/github/vscode-codeql/pull/3562)
+
+## 1.12.5 - 9 April 2024
+
+- Add new supported source and sink kinds in the CodeQL Model Editor [#3511](https://github.com/github/vscode-codeql/pull/3511)
+- Fix a bug where the test explorer wouldn't display certain tests. [#3527](https://github.com/github/vscode-codeql/pull/3527)
+- The "model dependency" operation in the model editor can now be cancelled. [#3517](https://github.com/github/vscode-codeql/pull/3517)
+
+## 1.12.4 - 20 March 2024
+
+- Don't show notification after local query cancellation. [#3489](https://github.com/github/vscode-codeql/pull/3489)
+- Databases created from [CodeQL test cases](https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/testing-custom-queries) are now copied into a shared VS Code storage location. This avoids a bug where re-running test cases would fail if the test's database is already imported into the workspace. [#3433](https://github.com/github/vscode-codeql/pull/3433)
+
+## 1.12.3 - 29 February 2024
+
+- Update variant analysis view to show when cancelation is in progress. [#3405](https://github.com/github/vscode-codeql/pull/3405)
+- Remove support for CodeQL CLI versions older than 2.13.5. [#3371](https://github.com/github/vscode-codeql/pull/3371)
+- Add a timeout to downloading databases and the CodeQL CLI. These can be changed using the `codeQL.addingDatabases.downloadTimeout` and `codeQL.cli.downloadTimeout` settings respectively. [#3373](https://github.com/github/vscode-codeql/pull/3373)
+- When downloading a CodeQL database through the model editor, only use credentials when in canary mode. [#3440](https://github.com/github/vscode-codeql/pull/3440)
+
+## 1.12.2 - 14 February 2024
+
+- Stop allowing running variant analyses with a query outside of the workspace. [#3302](https://github.com/github/vscode-codeql/pull/3302)
+
+## 1.12.1 - 31 January 2024
+
+- Enable collection of telemetry for the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting. [#3238](https://github.com/github/vscode-codeql/pull/3238)
+- In the CodeQL model editor, you can now select individual method rows and save changes to only the selected rows, instead of having to save the entire library model. [#3156](https://github.com/github/vscode-codeql/pull/3156)
+- If you run a query without having selected a database, we show a more intuitive prompt to help you select a database. [#3214](https://github.com/github/vscode-codeql/pull/3214)
+- Error messages returned from the CodeQL CLI are now less verbose and more user-friendly. [#3259](https://github.com/github/vscode-codeql/pull/3259)
+- The UI for browsing and running CodeQL tests has moved to use VS Code's built-in test UI. This makes the CodeQL test UI more consistent with the test UIs for other languages.
+  This change means that this extension no longer depends on the "Test Explorer UI" and "Test Adapter Converter" extensions. You can uninstall those two extensions if they are
+  not being used by any other extensions you may have installed. [#3232](https://github.com/github/vscode-codeql/pull/3232)
+
 ## 1.12.0 - 11 January 2024
 
 - Add a prompt for downloading a GitHub database when opening a GitHub repository. [#3138](https://github.com/github/vscode-codeql/pull/3138)
@@ -460,7 +508,7 @@ No user facing changes.
 - Allow setting `codeQL.runningQueries.numberOfThreads` and `codeQL.runningTests.numberOfThreads` to 0, (which is interpreted as 'use one thread per core on the machine').
 - Clear the problems view of all CodeQL query results when a database is removed.
 - Add a `View DIL` command on query history items. This opens a text editor containing the Datalog Intermediary Language representation of the compiled query.
-- Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/exploring-the-structure-of-your-source-code.html).
+- Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/exploring-the-structure-of-your-source-code/).
 - The `codeQL.runningTests.numberOfThreads` setting is now used correctly when running tests.
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the qlpacks.
 - Ensure output of CodeQL test runs includes compilation error messages and test failure messages.

extensions/ql-vscode/README.md

@@ -9,16 +9,16 @@ This project is an extension for Visual Studio Code that adds rich language supp
 
 To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
+You can also read full documentation for the extension on the [GitHub documentation website](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension).
+
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
-For information about other configurations, see the separate [CodeQL help](https://codeql.github.com/docs/codeql-for-visual-studio-code/).
+For information about other configurations, see "[Using the advanced functionality of the CodeQL for VS Code extension](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension)" in the GitHub documentation.
 
 ### Quick start: Installing and configuring the extension
 
 1. [Install the extension](#installing-the-extension).  
-   *Note: vscode-codeql installs the following dependencies for required functionality: [Test Adapter Converter](https://marketplace.visualstudio.com/items?itemName=ms-vscode.test-adapter-converter), [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer).*
-
 1. [Check access to the CodeQL CLI](#checking-access-to-the-codeql-cli).
 1. [Clone the CodeQL starter workspace](#cloning-the-codeql-starter-workspace).
 
@@ -44,7 +44,7 @@ The CodeQL extension requires a minimum of Visual Studio Code 1.39. Older versio
 
 The extension uses the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
 
-If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#configuring-access-to-the-codeql-cli).
+If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see "[Configuring access to the CodeQL CLI](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension/configuring-access-to-the-codeql-cli)" in the GitHub documentation.
 
 If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Log** in the **Output** view for any error messages.
 
@@ -54,7 +54,7 @@ When you're working with CodeQL, you need access to the standard CodeQL librarie
 Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
-For information about configuring an existing workspace for CodeQL, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#updating-an-existing-workspace-for-codeql).
+For information about configuring an existing workspace for CodeQL, see "[Setting up a CodeQL workspace](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension/setting-up-a-codeql-workspace#option-2-updating-an-existing-workspace-for-codeql-advanced)" in the GitHub documentation.
 
 ## Upgrading CodeQL standard libraries
 
@@ -87,7 +87,7 @@ While you can use the [CodeQL CLI to create your own databases](https://codeql.g
 1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
 1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
-For more information, see [Choosing a database](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#choosing-a-database) on codeql.github.com.
+For more information, see "[Managing CodeQL databases](https://docs.github.com/code-security/codeql-for-vs-code/getting-started-with-codeql-for-vs-code/managing-codeql-databases#choosing-a-database-to-analyze)" in the GitHub documentation.
 
 ### Running a query
 
@@ -108,12 +108,12 @@ If you wish to navigate the query results from your keyboard, you can bind short
 
 ## What next?
 
-For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:
+We recommend reading the [full documentation for the extension](https://docs.github.com/code-security/codeql-for-vs-code/) on the GitHub documentation website. You may also find the following resources useful:
 
 - [Create a database for a different codebase](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/).
-- [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
+- [Try out variant analysis](https://docs.github.com/code-security/codeql-for-vs-code/getting-started-with-codeql-for-vs-code/running-codeql-queries-at-scale-with-multi-repository-variant-analysis).
 - [Learn more about CodeQL](https://codeql.github.com/docs/).
-- [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
+- [Read how security researchers use CodeQL to find CVEs](https://github.blog/tag/github-security-lab/).
 
 ## License
 
@@ -121,4 +121,4 @@ The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the
 
 ## Data and Telemetry
 
-If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information about telemetry, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code).
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information, see "[Telemetry in CodeQL for Visual Studio Code](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension/telemetry-in-codeql-for-visual-studio-code)" in the GitHub documentation.

extensions/ql-vscode/gulpfile.ts/textmate-grammar.ts

@@ -0,0 +1,91 @@
+/**
+ * A subset of the standard TextMate grammar that is used by our transformation
+ * step. For a full JSON schema, see:
+ * https://github.com/martinring/tmlanguage/blob/478ad124a21933cd4b0b65f1ee7ee18ee1f87473/tmlanguage.json
+ */
+export interface TextmateGrammar {
+  patterns: Pattern[];
+  repository?: Record<string, Pattern>;
+}
+
+/**
+ * The extended TextMate grammar as used by our transformation step. This is a superset of the
+ * standard TextMate grammar, and includes additional fields that are used by our transformation
+ * step.
+ *
+ * Any comment of the form `(?#ref-id)` in a `match`, `begin`, or `end` property will be replaced
+ * with the match text of the rule named "ref-id". If the rule named "ref-id" consists of just a
+ * `patterns` property with a list of `include` directives, the replacement pattern is the
+ * disjunction of the match patterns of all of the included rules.
+ */
+export interface ExtendedTextmateGrammar<MatchType = string> {
+  /**
+   * This represents the set of regular expression options to apply to all regular
+   * expressions throughout the file.
+   */
+  regexOptions?: string;
+  /**
+   * This element defines a map of macro names to replacement text. When a `match`, `begin`, or
+   * `end` property has a value that is a single-key map, the value is replaced with the value of the
+   * macro named by the key, with any use of `(?#)` in the macro text replaced with the text of the
+   * value of the key, surrounded by a non-capturing group (`(?:)`). For example:
+   *
+   * The `beginPattern` and `endPattern` Properties
+   * A rule can have a `beginPattern` or `endPattern` property whose value is a reference to another
+   * rule (e.g. `#other-rule`). The `beginPattern` property is replaced as follows:
+   *
+   * my-rule:
+   *   beginPattern: '#other-rule'
+   *
+   * would be transformed to
+   *
+   * my-rule:
+   *   begin: '(?#other-rule)'
+   *   beginCaptures:
+   *     '0':
+   *       patterns:
+   *       - include: '#other-rule'
+   *
+   * An `endPattern` property is transformed similary.
+   *
+   * macros:
+   *   repeat: '(?#)*'
+   * repository:
+   *   multi-letter:
+   *     match:
+   *       repeat: '[A-Za-z]'
+   *     name: scope.multi-letter
+   *
+   * would be transformed to
+   *
+   * repository:
+   *   multi-letter:
+   *     match: '(?:[A-Za-z])*'
+   *     name: scope.multi-letter
+   */
+  macros?: Record<string, string>;
+
+  patterns: Array<Pattern<MatchType>>;
+  repository?: Record<string, Pattern<MatchType>>;
+}
+
+export interface Pattern<MatchType = string> {
+  include?: string;
+  match?: MatchType;
+  begin?: MatchType;
+  end?: MatchType;
+  while?: MatchType;
+  captures?: Record<string, PatternCapture>;
+  beginCaptures?: Record<string, PatternCapture>;
+  endCaptures?: Record<string, PatternCapture>;
+  patterns?: Array<Pattern<MatchType>>;
+  beginPattern?: string;
+  endPattern?: string;
+}
+
+export interface PatternCapture {
+  name?: string;
+  patterns?: Pattern[];
+}
+
+export type ExtendedMatchType = string | Record<string, string>;

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -3,6 +3,12 @@ import { load } from "js-yaml";
 import { obj } from "through2";
 import PluginError from "plugin-error";
 import type Vinyl from "vinyl";
+import type {
+  ExtendedMatchType,
+  ExtendedTextmateGrammar,
+  Pattern,
+  TextmateGrammar,
+} from "./textmate-grammar";
 
 /**
  * Replaces all rule references with the match pattern of the referenced rule.
@@ -34,7 +40,9 @@ function replaceReferencesWithStrings(
  * @param yaml The root of the YAML document.
  * @returns A map from macro name to replacement text.
  */
-function gatherMacros(yaml: any): Map<string, string> {
+function gatherMacros<T>(
+  yaml: ExtendedTextmateGrammar<T>,
+): Map<string, string> {
   const macros = new Map<string, string>();
   for (const key in yaml.macros) {
     macros.set(key, yaml.macros[key]);
@@ -51,7 +59,7 @@ function gatherMacros(yaml: any): Map<string, string> {
  * @returns The match text for the rule. This is either the value of the rule's `match` property,
  *  or the disjunction of the match text of all of the other rules `include`d by this rule.
  */
-function getNodeMatchText(rule: any): string {
+function getNodeMatchText(rule: Pattern): string {
   if (rule.match !== undefined) {
     // For a match string, just use that string as the replacement.
     return rule.match;
@@ -78,7 +86,7 @@ function getNodeMatchText(rule: any): string {
  * @returns A map whose keys are the names of rules, and whose values are the corresponding match
  *  text of each rule.
  */
-function gatherMatchTextForRules(yaml: any): Map<string, string> {
+function gatherMatchTextForRules(yaml: TextmateGrammar): Map<string, string> {
   const replacements = new Map<string, string>();
   for (const key in yaml.repository) {
     const node = yaml.repository[key];
@@ -94,9 +102,14 @@ function gatherMatchTextForRules(yaml: any): Map<string, string> {
  * @param yaml The root of the YAML document.
  * @param action Callback to invoke on each rule.
  */
-function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
+function visitAllRulesInFile<T>(
+  yaml: ExtendedTextmateGrammar<T>,
+  action: (rule: Pattern<T>) => void,
+) {
   visitAllRulesInRuleMap(yaml.patterns, action);
-  visitAllRulesInRuleMap(yaml.repository, action);
+  if (yaml.repository) {
+    visitAllRulesInRuleMap(Object.values(yaml.repository), action);
+  }
 }
 
 /**
@@ -107,9 +120,11 @@ function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
  * @param ruleMap The map or array of rules to visit.
  * @param action Callback to invoke on each rule.
  */
-function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
-  for (const key in ruleMap) {
-    const rule = ruleMap[key];
+function visitAllRulesInRuleMap<T>(
+  ruleMap: Array<Pattern<T>>,
+  action: (rule: Pattern<T>) => void,
+) {
+  for (const rule of ruleMap) {
     if (typeof rule === "object") {
       action(rule);
       if (rule.patterns !== undefined) {
@@ -125,16 +140,22 @@ function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
  * @param rule The rule whose matches are to be transformed.
  * @param action The transformation to make on each match pattern.
  */
-function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
+function visitAllMatchesInRule<T>(rule: Pattern<T>, action: (match: T) => T) {
   for (const key in rule) {
     switch (key) {
       case "begin":
       case "end":
       case "match":
-      case "while":
-        rule[key] = action(rule[key]);
-        break;
+      case "while": {
+        const ruleElement = rule[key];
 
+        if (!ruleElement) {
+          continue;
+        }
+
+        rule[key] = action(ruleElement);
+        break;
+      }
       default:
         break;
     }
@@ -148,14 +169,17 @@ function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
  * @param rule Rule to be transformed.
  * @param key Base key of the property to be transformed.
  */
-function expandPatternMatchProperties(rule: any, key: "begin" | "end") {
-  const patternKey = `${key}Pattern`;
-  const capturesKey = `${key}Captures`;
+function expandPatternMatchProperties<T>(
+  rule: Pattern<T>,
+  key: "begin" | "end",
+) {
+  const patternKey = `${key}Pattern` as const;
+  const capturesKey = `${key}Captures` as const;
   const pattern = rule[patternKey];
   if (pattern !== undefined) {
     const patterns: string[] = Array.isArray(pattern) ? pattern : [pattern];
-    rule[key] = patterns.map((p) => `((?${p}))`).join("|");
-    const captures: { [index: string]: any } = {};
+    rule[key] = patterns.map((p) => `((?${p}))`).join("|") as T;
+    const captures: Pattern["captures"] = {};
     for (const patternIndex in patterns) {
       captures[(Number(patternIndex) + 1).toString()] = {
         patterns: [
@@ -175,7 +199,7 @@ function expandPatternMatchProperties(rule: any, key: "begin" | "end") {
  *
  * @param yaml The root of the YAML document.
  */
-function transformFile(yaml: any) {
+function transformFile(yaml: ExtendedTextmateGrammar<ExtendedMatchType>) {
   const macros = gatherMacros(yaml);
   visitAllRulesInFile(yaml, (rule) => {
     expandPatternMatchProperties(rule, "begin");
@@ -198,24 +222,29 @@ function transformFile(yaml: any) {
 
   yaml.macros = undefined;
 
-  const replacements = gatherMatchTextForRules(yaml);
+  // We have removed all object match properties, so we don't have an extended match type anymore.
+  const macrolessYaml = yaml as ExtendedTextmateGrammar;
+
+  const replacements = gatherMatchTextForRules(macrolessYaml);
   // Expand references in matches.
-  visitAllRulesInFile(yaml, (rule) => {
+  visitAllRulesInFile(macrolessYaml, (rule) => {
     visitAllMatchesInRule(rule, (match) => {
       return replaceReferencesWithStrings(match, replacements);
     });
   });
 
-  if (yaml.regexOptions !== undefined) {
-    const regexOptions = `(?${yaml.regexOptions})`;
-    visitAllRulesInFile(yaml, (rule) => {
+  if (macrolessYaml.regexOptions !== undefined) {
+    const regexOptions = `(?${macrolessYaml.regexOptions})`;
+    visitAllRulesInFile(macrolessYaml, (rule) => {
       visitAllMatchesInRule(rule, (match) => {
         return regexOptions + match;
       });
     });
 
-    yaml.regexOptions = undefined;
+    macrolessYaml.regexOptions = undefined;
   }
+
+  return macrolessYaml;
 }
 
 export function transpileTextMateGrammar() {
@@ -230,8 +259,8 @@ export function transpileTextMateGrammar() {
       } else if (file.isBuffer()) {
         const buf: Buffer = file.contents;
         const yamlText: string = buf.toString("utf8");
-        const jsonData: any = load(yamlText);
-        transformFile(jsonData);
+        const yamlData = load(yamlText) as TextmateGrammar;
+        const jsonData = transformFile(yamlData);
 
         file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), "utf8");
         file.extname = ".json";

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -77,5 +77,8 @@ export function copyWasmFiles() {
   // to configure the path to the WASM file. So, source-map will always load the file from `__dirname/mappings.wasm`.
   // In version 0.8.0, it may be possible to do this properly by calling SourceMapConsumer.initialize by
   // using the "browser" field in source-map's package.json to load the WASM file from a given file path.
-  return src("node_modules/source-map/lib/mappings.wasm").pipe(dest("out"));
+  return src("node_modules/source-map/lib/mappings.wasm", {
+    // WASM is a binary format, so don't try to re-encode it as text.
+    encoding: false,
+  }).pipe(dest("out"));
 }

extensions/ql-vscode/gulpfile.ts/view.ts

@@ -28,7 +28,7 @@ export function compileViewEsbuild() {
 }
 
 export function watchViewEsbuild() {
-  watch(["src/view/**/*.{ts,tsx}"], compileViewEsbuild);
+  watch(["src/**/*.{ts,tsx}"], compileViewEsbuild);
 }
 
 export function checkViewTypeScript() {
@@ -38,5 +38,5 @@ export function checkViewTypeScript() {
 }
 
 export function watchViewCheckTypeScript() {
-  watch(["src/view/**/*.{ts,tsx}"], checkViewTypeScript);
+  watch(["src/**/*.{ts,tsx}"], checkViewTypeScript);
 }

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.12.0",
+  "version": "1.14.0",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -14,14 +14,13 @@
   },
   "engines": {
     "vscode": "^1.82.0",
-    "node": "^18.15.0",
+    "node": "^20.14.0",
     "npm": ">=7.20.6"
   },
   "categories": [
     "Programming Languages"
   ],
   "extensionDependencies": [
-    "hbenl.vscode-test-explorer",
     "vscode.git"
   ],
   "capabilities": {
@@ -54,10 +53,10 @@
     "configurationDefaults": {
       "[ql]": {
         "debug.saveBeforeStart": "nonUntitledEditorsInActiveGroup",
-        "editor.wordBasedSuggestions": false
+        "editor.wordBasedSuggestions": "off"
       },
       "[dbscheme]": {
-        "editor.wordBasedSuggestions": false
+        "editor.wordBasedSuggestions": "off"
       }
     },
     "debuggers": [
@@ -179,6 +178,11 @@
             "type": "string",
             "default": "",
             "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
+          },
+          "codeQL.cli.downloadTimeout": {
+            "type": "integer",
+            "default": 10,
+            "description": "Download timeout in seconds for downloading the CLI distribution."
           }
         }
       },
@@ -377,6 +381,11 @@
         "title": "Adding databases",
         "order": 6,
         "properties": {
+          "codeQL.addingDatabases.downloadTimeout": {
+            "type": "integer",
+            "default": 10,
+            "description": "Download timeout in seconds for adding a CodeQL database."
+          },
           "codeQL.addingDatabases.allowHttp": {
             "type": "boolean",
             "default": false,
@@ -454,8 +463,25 @@
       },
       {
         "type": "object",
-        "title": "Log insights",
+        "title": "Model Editor",
         "order": 9,
+        "properties": {
+          "codeQL.model.packLocation": {
+            "type": "string",
+            "default": ".github/codeql/extensions/${name}-${language}",
+            "markdownDescription": "Location for newly created CodeQL model packs. The location can be either absolute or relative. If relative, it is relative to the workspace root.\n\nThe following variables are supported:\n* **${database}** - the name of the database\n* **${language}** - the name of the language\n* **${name}** - the name of the GitHub repository, or the name of the database if the database was not downloaded from GitHub\n* **${owner}** - the owner of the GitHub repository, or an empty string if the database was not downloaded from GitHub"
+          },
+          "codeQL.model.packName": {
+            "type": "string",
+            "default": "${owner}/${name}-${language}",
+            "markdownDescription": "Name of newly created CodeQL model packs. If the result is not a valid pack name, it will be converted to a valid pack name.\n\nThe following variables are supported:\n* **${database}** - the name of the database\n* **${language}** - the name of the language\n* **${name}** - the name of the GitHub repository, or the name of the database if the database was not downloaded from GitHub\n* **${owner}** - the owner of the GitHub repository, or an empty string if the database was not downloaded from GitHub"
+          }
+        }
+      },
+      {
+        "type": "object",
+        "title": "Log insights",
+        "order": 10,
         "properties": {
           "codeQL.logInsights.joinOrderWarningThreshold": {
             "type": "number",
@@ -469,7 +495,7 @@
       {
         "type": "object",
         "title": "Telemetry",
-        "order": 10,
+        "order": 11,
         "properties": {
           "codeQL.telemetry.enableTelemetry": {
             "type": "boolean",
@@ -546,6 +572,14 @@
         "command": "codeQL.runVariantAnalysisContextEditor",
         "title": "CodeQL: Run Variant Analysis"
       },
+      {
+        "command": "codeQL.runVariantAnalysisContextExplorer",
+        "title": "CodeQL: Run Variant Analysis"
+      },
+      {
+        "command": "codeQL.runVariantAnalysisPublishedPack",
+        "title": "CodeQL: Run Variant Analysis against published pack"
+      },
       {
         "command": "codeQL.exportSelectedVariantAnalysisResults",
         "title": "CodeQL: Export Variant Analysis Results"
@@ -721,6 +755,10 @@
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
       },
+      {
+        "command": "codeQL.importTestDatabase",
+        "title": "CodeQL: (Re-)Import Test Database"
+      },
       {
         "command": "codeQL.getCurrentDatabase",
         "title": "CodeQL: Get Current Database"
@@ -1305,7 +1343,12 @@
         {
           "command": "codeQL.setCurrentDatabase",
           "group": "9_qlCommands",
-          "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
+          "when": "resourceExtname != .testproj && (resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zipz)"
+        },
+        {
+          "command": "codeQL.importTestDatabase",
+          "group": "9_qlCommands",
+          "when": "explorerResourceIsFolder && resourceExtname == .testproj"
         },
         {
           "command": "codeQL.viewAstContextExplorer",
@@ -1322,6 +1365,11 @@
           "group": "9_qlCommands",
           "when": "resourceScheme != codeql-zip-archive"
         },
+        {
+          "command": "codeQL.runVariantAnalysisContextExplorer",
+          "group": "9_qlCommands",
+          "when": "resourceExtname == .ql && config.codeQL.canary && config.codeQL.variantAnalysis.multiQuery"
+        },
         {
           "command": "codeQL.openReferencedFileContextExplorer",
           "group": "9_qlCommands",
@@ -1398,6 +1446,14 @@
           "command": "codeQL.runVariantAnalysis",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runVariantAnalysisContextExplorer",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.runVariantAnalysisPublishedPack",
+          "when": "config.codeQL.canary && config.codeQL.variantAnalysis.multiQuery"
+        },
         {
           "command": "codeQL.runVariantAnalysisContextEditor",
           "when": "false"
@@ -1412,7 +1468,7 @@
         },
         {
           "command": "codeQL.quickEvalCount",
-          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+          "when": "editorLangId == ql"
         },
         {
           "command": "codeQL.quickEvalContextEditor",
@@ -1446,6 +1502,10 @@
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQL.importTestDatabase",
+          "when": "false"
+        },
         {
           "command": "codeQL.getCurrentDatabase",
           "when": "false"
@@ -1730,8 +1790,7 @@
           "when": "false"
         },
         {
-          "command": "codeQL.trimCache",
-          "when": "codeql.supportsTrimCache"
+          "command": "codeQL.trimCache"
         }
       ],
       "editor/context": [
@@ -1895,77 +1954,79 @@
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",
     "lint:markdown": "markdownlint-cli2 \"../../**/*.{md,mdx}\" \"!**/node_modules/**\" \"!**/.vscode-test/**\" \"!**/build/cli/v*/**\"",
-    "find-deadcode": "ts-node scripts/find-deadcode.ts",
+    "find-deadcode": "vite-node scripts/find-deadcode.ts",
     "format-staged": "lint-staged",
     "storybook": "storybook dev -p 6006",
     "build-storybook": "storybook build",
-    "lint:scenarios": "ts-node scripts/lint-scenarios.ts",
+    "lint:scenarios": "vite-node scripts/lint-scenarios.ts",
     "generate": "npm-run-all -p generate:*",
-    "generate:schemas": "ts-node scripts/generate-schemas.ts",
-    "generate:chromium-version": "ts-node scripts/generate-chromium-version.ts",
+    "generate:schemas": "vite-node scripts/generate-schemas.ts",
+    "generate:chromium-version": "vite-node scripts/generate-chromium-version.ts",
     "check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
     "postinstall": "patch-package",
     "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
+    "@floating-ui/react": "^0.26.12",
     "@octokit/plugin-retry": "^6.0.1",
+    "@octokit/plugin-throttling": "^8.0.0",
     "@octokit/rest": "^20.0.2",
-    "@vscode/codicons": "^0.0.35",
+    "@vscode/codicons": "^0.0.36",
     "@vscode/debugadapter": "^1.59.0",
-    "@vscode/debugprotocol": "^1.59.0",
+    "@vscode/debugprotocol": "^1.65.0",
     "@vscode/webview-ui-toolkit": "^1.0.1",
     "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
-    "chokidar": "^3.5.3",
-    "d3": "^7.6.1",
+    "chokidar": "^3.6.0",
+    "d3": "^7.9.0",
     "d3-graphviz": "^5.0.2",
     "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",
-    "msw": "^2.0.11",
-    "nanoid": "^5.0.1",
+    "msw": "^2.2.13",
+    "nanoid": "^5.0.7",
     "node-fetch": "^2.6.7",
     "p-queue": "^8.0.1",
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "semver": "^7.5.2",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "semver": "^7.6.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
     "stream-json": "^1.7.3",
-    "styled-components": "^6.0.2",
+    "styled-components": "^6.1.9",
     "tmp": "^0.2.1",
     "tmp-promise": "^3.0.2",
     "tree-kill": "^1.2.2",
     "vscode-extension-telemetry": "^0.1.6",
-    "vscode-jsonrpc": "^8.0.2",
+    "vscode-jsonrpc": "^8.2.1",
     "vscode-languageclient": "^8.0.2",
-    "vscode-test-adapter-api": "^1.7.0",
-    "vscode-test-adapter-util": "^0.7.0",
     "yauzl": "^2.10.0",
-    "zip-a-folder": "^3.1.3"
+    "zip-a-folder": "^3.1.6"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.13",
-    "@babel/plugin-transform-modules-commonjs": "^7.18.6",
-    "@babel/preset-env": "^7.23.7",
-    "@babel/preset-react": "^7.18.6",
+    "@babel/core": "^7.24.6",
+    "@babel/plugin-transform-modules-commonjs": "^7.24.7",
+    "@babel/preset-env": "^7.24.4",
+    "@babel/preset-react": "^7.24.1",
     "@babel/preset-typescript": "^7.21.4",
-    "@faker-js/faker": "^8.0.2",
-    "@github/markdownlint-github": "^0.6.0",
-    "@octokit/plugin-throttling": "^8.0.0",
-    "@storybook/addon-a11y": "^7.4.6",
-    "@storybook/addon-actions": "^7.1.0",
-    "@storybook/addon-essentials": "^7.1.0",
-    "@storybook/addon-interactions": "^7.1.0",
-    "@storybook/addon-links": "^7.1.0",
-    "@storybook/components": "^7.6.7",
-    "@storybook/csf": "^0.1.1",
-    "@storybook/manager-api": "^7.6.6",
-    "@storybook/react": "^7.1.0",
-    "@storybook/react-webpack5": "^7.6.7",
-    "@storybook/theming": "^7.6.7",
-    "@testing-library/dom": "^9.3.0",
-    "@testing-library/jest-dom": "^6.2.0",
-    "@testing-library/react": "^14.0.0",
+    "@faker-js/faker": "^8.4.1",
+    "@github/markdownlint-github": "^0.6.2",
+    "@playwright/test": "^1.40.1",
+    "@storybook/addon-a11y": "^8.2.7",
+    "@storybook/addon-actions": "^8.2.7",
+    "@storybook/addon-essentials": "^8.2.7",
+    "@storybook/addon-interactions": "^8.2.7",
+    "@storybook/addon-links": "^8.2.7",
+    "@storybook/blocks": "^8.0.2",
+    "@storybook/components": "^8.2.7",
+    "@storybook/csf": "^0.1.11",
+    "@storybook/icons": "^1.2.10",
+    "@storybook/manager-api": "^8.2.7",
+    "@storybook/react": "^8.2.7",
+    "@storybook/react-vite": "^8.2.7",
+    "@storybook/theming": "^8.2.4",
+    "@testing-library/dom": "^10.4.0",
+    "@testing-library/jest-dom": "^6.4.8",
+    "@testing-library/react": "^16.0.0",
     "@testing-library/user-event": "^14.5.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/d3": "^7.4.0",
@@ -1974,70 +2035,68 @@
     "@types/fs-extra": "^11.0.1",
     "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "^1.1.0",
-    "@types/jest": "^29.0.2",
+    "@types/jest": "^29.5.12",
     "@types/js-yaml": "^4.0.6",
     "@types/nanoid": "^3.0.0",
-    "@types/node": "18.15.*",
+    "@types/node": "20.14.*",
     "@types/node-fetch": "^2.5.2",
-    "@types/react": "^18.0.28",
-    "@types/react-dom": "^18.0.11",
+    "@types/react": "^18.3.1",
+    "@types/react-dom": "^18.3.0",
     "@types/sarif": "^2.1.2",
-    "@types/semver": "^7.2.0",
+    "@types/semver": "^7.5.8",
     "@types/stream-json": "^1.7.1",
     "@types/styled-components": "^5.1.11",
     "@types/tar-stream": "^3.1.3",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.2.6",
-    "@types/unzipper": "^0.10.1",
     "@types/vscode": "^1.82.0",
     "@types/yauzl": "^2.10.3",
-    "@typescript-eslint/eslint-plugin": "^6.18.0",
-    "@typescript-eslint/parser": "^6.16.0",
-    "@vscode/test-electron": "^2.2.0",
-    "@vscode/vsce": "^2.19.0",
+    "@typescript-eslint/eslint-plugin": "^7.5.0",
+    "@typescript-eslint/parser": "^7.5.0",
+    "@vscode/test-electron": "^2.3.9",
+    "@vscode/vsce": "^2.24.0",
     "ansi-colors": "^4.1.1",
-    "applicationinsights": "^2.3.5",
+    "applicationinsights": "^2.9.5",
     "cosmiconfig": "^9.0.0",
     "cross-env": "^7.0.3",
-    "css-loader": "^6.8.1",
     "del": "^6.0.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-import-resolver-typescript": "^3.6.1",
+    "eslint-plugin-deprecation": "^2.0.0",
     "eslint-plugin-etc": "^2.0.2",
-    "eslint-plugin-github": "^4.4.1",
+    "eslint-plugin-github": "^4.10.2",
     "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jest-dom": "^5.0.1",
-    "eslint-plugin-prettier": "^5.0.0",
-    "eslint-plugin-react": "^7.31.8",
-    "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-storybook": "^0.6.4",
-    "file-loader": "^6.2.0",
+    "eslint-plugin-jest-dom": "^5.2.0",
+    "eslint-plugin-prettier": "^5.1.3",
+    "eslint-plugin-react": "^7.34.1",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-storybook": "^0.8.0",
     "glob": "^10.0.0",
-    "gulp": "^4.0.2",
+    "gulp": "^5.0.0",
     "gulp-esbuild": "^0.12.0",
     "gulp-replace": "^1.1.3",
     "gulp-typescript": "^5.0.1",
-    "husky": "^8.0.0",
+    "husky": "^9.0.11",
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "^15.0.2",
-    "markdownlint-cli2": "^0.11.0",
-    "markdownlint-cli2-formatter-pretty": "^0.0.5",
-    "mini-css-extract-plugin": "^2.6.1",
+    "lint-staged": "^15.2.2",
+    "markdownlint-cli2": "^0.13.0",
+    "markdownlint-cli2-formatter-pretty": "^0.0.6",
     "npm-run-all": "^4.1.5",
     "patch-package": "^8.0.0",
-    "prettier": "^3.0.0",
-    "storybook": "^7.6.7",
-    "tar-stream": "^3.0.0",
+    "prettier": "^3.2.5",
+    "storybook": "^8.2.7",
+    "tar-stream": "^3.1.7",
     "through2": "^4.0.2",
-    "ts-jest": "^29.0.1",
-    "ts-json-schema-generator": "^1.1.2",
-    "ts-loader": "^9.4.2",
-    "ts-node": "^10.7.0",
-    "ts-unused-exports": "^10.0.0",
-    "typescript": "^5.0.2"
+    "ts-jest": "^29.1.4",
+    "ts-json-schema-generator": "^2.1.1",
+    "ts-node": "^10.9.2",
+    "ts-unused-exports": "^10.1.0",
+    "typescript": "^5.0.2",
+    "vite": "^5.2.11",
+    "vite-node": "^1.5.3"
   },
   "lint-staged": {
     "./**/*.{json,css,scss}": [

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -15,6 +15,7 @@ import { pathExists, readJson, writeJson } from "fs-extra";
 import { resolve, relative } from "path";
 
 import type { Octokit } from "@octokit/core";
+import type { EndpointDefaults } from "@octokit/types";
 import type { RestEndpointMethodTypes } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
 
@@ -42,7 +43,7 @@ const octokit = new MyOctokit({
   throttle: {
     onRateLimit: (
       retryAfter: number,
-      options: any,
+      options: EndpointDefaults,
       octokit: Octokit,
     ): boolean => {
       octokit.log.warn(
@@ -53,7 +54,7 @@ const octokit = new MyOctokit({
     },
     onSecondaryRateLimit: (
       _retryAfter: number,
-      options: any,
+      options: EndpointDefaults,
       octokit: Octokit,
     ): void => {
       octokit.log.warn(

extensions/ql-vscode/scripts/find-deadcode.ts

@@ -9,12 +9,13 @@ function ignoreFile(file: string): boolean {
     containsPath(".storybook", file) ||
     containsPath(join("src", "stories"), file) ||
     pathsEqual(
-      join("test", "vscode-tests", "jest-runner-installed-extensions.ts"),
+      join("test", "vscode-tests", "jest-runner-vscode-codeql-cli.ts"),
       file,
     ) ||
     basename(file) === "jest.config.ts" ||
     basename(file) === "index.tsx" ||
-    basename(file) === "index.ts"
+    basename(file) === "index.ts" ||
+    basename(file) === "playwright.config.ts"
   );
 }
 

extensions/ql-vscode/scripts/source-map.ts

@@ -243,7 +243,7 @@ type WorkflowRunListItem = {
 async function replaceAsync(
   str: string,
   regex: RegExp,
-  replacer: (substring: string, ...args: any[]) => Promise<string>,
+  replacer: (substring: string, ...args: string[]) => Promise<string>,
 ) {
   const promises: Array<Promise<string>> = [];
   str.replace(regex, (match, ...args) => {

extensions/ql-vscode/scripts/util/vscode-versions.ts

@@ -45,8 +45,9 @@ export async function getVersionInformation(
   vscodeVersion: string,
 ): Promise<VersionInformation> {
   const vsCodePackageJson = await getVsCodePackageJson(vscodeVersion);
-  const electronVersion = minVersion(vsCodePackageJson.devDependencies.electron)
-    ?.version;
+  const electronVersion = minVersion(
+    vsCodePackageJson.devDependencies.electron,
+  )?.version;
   if (!electronVersion) {
     throw new Error("Could not find Electron version");
   }

extensions/ql-vscode/src/additional-typings.d.ts

@@ -1,15 +0,0 @@
-/**
- * The d3 library is designed to work in both the browser and
- * node. Consequently their typings files refer to both node
- * types like `Buffer` (which don't exist in the browser), and browser
- * types like `Blob` (which don't exist in node). Instead of sticking
- * all of `dom` in `compilerOptions.lib`, it suffices just to put in a
- * stub definition of the affected types so that compilation
- * succeeds.
- */
-
-declare type RequestInit = Record<string, unknown>;
-declare type ElementTagNameMap = any;
-declare type NodeListOf<T> = Record<string, T>;
-declare type Node = Record<string, unknown>;
-declare type XMLDocument = Record<string, unknown>;

extensions/ql-vscode/src/codeql-cli/cli-command.ts

@@ -3,7 +3,10 @@ import { promisify } from "util";
 
 import type { BaseLogger } from "../common/logging";
 import type { ProgressReporter } from "../common/logging/vscode";
-import { getChildProcessErrorMessage } from "../common/helpers-pure";
+import {
+  getChildProcessErrorMessage,
+  getErrorMessage,
+} from "../common/helpers-pure";
 
 /**
  * Flags to pass to all cli commands.
@@ -11,26 +14,27 @@ import { getChildProcessErrorMessage } from "../common/helpers-pure";
 export const LOGGING_FLAGS = ["-v", "--log-to-stderr"];
 
 /**
- * Runs a CodeQL CLI command without invoking the CLI server, returning the output as a string.
+ * Runs a CodeQL CLI command without invoking the CLI server, deserializing the output as JSON.
  * @param codeQlPath The path to the CLI.
  * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
  * @param commandArgs The arguments to pass to the `codeql` command.
  * @param description Description of the action being run, to be shown in log and error messages.
  * @param logger Logger to write command log messages, e.g. to an output channel.
  * @param progressReporter Used to output progress messages, e.g. to the status bar.
- * @returns The contents of the command's stdout, if the command succeeded.
+ * @returns A JSON object parsed from the contents of the command's stdout, if the command succeeded.
  */
-export async function runCodeQlCliCommand(
+export async function runJsonCodeQlCliCommand<OutputType>(
   codeQlPath: string,
   command: string[],
   commandArgs: string[],
   description: string,
   logger: BaseLogger,
   progressReporter?: ProgressReporter,
-): Promise<string> {
+): Promise<OutputType> {
   // Add logging arguments first, in case commandArgs contains positional parameters.
   const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
   const argsString = args.join(" ");
+  let stdout: string;
   try {
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
@@ -41,10 +45,18 @@ export async function runCodeQlCliCommand(
     const result = await promisify(execFile)(codeQlPath, args);
     void logger.log(result.stderr);
     void logger.log("CLI command succeeded.");
-    return result.stdout;
+    stdout = result.stdout;
   } catch (err) {
     throw new Error(
       `${description} failed: ${getChildProcessErrorMessage(err)}`,
     );
   }
+
+  try {
+    return JSON.parse(stdout) as OutputType;
+  } catch (err) {
+    throw new Error(
+      `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/cli-errors.ts

@@ -0,0 +1,102 @@
+import { asError, getErrorMessage } from "../common/helpers-pure";
+
+// https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/exit-codes
+const EXIT_CODE_USER_ERROR = 2;
+const EXIT_CODE_CANCELLED = 98;
+
+export class ExitCodeError extends Error {
+  constructor(public readonly exitCode: number | null) {
+    super(`Process exited with code ${exitCode}`);
+  }
+}
+
+export class CliError extends Error {
+  constructor(
+    message: string,
+    public readonly stderr: string | undefined,
+    public readonly cause: Error,
+    public readonly commandDescription: string,
+    public readonly commandArgs: string[],
+  ) {
+    super(message);
+  }
+}
+
+export function getCliError(
+  e: unknown,
+  stderr: string | undefined,
+  commandDescription: string,
+  commandArgs: string[],
+): CliError {
+  const error = asError(e);
+
+  if (!(error instanceof ExitCodeError) || !stderr) {
+    return formatCliErrorFallback(
+      error,
+      stderr,
+      commandDescription,
+      commandArgs,
+    );
+  }
+
+  switch (error.exitCode) {
+    case EXIT_CODE_USER_ERROR: {
+      // This is an error that we should try to format nicely
+      const fatalErrorIndex = stderr.lastIndexOf("A fatal error occurred: ");
+      if (fatalErrorIndex !== -1) {
+        return new CliError(
+          stderr.slice(fatalErrorIndex),
+          stderr,
+          error,
+          commandDescription,
+          commandArgs,
+        );
+      }
+
+      break;
+    }
+    case EXIT_CODE_CANCELLED: {
+      const cancellationIndex = stderr.lastIndexOf(
+        "Computation was cancelled: ",
+      );
+      if (cancellationIndex !== -1) {
+        return new CliError(
+          stderr.slice(cancellationIndex),
+          stderr,
+          error,
+          commandDescription,
+          commandArgs,
+        );
+      }
+
+      break;
+    }
+  }
+
+  return formatCliErrorFallback(error, stderr, commandDescription, commandArgs);
+}
+
+function formatCliErrorFallback(
+  error: Error,
+  stderr: string | undefined,
+  commandDescription: string,
+  commandArgs: string[],
+): CliError {
+  if (stderr) {
+    return new CliError(
+      stderr,
+      undefined,
+      error,
+      commandDescription,
+      commandArgs,
+    );
+  }
+
+  return new CliError(
+    getErrorMessage(error),
+    undefined,
+    error,
+    commandDescription,
+    commandArgs,
+  );
+}

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,25 +1,50 @@
 import type { SemVer } from "semver";
 import { parse } from "semver";
-import { runCodeQlCliCommand } from "./cli-command";
+import { runJsonCodeQlCliCommand } from "./cli-command";
 import type { Logger } from "../common/logging";
 import { getErrorMessage } from "../common/helpers-pure";
 
+interface VersionResult {
+  version: string;
+  features: CliFeatures | undefined;
+}
+
+export interface CliFeatures {
+  featuresInVersionResult?: boolean;
+  mrvaPackCreate?: boolean;
+  generateSummarySymbolMap?: boolean;
+}
+
+export interface VersionAndFeatures {
+  version: SemVer;
+  features: CliFeatures;
+}
+
 /**
  * Get the version of a CodeQL CLI.
  */
 export async function getCodeQlCliVersion(
   codeQlPath: string,
   logger: Logger,
-): Promise<SemVer | undefined> {
+): Promise<VersionAndFeatures | undefined> {
   try {
-    const output: string = await runCodeQlCliCommand(
+    const output: VersionResult = await runJsonCodeQlCliCommand<VersionResult>(
       codeQlPath,
       ["version"],
-      ["--format=terse"],
+      ["--format=json"],
       "Checking CodeQL version",
       logger,
     );
-    return parse(output.trim()) || undefined;
+
+    const version = parse(output.version.trim()) || undefined;
+    if (version === undefined) {
+      return undefined;
+    }
+
+    return {
+      version,
+      features: output.features ?? {},
+    };
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -34,6 +34,9 @@ import { QueryLanguage } from "../common/query-language";
 import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
 import type { Position } from "../query-server/messages";
 import { LOGGING_FLAGS } from "./cli-command";
+import type { CliFeatures, VersionAndFeatures } from "./cli-version";
+import { ExitCodeError, getCliError } from "./cli-errors";
+import { UserCancellationException } from "../common/vscode/progress";
 
 /**
  * The version of the SARIF format that we are using.
@@ -121,6 +124,16 @@ type GenerateExtensiblePredicateMetadataResult = {
   }>;
 };
 
+type PackDownloadResult = {
+  // There are other properties in this object, but they are
+  // not relevant for its use in the extension, so we omit them.
+  packs: Array<{
+    name: string;
+    version: string;
+  }>;
+  packDir: string;
+};
+
 /**
  * The expected output of `codeql resolve qlref`.
  */
@@ -144,9 +157,17 @@ type ResolvedQueries = string[];
 type ResolvedTests = string[];
 
 /**
- * A compilation message for a test message (either an error or a warning)
+ * The severity of a compilation message for a test message.
  */
-interface CompilationMessage {
+export enum CompilationMessageSeverity {
+  Error = "ERROR",
+  Warning = "WARNING",
+}
+
+/**
+ * A compilation message for a test message (either an error or a warning).
+ */
+export interface CompilationMessage {
   /**
    * The text of the message
    */
@@ -158,7 +179,7 @@ interface CompilationMessage {
   /**
    * The severity of the message
    */
-  severity: number;
+  severity: CompilationMessageSeverity;
 }
 
 /**
@@ -193,7 +214,40 @@ type OnLineCallback = (
   line: string,
 ) => Promise<string | undefined> | string | undefined;
 
-type VersionChangedListener = (newVersion: SemVer | undefined) => void;
+type VersionChangedListener = (
+  newVersionAndFeatures: VersionAndFeatures | undefined,
+) => void;
+
+type RunOptions = {
+  /**
+   * Used to output progress messages, e.g. to the status bar.
+   */
+  progressReporter?: ProgressReporter;
+  /**
+   * Used for responding to interactive output on stdout/stdin.
+   */
+  onLine?: OnLineCallback;
+  /**
+   * If true, don't print logs to the CodeQL extension log.
+   */
+  silent?: boolean;
+  /**
+   * If true, run this command in a new process rather than in the CLI server.
+   */
+  runInNewProcess?: boolean;
+  /**
+   * If runInNewProcess is true, allows cancelling the command. If runInNewProcess
+   * is false or not specified, this option is ignored.
+   */
+  token?: CancellationToken;
+};
+
+type JsonRunOptions = RunOptions & {
+  /**
+   * Whether to add commandline arguments to specify the format as JSON.
+   */
+  addFormat?: boolean;
+};
 
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
@@ -211,8 +265,8 @@ export class CodeQLCliServer implements Disposable {
   /**  A buffer with a single null byte. */
   nullBuffer: Buffer;
 
-  /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: SemVer | undefined;
+  /** Version of current cli and its supported features, lazily computed by the `getVersion()` method */
+  private _versionAndFeatures: VersionAndFeatures | undefined;
 
   private _versionChangedListeners: VersionChangedListener[] = [];
 
@@ -288,7 +342,7 @@ export class CodeQLCliServer implements Disposable {
     const callback = (): void => {
       try {
         this.killProcessIfRunning();
-        this._version = undefined;
+        this._versionAndFeatures = undefined;
         this._supportedLanguages = undefined;
       } finally {
         this.runNext();
@@ -347,7 +401,6 @@ export class CodeQLCliServer implements Disposable {
     onLine?: OnLineCallback,
     silent?: boolean,
   ): Promise<string> {
-    const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
       throw new Error("runCodeQlCliInternal called while cli was running");
     }
@@ -359,8 +412,6 @@ export class CodeQLCliServer implements Disposable {
       }
       // Grab the process so that typescript know that it is always defined.
       const process = this.process;
-      // The array of fragments of stdout
-      const stdoutBuffers: Buffer[] = [];
 
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
@@ -372,26 +423,160 @@ export class CodeQLCliServer implements Disposable {
         );
       }
       try {
-        await new Promise<void>((resolve, reject) => {
-          // Start listening to stdout
-          process.stdout.addListener("data", (newData: Buffer) => {
-            if (onLine) {
-              void (async () => {
-                const response = await onLine(newData.toString("utf-8"));
+        return await this.handleProcessOutput(process, {
+          handleNullTerminator: true,
+          onListenStart: (process) => {
+            // Write the command followed by a null terminator.
+            process.stdin.write(JSON.stringify(args), "utf8");
+            process.stdin.write(this.nullBuffer);
+          },
+          description,
+          args,
+          silent,
+          onLine,
+        });
+      } catch (err) {
+        // Kill the process if it isn't already dead.
+        this.killProcessIfRunning();
 
-                if (!response) {
-                  return;
-                }
+        throw err;
+      }
+    } finally {
+      this.commandInProcess = false;
+      // start running the next command immediately
+      this.runNext();
+    }
+  }
 
-                process.stdin.write(`${response}${EOL}`);
+  private async runCodeQlCliInNewProcess(
+    command: string[],
+    commandArgs: string[],
+    description: string,
+    onLine?: OnLineCallback,
+    silent?: boolean,
+    token?: CancellationToken,
+  ): Promise<string> {
+    const codeqlPath = await this.getCodeQlPath();
 
-                // Remove newData from stdoutBuffers because the data has been consumed
-                // by the onLine callback.
-                stdoutBuffers.splice(stdoutBuffers.indexOf(newData), 1);
-              })();
-            }
+    const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
+    const argsString = args.join(" ");
 
-            stdoutBuffers.push(newData);
+    // If we are running silently, we don't want to print anything to the console.
+    if (!silent) {
+      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+    }
+
+    const abortController = new AbortController();
+
+    const process = spawnChildProcess(codeqlPath, args, {
+      signal: abortController.signal,
+    });
+    if (!process || !process.pid) {
+      throw new Error(
+        `Failed to start ${description} using command ${codeqlPath} ${argsString}.`,
+      );
+    }
+
+    // We need to ensure that we're not killing the same process twice (since this may kill
+    // another process with the same PID), so keep track of whether we've already exited.
+    let exited = false;
+    process.on("exit", () => {
+      exited = true;
+    });
+
+    const cancellationRegistration = token?.onCancellationRequested((_e) => {
+      abortController.abort("Token was cancelled.");
+      if (process.pid && !exited) {
+        tk(process.pid);
+      }
+    });
+
+    try {
+      return await this.handleProcessOutput(process, {
+        handleNullTerminator: false,
+        description,
+        args,
+        silent,
+        onLine,
+      });
+    } catch (e) {
+      // If cancellation was requested, the error is probably just because the process was exited with SIGTERM.
+      if (token?.isCancellationRequested) {
+        void this.logger.log(
+          `The process was cancelled and exited with: ${getErrorMessage(e)}`,
+        );
+        throw new UserCancellationException(
+          `Command ${argsString} was cancelled.`,
+          true, // Don't show a warning message when the user manually cancelled the command.
+        );
+      }
+
+      throw e;
+    } finally {
+      process.stdin.end();
+      if (!exited) {
+        tk(process.pid);
+      }
+      process.stdout.destroy();
+      process.stderr.destroy();
+
+      cancellationRegistration?.dispose();
+    }
+  }
+
+  private async handleProcessOutput(
+    process: ChildProcessWithoutNullStreams,
+    {
+      handleNullTerminator,
+      args,
+      description,
+      onLine,
+      onListenStart,
+      silent,
+    }: {
+      handleNullTerminator: boolean;
+      args: string[];
+      description: string;
+      onLine?: OnLineCallback;
+      onListenStart?: (process: ChildProcessWithoutNullStreams) => void;
+      silent?: boolean;
+    },
+  ): Promise<string> {
+    const stderrBuffers: Buffer[] = [];
+    // The current buffer of stderr of a single line. To be used for logging.
+    let currentLineStderrBuffer: Buffer = Buffer.alloc(0);
+
+    // The listeners of the process. Declared here so they can be removed in the finally block.
+    let stdoutListener: ((newData: Buffer) => void) | undefined = undefined;
+    let stderrListener: ((newData: Buffer) => void) | undefined = undefined;
+    let closeListener: ((code: number | null) => void) | undefined = undefined;
+    let errorListener: ((err: Error) => void) | undefined = undefined;
+
+    try {
+      // The array of fragments of stdout
+      const stdoutBuffers: Buffer[] = [];
+
+      await new Promise<void>((resolve, reject) => {
+        stdoutListener = (newData: Buffer) => {
+          if (onLine) {
+            void (async () => {
+              const response = await onLine(newData.toString("utf-8"));
+
+              if (!response) {
+                return;
+              }
+
+              process.stdin.write(`${response}${EOL}`);
+
+              // Remove newData from stdoutBuffers because the data has been consumed
+              // by the onLine callback.
+              stdoutBuffers.splice(stdoutBuffers.indexOf(newData), 1);
+            })();
+          }
+
+          stdoutBuffers.push(newData);
+
+          if (handleNullTerminator) {
             // If the buffer ends in '0' then exit.
             // We don't have to check the middle as no output will be written after the null until
             // the next command starts
@@ -401,54 +586,112 @@ export class CodeQLCliServer implements Disposable {
             ) {
               resolve();
             }
-          });
-          // Listen to stderr
-          process.stderr.addListener("data", (newData: Buffer) => {
-            stderrBuffers.push(newData);
-          });
-          // Listen for process exit.
-          process.addListener("close", (code) => reject(code));
-          // Write the command followed by a null terminator.
-          process.stdin.write(JSON.stringify(args), "utf8");
-          process.stdin.write(this.nullBuffer);
-        });
-        // Join all the data together
-        const fullBuffer = Buffer.concat(stdoutBuffers);
-        // Make sure we remove the terminator;
-        const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        if (!silent) {
-          void this.logger.log("CLI command succeeded.");
-        }
-        return data;
-      } catch (err) {
-        // Kill the process if it isn't already dead.
-        this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
-        const newError =
-          stderrBuffers.length === 0
-            ? new Error(
-                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
-              )
-            : new Error(
-                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
-                  stderrBuffers,
-                ).toString("utf8")}`,
+          }
+        };
+        stderrListener = (newData: Buffer) => {
+          stderrBuffers.push(newData);
+
+          if (!silent) {
+            currentLineStderrBuffer = Buffer.concat([
+              currentLineStderrBuffer,
+              newData,
+            ]);
+
+            // Print the stderr to the logger as it comes in. We need to ensure that
+            // we don't split messages on the same line, so we buffer the stderr and
+            // split it on EOLs.
+            const eolBuffer = Buffer.from(EOL);
+
+            let hasCreatedSubarray = false;
+
+            let eolIndex;
+            while (
+              (eolIndex = currentLineStderrBuffer.indexOf(eolBuffer)) !== -1
+            ) {
+              const line = currentLineStderrBuffer.subarray(0, eolIndex);
+              void this.logger.log(line.toString("utf-8"));
+              currentLineStderrBuffer = currentLineStderrBuffer.subarray(
+                eolIndex + eolBuffer.length,
               );
-        newError.stack += getErrorStack(err);
-        throw newError;
-      } finally {
-        if (!silent) {
-          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
-        }
-        // Remove the listeners we set up.
-        process.stdout.removeAllListeners("data");
-        process.stderr.removeAllListeners("data");
-        process.removeAllListeners("close");
+              hasCreatedSubarray = true;
+            }
+
+            // We have created a subarray, which means that the complete original buffer is now referenced
+            // by the subarray. We need to create a new buffer to avoid memory leaks.
+            if (hasCreatedSubarray) {
+              currentLineStderrBuffer = Buffer.from(currentLineStderrBuffer);
+            }
+          }
+        };
+        closeListener = (code) => {
+          if (handleNullTerminator) {
+            reject(new ExitCodeError(code));
+          } else {
+            if (code === 0) {
+              resolve();
+            } else {
+              reject(new ExitCodeError(code));
+            }
+          }
+        };
+        errorListener = (err) => {
+          reject(err);
+        };
+
+        // Start listening to stdout
+        process.stdout.addListener("data", stdoutListener);
+        // Listen to stderr
+        process.stderr.addListener("data", stderrListener);
+        // Listen for process exit.
+        process.addListener("close", closeListener);
+        // Listen for errors
+        process.addListener("error", errorListener);
+
+        onListenStart?.(process);
+      });
+      // Join all the data together
+      const fullBuffer = Buffer.concat(stdoutBuffers);
+      // Make sure we remove the terminator
+      const data = fullBuffer.toString(
+        "utf8",
+        0,
+        handleNullTerminator ? fullBuffer.length - 1 : fullBuffer.length,
+      );
+      if (!silent) {
+        void this.logger.log(currentLineStderrBuffer.toString("utf8"));
+        currentLineStderrBuffer = Buffer.alloc(0);
+        void this.logger.log("CLI command succeeded.");
       }
+      return data;
+    } catch (err) {
+      // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
+      const cliError = getCliError(
+        err,
+        stderrBuffers.length > 0
+          ? Buffer.concat(stderrBuffers).toString("utf8")
+          : undefined,
+        description,
+        args,
+      );
+      cliError.stack += getErrorStack(err);
+      throw cliError;
     } finally {
-      this.commandInProcess = false;
-      // start running the next command immediately
-      this.runNext();
+      if (!silent && currentLineStderrBuffer.length > 0) {
+        void this.logger.log(currentLineStderrBuffer.toString("utf8"));
+      }
+      // Remove the listeners we set up.
+      if (stdoutListener) {
+        process.stdout.removeListener("data", stdoutListener);
+      }
+      if (stderrListener) {
+        process.stderr.removeListener("data", stderrListener);
+      }
+      if (closeListener) {
+        process.removeListener("close", closeListener);
+      }
+      if (errorListener) {
+        process.removeListener("error", errorListener);
+      }
     }
   }
 
@@ -565,6 +808,10 @@ export class CodeQLCliServer implements Disposable {
    * @param description Description of the action being run, to be shown in log and error messages.
    * @param progressReporter Used to output progress messages, e.g. to the status bar.
    * @param onLine Used for responding to interactive output on stdout/stdin.
+   * @param silent If true, don't print logs to the CodeQL extension log.
+   * @param runInNewProcess If true, run this command in a new process rather than in the CLI server.
+   * @param token If runInNewProcess is true, allows cancelling the command. If runInNewProcess
+   *              is false or not specified, this option is ignored.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
   runCodeQlCliCommand(
@@ -575,16 +822,25 @@ export class CodeQLCliServer implements Disposable {
       progressReporter,
       onLine,
       silent = false,
-    }: {
-      progressReporter?: ProgressReporter;
-      onLine?: OnLineCallback;
-      silent?: boolean;
-    } = {},
+      runInNewProcess = false,
+      token,
+    }: RunOptions = {},
   ): Promise<string> {
     if (progressReporter) {
       progressReporter.report({ message: description });
     }
 
+    if (runInNewProcess) {
+      return this.runCodeQlCliInNewProcess(
+        command,
+        commandArgs,
+        description,
+        onLine,
+        silent,
+        token,
+      );
+    }
+
     return new Promise((resolve, reject) => {
       // Construct the command that actually does the work
       const callback = (): void => {
@@ -617,24 +873,13 @@ export class CodeQLCliServer implements Disposable {
    * @param description Description of the action being run, to be shown in log and error messages.
    * @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
    * @param progressReporter Used to output progress messages, e.g. to the status bar.
-   * @param onLine Used for responding to interactive output on stdout/stdin.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
   async runJsonCodeQlCliCommand<OutputType>(
     command: string[],
     commandArgs: string[],
     description: string,
-    {
-      addFormat = true,
-      progressReporter,
-      onLine,
-      silent = false,
-    }: {
-      addFormat?: boolean;
-      progressReporter?: ProgressReporter;
-      onLine?: OnLineCallback;
-      silent?: boolean;
-    } = {},
+    { addFormat = true, ...runOptions }: JsonRunOptions = {},
   ): Promise<OutputType> {
     let args: string[] = [];
     if (addFormat) {
@@ -642,11 +887,12 @@ export class CodeQLCliServer implements Disposable {
       args = args.concat(["--format", "json"]);
     }
     args = args.concat(commandArgs);
-    const result = await this.runCodeQlCliCommand(command, args, description, {
-      progressReporter,
-      onLine,
-      silent,
-    });
+    const result = await this.runCodeQlCliCommand(
+      command,
+      args,
+      description,
+      runOptions,
+    );
     try {
       return JSON.parse(result) as OutputType;
     } catch (err) {
@@ -674,21 +920,14 @@ export class CodeQLCliServer implements Disposable {
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
    * @param commandArgs The arguments to pass to the `codeql` command.
    * @param description Description of the action being run, to be shown in log and error messages.
-   * @param addFormat Whether or not to add commandline arguments to specify the format as JSON.
-   * @param progressReporter Used to output progress messages, e.g. to the status bar.
+   * @param runOptions Options for running the command.
    * @returns The contents of the command's stdout, if the command succeeded.
    */
   async runJsonCodeQlCliCommandWithAuthentication<OutputType>(
     command: string[],
     commandArgs: string[],
     description: string,
-    {
-      addFormat,
-      progressReporter,
-    }: {
-      addFormat?: boolean;
-      progressReporter?: ProgressReporter;
-    } = {},
+    runOptions: Omit<JsonRunOptions, "onLine"> = {},
   ): Promise<OutputType> {
     const accessToken = await this.app.credentials.getExistingAccessToken();
 
@@ -699,8 +938,7 @@ export class CodeQLCliServer implements Disposable {
       [...extraArgs, ...commandArgs],
       description,
       {
-        addFormat,
-        progressReporter,
+        ...runOptions,
         onLine: async (line) => {
           if (line.startsWith("Enter value for --github-auth-stdin")) {
             try {
@@ -973,10 +1211,15 @@ export class CodeQLCliServer implements Disposable {
     outputPath: string,
     endSummaryPath: string,
   ): Promise<string> {
+    const supportsGenerateSummarySymbolMap =
+      await this.cliConstraints.supportsGenerateSummarySymbolMap();
     const subcommandArgs = [
       "--format=text",
       `--end-summary=${endSummaryPath}`,
       "--sourcemap",
+      ...(supportsGenerateSummarySymbolMap
+        ? ["--summary-symbol-map", "--minify-output"]
+        : []),
       inputPath,
       outputPath,
     ];
@@ -1256,12 +1499,6 @@ export class CodeQLCliServer implements Disposable {
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
     if (extensionPacksOnly) {
-      if (!(await this.cliConstraints.supportsQlpacksKind())) {
-        void this.logger.log(
-          "Warning: Running with extension packs is only supported by CodeQL CLI v2.12.3 or later.",
-        );
-        return {};
-      }
       args.push("--kind", "extension", "--no-recursive");
     } else if (kind) {
       args.push("--kind", kind);
@@ -1369,25 +1606,30 @@ export class CodeQLCliServer implements Disposable {
   async packAdd(dir: string, queryLanguage: QueryLanguage) {
     const args = ["--dir", dir];
     args.push(`codeql/${queryLanguage}-all`);
-    return this.runJsonCodeQlCliCommandWithAuthentication(
+    return this.runCodeQlCliCommand(
       ["pack", "add"],
       args,
       `Adding and installing ${queryLanguage} pack dependency.`,
-      {
-        addFormat: false,
-      },
     );
   }
 
   /**
    * Downloads a specified pack.
    * @param packs The `<package-scope/name[@version]>` of the packs to download.
+   * @param token The cancellation token. If not specified, the command will be run in the CLI server.
    */
-  async packDownload(packs: string[]) {
+  async packDownload(
+    packs: string[],
+    token?: CancellationToken,
+  ): Promise<PackDownloadResult> {
     return this.runJsonCodeQlCliCommandWithAuthentication(
       ["pack", "download"],
       packs,
       "Downloading packs",
+      {
+        runInNewProcess: !!token, // Only run in a new process if a token is provided
+        token,
+      },
     );
   }
 
@@ -1400,15 +1642,13 @@ export class CodeQLCliServer implements Disposable {
       args.push("--mode", "update");
     }
     if (workspaceFolders?.length > 0) {
-      if (await this.cliConstraints.supportsAdditionalPacksInstall()) {
-        args.push(
-          // Allow prerelease packs from the ql submodule.
-          "--allow-prerelease",
-          // Allow the use of --additional-packs argument without issueing a warning
-          "--no-strict-mode",
-          ...this.getAdditionalPacksArg(workspaceFolders),
-        );
-      }
+      args.push(
+        // Allow prerelease packs from the ql submodule.
+        "--allow-prerelease",
+        // Allow the use of --additional-packs argument without issueing a warning
+        "--no-strict-mode",
+        ...this.getAdditionalPacksArg(workspaceFolders),
+      );
     }
     return this.runJsonCodeQlCliCommandWithAuthentication(
       ["pack", "install"],
@@ -1417,16 +1657,30 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Compile a CodeQL pack and bundle it into a single file.
+   *
+   * @param sourcePackDir The directory of the input CodeQL pack.
+   * @param workspaceFolders The workspace folders to search for additional packs.
+   * @param outputBundleFile The path to the output bundle file.
+   * @param outputPackDir The directory to contain the unbundled output pack.
+   * @param moreOptions Additional options to be passed to `codeql pack bundle`.
+   * @param token Cancellation token for the operation.
+   */
   async packBundle(
-    dir: string,
+    sourcePackDir: string,
     workspaceFolders: string[],
-    outputPath: string,
+    outputBundleFile: string,
+    outputPackDir: string,
     moreOptions: string[],
+    token?: CancellationToken,
   ): Promise<void> {
     const args = [
       "-o",
-      outputPath,
-      dir,
+      outputBundleFile,
+      sourcePackDir,
+      "--pack-path",
+      outputPackDir,
       ...moreOptions,
       ...this.getAdditionalPacksArg(workspaceFolders),
     ];
@@ -1435,6 +1689,10 @@ export class CodeQLCliServer implements Disposable {
       ["pack", "bundle"],
       args,
       "Bundling pack",
+      {
+        runInNewProcess: true,
+        token,
+      },
     );
   }
 
@@ -1481,29 +1739,21 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  public async getVersion() {
-    if (!this._version) {
-      try {
-        const newVersion = await this.refreshVersion();
-        this._version = newVersion;
-        this._versionChangedListeners.forEach((listener) =>
-          listener(newVersion),
-        );
+  public async getVersion(): Promise<SemVer> {
+    return (await this.getVersionAndFeatures()).version;
+  }
 
-        // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
-        await this.app.commands.execute(
-          "setContext",
-          "codeql.supportsQuickEvalCount",
-          newVersion.compare(
-            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
-          ) >= 0,
-        );
-        await this.app.commands.execute(
-          "setContext",
-          "codeql.supportsTrimCache",
-          newVersion.compare(
-            CliVersionConstraint.CLI_VERSION_WITH_TRIM_CACHE,
-          ) >= 0,
+  public async getFeatures(): Promise<CliFeatures> {
+    return (await this.getVersionAndFeatures()).features;
+  }
+
+  private async getVersionAndFeatures(): Promise<VersionAndFeatures> {
+    if (!this._versionAndFeatures) {
+      try {
+        const newVersionAndFeatures = await this.refreshVersion();
+        this._versionAndFeatures = newVersionAndFeatures;
+        this._versionChangedListeners.forEach((listener) =>
+          listener(newVersionAndFeatures),
         );
       } catch (e) {
         this._versionChangedListeners.forEach((listener) =>
@@ -1512,23 +1762,23 @@ export class CodeQLCliServer implements Disposable {
         throw e;
       }
     }
-    return this._version;
+    return this._versionAndFeatures;
   }
 
   public addVersionChangedListener(listener: VersionChangedListener) {
-    if (this._version) {
-      listener(this._version);
+    if (this._versionAndFeatures) {
+      listener(this._versionAndFeatures);
     }
     this._versionChangedListeners.push(listener);
   }
 
-  private async refreshVersion() {
+  private async refreshVersion(): Promise<VersionAndFeatures> {
     const distribution = await this.distributionProvider.getDistribution();
     switch (distribution.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
       // eslint-disable-next-line no-fallthrough -- Intentional fallthrough
       case FindDistributionResultKind.IncompatibleDistribution:
-        return distribution.version;
+        return distribution.versionAndFeatures;
 
       default:
         // We should not get here because if no distributions are available, then
@@ -1541,11 +1791,8 @@ export class CodeQLCliServer implements Disposable {
     return paths.length ? ["--additional-packs", paths.join(delimiter)] : [];
   }
 
-  public async useExtensionPacks(): Promise<boolean> {
-    return (
-      this.cliConfig.useExtensionPacks &&
-      (await this.cliConstraints.supportsQlpacksKind())
-    );
+  public useExtensionPacks(): boolean {
+    return this.cliConfig.useExtensionPacks;
   }
 
   public async setUseExtensionPacks(useExtensionPacks: boolean) {
@@ -1573,8 +1820,8 @@ export function spawnServer(
   command: string[],
   commandArgs: string[],
   logger: Logger,
-  stderrListener: (data: any) => void,
-  stdoutListener?: (data: any) => void,
+  stderrListener: (data: string | Buffer) => void,
+  stdoutListener?: (data: string | Buffer) => void,
   progressReporter?: ProgressReporter,
 ): ChildProcessWithoutNullStreams {
   // Enable verbose logging.
@@ -1594,7 +1841,7 @@ export function spawnServer(
     );
   }
 
-  let lastStdout: any = undefined;
+  let lastStdout: string | Buffer | undefined = undefined;
   child.stdout!.on("data", (data) => {
     lastStdout = data;
   });
@@ -1647,7 +1894,7 @@ function isEnvTrue(name: string): boolean {
   );
 }
 
-export function shouldDebugIdeServer() {
+export function shouldDebugLanguageServer() {
   return isEnvTrue("IDE_SERVER_JAVA_DEBUG");
 }
 
@@ -1662,47 +1909,16 @@ function shouldDebugCliServer() {
 export class CliVersionConstraint {
   // The oldest version of the CLI that we support. This is used to determine
   // whether to show a warning about the CLI being too old on startup.
-  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.11.6");
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("v2.15.5");
 
-  /**
-   * CLI version that supports the `--kind` option for the `resolve qlpacks` command.
-   */
-  public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
-
-  /**
-   * CLI version that supports the `--additional-packs` option for the `pack install` command.
-   */
-  public static CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL = new SemVer(
-    "2.12.4",
-  );
-
-  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
-
-  /**
-   * CLI version where the query server supports quick-eval count mode.
-   */
-  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
-
-  /**
-   * CLI version where the `generate extensible-predicate-metadata`
-   * command was implemented.
-   */
-  public static CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA = new SemVer(
-    "2.14.3",
+  public static CLI_VERSION_WITHOUT_MRVA_EXTENSIBLE_PREDICATE_HACK = new SemVer(
+    "2.16.1",
   );
 
   /**
-   * CLI version where the langauge server supports visisbility change notifications.
+   * CLI version where there is support for multiple queries on the pack create command.
    */
-  public static CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS = new SemVer(
-    "2.14.0",
-  );
-
-  /**
-   * CLI version where the query server supports the `evaluation/trimCache` method
-   * with `codeql database cleanup --mode=trim` semantics.
-   */
-  public static CLI_VERSION_WITH_TRIM_CACHE = new SemVer("2.15.1");
+  public static CLI_VERSION_WITH_MULTI_QUERY_PACK_CREATE = new SemVer("2.16.1");
 
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
@@ -1712,37 +1928,24 @@ export class CliVersionConstraint {
     return (await this.cli.getVersion()).compare(v) >= 0;
   }
 
-  async supportsQlpacksKind() {
+  async preservesExtensiblePredicatesInMrvaPack() {
+    // Negated, because we _stopped_ preserving these in 2.16.1.
+    return !(await this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITHOUT_MRVA_EXTENSIBLE_PREDICATE_HACK,
+    ));
+  }
+
+  async supportsPackCreateWithMultipleQueries() {
     return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
+      CliVersionConstraint.CLI_VERSION_WITH_MULTI_QUERY_PACK_CREATE,
     );
   }
 
-  async supportsAdditionalPacksInstall() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
-    );
+  async supportsMrvaPackCreate(): Promise<boolean> {
+    return (await this.cli.getFeatures()).mrvaPackCreate === true;
   }
 
-  async usesGlobalCompilationCache() {
-    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
-  }
-
-  async supportsVisibilityNotifications() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
-    );
-  }
-
-  async supportsQuickEvalCount() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
-    );
-  }
-
-  async supportsGenerateExtensiblePredicateMetadata() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA,
-    );
+  async supportsGenerateSummarySymbolMap(): Promise<boolean> {
+    return (await this.cli.getFeatures()).generateSummarySymbolMap === true;
   }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -1,11 +1,12 @@
+import type { WriteStream } from "fs";
 import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
-import type { SemVer } from "semver";
 import { Range, satisfies } from "semver";
 import type { Event, ExtensionContext } from "vscode";
 import type { DistributionConfig } from "../config";
 import { extLogger } from "../common/logging/vscode";
+import type { VersionAndFeatures } from "./cli-version";
 import { getCodeQlCliVersion } from "./cli-version";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { reportStreamProgress } from "../common/vscode/progress";
@@ -26,6 +27,8 @@ import { unzipToDirectoryConcurrently } from "../common/unzip-concurrently";
 import { reportUnzipProgress } from "../common/vscode/unzip-progress";
 import type { Release } from "./distribution/release";
 import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";
+import { createTimeoutSignal } from "../common/fetch-stream";
+import { AbortError } from "node-fetch";
 
 /**
  * distribution.ts
@@ -88,11 +91,11 @@ export class DistributionManager implements DistributionProvider {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(
+    const versionAndFeatures = await getCodeQlCliVersion(
       distribution.codeQlPath,
       extLogger,
     );
-    if (version === undefined) {
+    if (versionAndFeatures === undefined) {
       return {
         distribution,
         kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
@@ -119,17 +122,21 @@ export class DistributionManager implements DistributionProvider {
       distribution.kind !== DistributionKind.ExtensionManaged ||
       this.config.includePrerelease;
 
-    if (!satisfies(version, this.versionRange, { includePrerelease })) {
+    if (
+      !satisfies(versionAndFeatures.version, this.versionRange, {
+        includePrerelease,
+      })
+    ) {
       return {
         distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
-        version,
+        versionAndFeatures,
       };
     }
     return {
       distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
-      version,
+      versionAndFeatures,
     };
   }
 
@@ -380,15 +387,25 @@ class ExtensionSpecificDistributionManager {
       );
     }
 
-    const assetStream =
-      await this.createReleasesApiConsumer().streamBinaryContentOfAsset(
-        assets[0],
-      );
+    const {
+      signal,
+      onData,
+      dispose: disposeTimeout,
+    } = createTimeoutSignal(this.config.downloadTimeout);
+
     const tmpDirectory = await mkdtemp(join(tmpdir(), "vscode-codeql"));
 
+    let archiveFile: WriteStream | undefined = undefined;
+
     try {
+      const assetStream =
+        await this.createReleasesApiConsumer().streamBinaryContentOfAsset(
+          assets[0],
+          signal,
+        );
+
       const archivePath = join(tmpDirectory, "distributionDownload.zip");
-      const archiveFile = createWriteStream(archivePath);
+      archiveFile = createWriteStream(archivePath);
 
       const contentLength = assetStream.headers.get("content-length");
       const totalNumBytes = contentLength
@@ -401,12 +418,23 @@ class ExtensionSpecificDistributionManager {
         progressCallback,
       );
 
-      await new Promise((resolve, reject) =>
+      assetStream.body.on("data", onData);
+
+      await new Promise((resolve, reject) => {
+        if (!archiveFile) {
+          throw new Error("Invariant violation: archiveFile not set");
+        }
+
         assetStream.body
           .pipe(archiveFile)
           .on("finish", resolve)
-          .on("error", reject),
-      );
+          .on("error", reject);
+
+        // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
+        assetStream.body.on("error", reject);
+      });
+
+      disposeTimeout();
 
       await this.bumpDistributionFolderIndex();
 
@@ -423,7 +451,19 @@ class ExtensionSpecificDistributionManager {
             )
           : undefined,
       );
+    } catch (e) {
+      if (e instanceof AbortError) {
+        const thrownError = new AbortError("The download timed out.");
+        thrownError.stack = e.stack;
+        throw thrownError;
+      }
+
+      throw e;
     } finally {
+      disposeTimeout();
+
+      archiveFile?.close();
+
       await remove(tmpDirectory);
     }
   }
@@ -599,7 +639,7 @@ interface DistributionResult {
 
 interface CompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.CompatibleDistribution;
-  version: SemVer;
+  versionAndFeatures: VersionAndFeatures;
 }
 
 interface UnknownCompatibilityDistributionResult extends DistributionResult {
@@ -608,7 +648,7 @@ interface UnknownCompatibilityDistributionResult extends DistributionResult {
 
 interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: SemVer;
+  versionAndFeatures: VersionAndFeatures;
 }
 
 interface NoDistributionResult {

extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts

@@ -90,21 +90,28 @@ export class ReleasesApiConsumer {
 
   public async streamBinaryContentOfAsset(
     asset: ReleaseAsset,
+    signal?: AbortSignal,
   ): Promise<Response> {
     const apiPath = `/repos/${this.repositoryNwo}/releases/assets/${asset.id}`;
 
-    return await this.makeApiCall(apiPath, {
-      accept: "application/octet-stream",
-    });
+    return await this.makeApiCall(
+      apiPath,
+      {
+        accept: "application/octet-stream",
+      },
+      signal,
+    );
   }
 
   protected async makeApiCall(
     apiPath: string,
     additionalHeaders: { [key: string]: string } = {},
+    signal?: AbortSignal,
   ): Promise<Response> {
     const response = await this.makeRawRequest(
       ReleasesApiConsumer.apiBase + apiPath,
       Object.assign({}, this.defaultHeaders, additionalHeaders),
+      signal,
     );
 
     if (!response.ok) {
@@ -129,11 +136,13 @@ export class ReleasesApiConsumer {
   private async makeRawRequest(
     requestUrl: string,
     headers: { [key: string]: string },
+    signal?: AbortSignal,
     redirectCount = 0,
   ): Promise<Response> {
     const response = await fetch(requestUrl, {
       headers,
       redirect: "manual",
+      signal,
     });
 
     const redirectUrl = response.headers.get("location");
@@ -153,7 +162,12 @@ export class ReleasesApiConsumer {
         // mechanism is provided.
         delete headers["authorization"];
       }
-      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
+      return await this.makeRawRequest(
+        redirectUrl,
+        headers,
+        signal,
+        redirectCount + 1,
+      );
     }
 
     return response;
@@ -191,7 +205,7 @@ export interface GithubRelease {
  *
  * This type must match the format of the GitHub API and is not intended to be used outside of this file except for tests. Please use the `ReleaseAsset` type instead.
  */
-export interface GithubReleaseAsset {
+interface GithubReleaseAsset {
   id: number;
   name: string;
   size: number;

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -1,5 +1,5 @@
 import type { CodeQLCliServer } from "./cli";
-import type { Uri } from "vscode";
+import type { CancellationToken, Uri } from "vscode";
 import { window } from "vscode";
 import {
   getLanguageDisplayName,
@@ -50,6 +50,7 @@ export async function findLanguage(
 export async function askForLanguage(
   cliServer: CodeQLCliServer,
   throwOnEmpty = true,
+  token?: CancellationToken,
 ): Promise<QueryLanguage | undefined> {
   const supportedLanguages = await cliServer.getSupportedLanguages();
 
@@ -62,10 +63,14 @@ export async function askForLanguage(
     }))
     .sort((a, b) => a.label.localeCompare(b.label));
 
-  const selectedItem = await window.showQuickPick(items, {
-    placeHolder: "Select target language for your query",
-    ignoreFocusOut: true,
-  });
+  const selectedItem = await window.showQuickPick(
+    items,
+    {
+      placeHolder: "Select target query language",
+      ignoreFocusOut: true,
+    },
+    token,
+  );
   if (!selectedItem) {
     // This only happens if the user cancels the quick pick.
     if (throwOnEmpty) {

extensions/ql-vscode/src/common/authentication.ts

@@ -31,4 +31,9 @@ export interface Credentials {
    * @returns An OAuth access token, or undefined.
    */
   getExistingAccessToken(): Promise<string | undefined>;
+
+  /**
+   * Returns the ID of the authentication provider to use.
+   */
+  authProviderId: string;
 }

extensions/ql-vscode/src/common/commands.ts

@@ -1,10 +1,9 @@
 import type { CommandManager } from "../packages/commands";
-import type { Uri, Range, TextDocumentShowOptions } from "vscode";
+import type { Uri, Range, TextDocumentShowOptions, TestItem } from "vscode";
 import type { AstItem } from "../language-support";
 import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
 import type { DatabaseItem } from "../databases/local-databases";
 import type { QueryHistoryInfo } from "../query-history/query-history-info";
-import type { TestTreeNode } from "../query-testing/test-tree-node";
 import type {
   VariantAnalysis,
   VariantAnalysisScannedRepository,
@@ -221,6 +220,7 @@ export type LocalDatabasesCommands = {
 
   // Explorer context menu
   "codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
+  "codeQL.importTestDatabase": (uri: Uri) => Promise<void>;
 
   // Database panel view title commands
   "codeQLDatabases.chooseDatabaseFolder": () => Promise<void>;
@@ -273,12 +273,17 @@ export type VariantAnalysisCommands = {
   "codeQL.openVariantAnalysisLogs": (
     variantAnalysisId: number,
   ) => Promise<void>;
+  "codeQLModelAlerts.openVariantAnalysisLogs": (
+    variantAnalysisId: number,
+  ) => Promise<void>;
   "codeQL.openVariantAnalysisView": (
     variantAnalysisId: number,
   ) => Promise<void>;
-  "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
-  "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQL.runVariantAnalysis": () => Promise<void>;
+  "codeQL.runVariantAnalysisContextEditor": (uri: Uri) => Promise<void>;
+  "codeQL.runVariantAnalysisContextExplorer": ExplorerSelectionCommandFunction<Uri>;
   "codeQLQueries.runVariantAnalysisContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQL.runVariantAnalysisPublishedPack": () => Promise<void>;
 };
 
 export type DatabasePanelCommands = {
@@ -334,11 +339,9 @@ export type SummaryLanguageSupportCommands = {
 };
 
 export type TestUICommands = {
-  "codeQLTests.showOutputDifferences": (node: TestTreeNode) => Promise<void>;
-  "codeQLTests.acceptOutput": (node: TestTreeNode) => Promise<void>;
-  "codeQLTests.acceptOutputContextTestItem": (
-    node: TestTreeNode,
-  ) => Promise<void>;
+  "codeQLTests.showOutputDifferences": (node: TestItem) => Promise<void>;
+  "codeQLTests.acceptOutput": (node: TestItem) => Promise<void>;
+  "codeQLTests.acceptOutputContextTestItem": (node: TestItem) => Promise<void>;
 };
 
 export type MockGitHubApiServerCommands = {

extensions/ql-vscode/src/common/config-template.ts

@@ -0,0 +1,57 @@
+// Based on https://github.com/microsoft/vscode/blob/edfd5b8ba54d50f3f5c2ebee877af088803def88/src/vs/base/common/labels.ts#L316C1-L400
+
+/**
+ * Helper to insert values for specific template variables into the string. E.g. "this ${is} a ${template}" can be
+ * passed to this function together with an object that maps "is" and "template" to strings to have them replaced.
+ *
+ * @param template string to which template is applied
+ * @param values the values of the templates to use
+ */
+export function substituteConfigVariables(
+  template: string,
+  values: {
+    [key: string]: string | undefined | null;
+  },
+): string {
+  const segments: string[] = [];
+
+  let inVariable = false;
+  let currentValue = "";
+  for (const char of template) {
+    // Beginning of variable
+    if (char === "$" || (inVariable && char === "{")) {
+      if (currentValue) {
+        segments.push(currentValue);
+      }
+
+      currentValue = "";
+      inVariable = true;
+    }
+
+    // End of variable
+    else if (char === "}" && inVariable) {
+      const resolved = values[currentValue];
+
+      // Variable
+      if (resolved && resolved.length > 0) {
+        segments.push(resolved);
+      }
+      // If the variable, doesn't exist, we discard it (i.e. replace it by the empty string)
+
+      currentValue = "";
+      inVariable = false;
+    }
+
+    // Text or Variable Name
+    else {
+      currentValue += char;
+    }
+  }
+
+  // Tail
+  if (currentValue && !inVariable) {
+    segments.push(currentValue);
+  }
+
+  return segments.join("");
+}

extensions/ql-vscode/src/common/disposable-object.ts

@@ -1,7 +1,7 @@
 // Avoid explicitly referencing Disposable type in vscode.
 // This file cannot have dependencies on the vscode API.
 export interface Disposable {
-  dispose(): any;
+  dispose(): unknown;
 }
 
 export type DisposeHandler = (disposable: Disposable) => void;

extensions/ql-vscode/src/common/errors.ts

@@ -84,12 +84,13 @@ export interface ErrorLike {
   stack?: string;
 }
 
-function isErrorLike(error: any): error is ErrorLike {
-  if (
+function isErrorLike(error: unknown): error is ErrorLike {
+  return (
+    error !== undefined &&
+    error !== null &&
+    typeof error === "object" &&
+    "message" in error &&
     typeof error.message === "string" &&
-    (error.stack === undefined || typeof error.stack === "string")
-  ) {
-    return true;
-  }
-  return false;
+    (!("stack" in error) || typeof error.stack === "string")
+  );
 }

extensions/ql-vscode/src/common/fetch-stream.ts

@@ -0,0 +1,36 @@
+import { clearTimeout } from "node:timers";
+
+export function createTimeoutSignal(timeoutSeconds: number): {
+  signal: AbortSignal;
+  onData: () => void;
+  dispose: () => void;
+} {
+  const timeout = timeoutSeconds * 1000;
+
+  const abortController = new AbortController();
+
+  let timeoutId: NodeJS.Timeout;
+
+  // If we don't get any data within the timeout, abort the download
+  timeoutId = setTimeout(() => {
+    abortController.abort();
+  }, timeout);
+
+  // If we receive any data within the timeout, reset the timeout
+  const onData = () => {
+    clearTimeout(timeoutId);
+    timeoutId = setTimeout(() => {
+      abortController.abort();
+    }, timeout);
+  };
+
+  const dispose = () => {
+    clearTimeout(timeoutId);
+  };
+
+  return {
+    signal: abortController.signal,
+    onData,
+    dispose,
+  };
+}

extensions/ql-vscode/src/common/filenames.ts

@@ -0,0 +1,44 @@
+type FilenameOptions = {
+  removeDots?: boolean;
+};
+
+/**
+ * This will create a filename from an arbitrary string by removing
+ * all characters which are not allowed in filenames and making them
+ * more filesystem-friendly be replacing undesirable characters with
+ * hyphens. The result will always be lowercase ASCII.
+ *
+ * @param str The string to create a filename from
+ * @param removeDots Whether to remove dots from the filename [default: false]
+ * @returns The filename
+ */
+export function createFilenameFromString(
+  str: string,
+  { removeDots }: FilenameOptions = {},
+) {
+  let fileName = str;
+
+  // Lowercase everything
+  fileName = fileName.toLowerCase();
+
+  // Replace all spaces, underscores, slashes, and backslashes with hyphens
+  fileName = fileName.replaceAll(/[\s_/\\]+/g, "-");
+
+  // Replace all characters which are not allowed by empty strings
+  fileName = fileName.replaceAll(/[^a-z0-9.-]/g, "");
+
+  // Remove any leading or trailing hyphens or dots
+  fileName = fileName.replaceAll(/^[.-]+|[.-]+$/g, "");
+
+  // Replace dots by hyphens if dots are not allowed
+  if (removeDots) {
+    fileName = fileName.replaceAll(/\./g, "-");
+  }
+
+  // Remove any duplicate hyphens
+  fileName = fileName.replaceAll(/-{2,}/g, "-");
+  // Remove any duplicate dots
+  fileName = fileName.replaceAll(/\.{2,}/g, ".");
+
+  return fileName;
+}

extensions/ql-vscode/src/common/files.ts

@@ -1,5 +1,5 @@
-import { pathExists, stat, readdir, opendir } from "fs-extra";
-import { isAbsolute, join, relative, resolve } from "path";
+import { pathExists, stat, readdir, opendir, lstatSync } from "fs-extra";
+import { dirname, isAbsolute, join, relative, resolve } from "path";
 import { tmpdir as osTmpdir } from "os";
 
 /**
@@ -124,11 +124,84 @@ export interface IOError {
   readonly code: string;
 }
 
-export function isIOError(e: any): e is IOError {
-  return e.code !== undefined && typeof e.code === "string";
+export function isIOError(e: unknown): e is IOError {
+  return (
+    e !== undefined &&
+    e !== null &&
+    typeof e === "object" &&
+    "code" in e &&
+    typeof e.code === "string"
+  );
 }
 
 // This function is a wrapper around `os.tmpdir()` to make it easier to mock in tests.
 export function tmpdir(): string {
   return osTmpdir();
 }
+
+/**
+ * Finds the common parent directory of an arbitrary number of absolute paths. The result
+ * will be an absolute path.
+ * @param paths The array of paths.
+ * @returns The common parent directory of the paths.
+ */
+export function findCommonParentDir(...paths: string[]): string {
+  if (paths.length === 0) {
+    throw new Error("At least one path must be provided");
+  }
+  if (paths.some((path) => !isAbsolute(path))) {
+    throw new Error("All paths must be absolute");
+  }
+
+  paths = paths.map((path) => normalizePath(path));
+
+  // If there's only one path and it's a file, return its dirname
+  if (paths.length === 1) {
+    return lstatSync(paths[0]).isFile() ? dirname(paths[0]) : paths[0];
+  }
+
+  let commonDir = paths[0];
+  while (!paths.every((path) => containsPath(commonDir, path))) {
+    if (isTopLevelPath(commonDir)) {
+      throw new Error(
+        "Reached filesystem root and didn't find a common parent directory",
+      );
+    }
+    commonDir = dirname(commonDir);
+  }
+
+  return commonDir;
+}
+
+function isTopLevelPath(path: string): boolean {
+  return dirname(path) === path;
+}
+
+/**
+ * Recursively looks for a file in a directory. If the file exists, then returns the directory containing the file.
+ *
+ * @param dir The directory to search
+ * @param toFind The file to recursively look for in this directory
+ *
+ * @returns the directory containing the file, or undefined if not found.
+ */
+export async function findDirWithFile(
+  dir: string,
+  ...toFind: string[]
+): Promise<string | undefined> {
+  if (!(await stat(dir)).isDirectory()) {
+    return;
+  }
+  const files = await readdir(dir);
+  if (toFind.some((file) => files.includes(file))) {
+    return dir;
+  }
+  for (const file of files) {
+    const newPath = join(dir, file);
+    const result = await findDirWithFile(newPath, ...toFind);
+    if (result) {
+      return result;
+    }
+  }
+  return;
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -29,37 +29,45 @@ function validGitHubNwoOrOwner(
 
 /**
  * Extracts an NWO from a GitHub URL.
- * @param githubUrl The GitHub repository URL
+ * @param repositoryUrl The GitHub repository URL
+ * @param githubUrl The URL of the GitHub instance
  * @return The corresponding NWO, or undefined if the URL is not valid
  */
-export function getNwoFromGitHubUrl(githubUrl: string): string | undefined {
-  return getNwoOrOwnerFromGitHubUrl(githubUrl, "nwo");
+export function getNwoFromGitHubUrl(
+  repositoryUrl: string,
+  githubUrl: URL,
+): string | undefined {
+  return getNwoOrOwnerFromGitHubUrl(repositoryUrl, githubUrl, "nwo");
 }
 
 /**
  * Extracts an owner from a GitHub URL.
- * @param githubUrl The GitHub repository URL
+ * @param repositoryUrl The GitHub repository URL
+ * @param githubUrl The URL of the GitHub instance
  * @return The corresponding Owner, or undefined if the URL is not valid
  */
-export function getOwnerFromGitHubUrl(githubUrl: string): string | undefined {
-  return getNwoOrOwnerFromGitHubUrl(githubUrl, "owner");
+export function getOwnerFromGitHubUrl(
+  repositoryUrl: string,
+  githubUrl: URL,
+): string | undefined {
+  return getNwoOrOwnerFromGitHubUrl(repositoryUrl, githubUrl, "owner");
 }
 
 function getNwoOrOwnerFromGitHubUrl(
-  githubUrl: string,
+  repositoryUrl: string,
+  githubUrl: URL,
   kind: "owner" | "nwo",
 ): string | undefined {
+  const validHostnames = [githubUrl.hostname, `www.${githubUrl.hostname}`];
+
   try {
     let paths: string[];
-    const urlElements = githubUrl.split("/");
-    if (
-      urlElements[0] === "github.com" ||
-      urlElements[0] === "www.github.com"
-    ) {
-      paths = githubUrl.split("/").slice(1);
+    const urlElements = repositoryUrl.split("/");
+    if (validHostnames.includes(urlElements[0])) {
+      paths = repositoryUrl.split("/").slice(1);
     } else {
-      const uri = new URL(githubUrl);
-      if (uri.hostname !== "github.com" && uri.hostname !== "www.github.com") {
+      const uri = new URL(repositoryUrl);
+      if (!validHostnames.includes(uri.hostname)) {
         return;
       }
       paths = uri.pathname.split("/").filter((segment: string) => segment);

extensions/ql-vscode/src/common/helpers-pure.ts

@@ -7,6 +7,18 @@
 
 import { RedactableError } from "./errors";
 
+// Matches any type that is not an array. This is useful to help avoid
+// nested arrays, or for cases like createSingleSelectionCommand to avoid T
+// accidentally getting instantiated as DatabaseItem[] instead of DatabaseItem.
+export type NotArray =
+  | string
+  | bigint
+  | number
+  | boolean
+  | (object & {
+      length?: never;
+    });
+
 /**
  * This error is used to indicate a runtime failure of an exhaustivity check enforced at compile time.
  */

extensions/ql-vscode/src/common/interface-types.ts

@@ -10,10 +10,11 @@ import type {
 } from "../variant-analysis/shared/variant-analysis-filter-sort";
 import type { ErrorLike } from "../common/errors";
 import type { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
-import type { Method } from "../model-editor/method";
+import type { Method, MethodSignature } from "../model-editor/method";
 import type { ModeledMethod } from "../model-editor/modeled-method";
 import type {
   MethodModelingPanelViewState,
+  ModelAlertsViewState,
   ModelEditorViewState,
 } from "../model-editor/shared/view-state";
 import type { Mode } from "../model-editor/shared/mode";
@@ -24,6 +25,8 @@ import type {
   Row,
   UrlValueResolvable,
 } from "./raw-result-types";
+import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
+import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
 
 /**
  * This module contains types and code that are shared between
@@ -144,6 +147,21 @@ interface SetStateMsg {
   parsedResultSets: ParsedResultSets;
 }
 
+export interface UserSettings {
+  /** Whether to display links to the dataflow models that generated particular nodes in a flow path. */
+  shouldShowProvenance: boolean;
+}
+
+export const DEFAULT_USER_SETTINGS: UserSettings = {
+  shouldShowProvenance: false,
+};
+
+/** Message indicating that the user's configuration settings have changed. */
+interface SetUserSettingsMsg {
+  t: "setUserSettings";
+  userSettings: UserSettings;
+}
+
 /**
  * Message indicating that the results view should display interpreted
  * results.
@@ -188,6 +206,7 @@ interface UntoggleShowProblemsMsg {
 export type IntoResultsViewMsg =
   | ResultsUpdatingMsg
   | SetStateMsg
+  | SetUserSettingsMsg
   | ShowInterpretedPageMsg
   | NavigateMsg
   | UntoggleShowProblemsMsg;
@@ -205,13 +224,15 @@ export type FromResultsViewMsg =
   | OpenFileMsg;
 
 /**
- * Message from the results view to open a database source
+ * Message from the results view to open a source
  * file at the provided location.
  */
 interface ViewSourceFileMsg {
   t: "viewSourceFile";
   loc: UrlValueResolvable;
-  databaseUri: string;
+  /** URI of the database whose source archive contains the file, or `undefined` to open a file from
+   * the local disk. The latter case is used for opening links to data extension model files. */
+  databaseUri: string | undefined;
 }
 
 /**
@@ -338,7 +359,8 @@ interface ChangeCompareMessage {
 
 export type ToCompareViewMessage =
   | SetComparisonQueryInfoMessage
-  | SetComparisonsMessage;
+  | SetComparisonsMessage
+  | SetUserSettingsMsg;
 
 /**
  * Message to the compare view that sets the metadata of the compared queries.
@@ -527,9 +549,10 @@ interface SetMethodsMessage {
   methods: Method[];
 }
 
-interface SetModeledMethodsMessage {
-  t: "setModeledMethods";
+interface SetModeledAndModifiedMethodsMessage {
+  t: "setModeledAndModifiedMethods";
   methods: Record<string, ModeledMethod[]>;
+  modifiedMethodSignatures: string[];
 }
 
 interface SetModifiedMethodsMessage {
@@ -542,6 +565,11 @@ interface SetInProgressMethodsMessage {
   methods: string[];
 }
 
+interface SetProcessedByAutoModelMethodsMessage {
+  t: "setProcessedByAutoModelMethods";
+  methods: string[];
+}
+
 interface SwitchModeMessage {
   t: "switchMode";
   mode: Mode;
@@ -584,6 +612,23 @@ interface StopGeneratingMethodsFromLlmMessage {
   packageName: string;
 }
 
+interface StartModelEvaluationMessage {
+  t: "startModelEvaluation";
+}
+
+interface StopModelEvaluationMessage {
+  t: "stopModelEvaluation";
+}
+
+interface OpenModelAlertsViewMessage {
+  t: "openModelAlertsView";
+}
+
+interface RevealInModelAlertsViewMessage {
+  t: "revealInModelAlertsView";
+  modeledMethod: ModeledMethod;
+}
+
 interface ModelDependencyMessage {
   t: "modelDependency";
 }
@@ -609,18 +654,36 @@ interface SetInProgressMessage {
   inProgress: boolean;
 }
 
+interface SetProcessedByAutoModelMessage {
+  t: "setProcessedByAutoModel";
+  processedByAutoModel: boolean;
+}
+
 interface RevealMethodMessage {
   t: "revealMethod";
   methodSignature: string;
 }
 
+interface SetAccessPathSuggestionsMessage {
+  t: "setAccessPathSuggestions";
+  accessPathSuggestions: AccessPathSuggestionOptions;
+}
+
+interface SetModelEvaluationRunMessage {
+  t: "setModelEvaluationRun";
+  run: ModelEvaluationRunState | undefined;
+}
+
 export type ToModelEditorMessage =
   | SetExtensionPackStateMessage
   | SetMethodsMessage
-  | SetModeledMethodsMessage
+  | SetModeledAndModifiedMethodsMessage
   | SetModifiedMethodsMessage
   | SetInProgressMethodsMessage
-  | RevealMethodMessage;
+  | SetProcessedByAutoModelMethodsMessage
+  | RevealMethodMessage
+  | SetAccessPathSuggestionsMessage
+  | SetModelEvaluationRunMessage;
 
 export type FromModelEditorMessage =
   | CommonFromViewMessages
@@ -635,11 +698,15 @@ export type FromModelEditorMessage =
   | StopGeneratingMethodsFromLlmMessage
   | ModelDependencyMessage
   | HideModeledMethodsMessage
-  | SetMultipleModeledMethodsMessage;
+  | SetMultipleModeledMethodsMessage
+  | StartModelEvaluationMessage
+  | StopModelEvaluationMessage
+  | OpenModelAlertsViewMessage
+  | RevealInModelAlertsViewMessage;
 
 interface RevealInEditorMessage {
   t: "revealInModelEditor";
-  method: Method;
+  method: MethodSignature;
 }
 
 interface StartModelingMessage {
@@ -657,29 +724,67 @@ interface SetMethodModelingPanelViewStateMessage {
   viewState: MethodModelingPanelViewState;
 }
 
-interface SetMethodMessage {
-  t: "setMethod";
-  method: Method | undefined;
-}
-
 interface SetMethodModifiedMessage {
   t: "setMethodModified";
   isModified: boolean;
 }
 
+interface SetNoMethodSelectedMessage {
+  t: "setNoMethodSelected";
+}
+
 interface SetSelectedMethodMessage {
   t: "setSelectedMethod";
   method: Method;
   modeledMethods: ModeledMethod[];
   isModified: boolean;
   isInProgress: boolean;
+  processedByAutoModel: boolean;
 }
 
 export type ToMethodModelingMessage =
   | SetMethodModelingPanelViewStateMessage
-  | SetMethodMessage
   | SetMultipleModeledMethodsMessage
   | SetMethodModifiedMessage
+  | SetNoMethodSelectedMessage
   | SetSelectedMethodMessage
   | SetInModelingModeMessage
-  | SetInProgressMessage;
+  | SetInProgressMessage
+  | SetProcessedByAutoModelMessage;
+
+interface SetModelAlertsViewStateMessage {
+  t: "setModelAlertsViewState";
+  viewState: ModelAlertsViewState;
+}
+
+interface OpenModelPackMessage {
+  t: "openModelPack";
+  path: string;
+}
+
+interface OpenActionsLogsMessage {
+  t: "openActionsLogs";
+  variantAnalysisId: number;
+}
+
+interface StopEvaluationRunMessage {
+  t: "stopEvaluationRun";
+}
+
+interface RevealModelMessage {
+  t: "revealModel";
+  modeledMethod: ModeledMethod;
+}
+
+export type ToModelAlertsMessage =
+  | SetModelAlertsViewStateMessage
+  | SetVariantAnalysisMessage
+  | SetRepoResultsMessage
+  | RevealModelMessage;
+
+export type FromModelAlertsMessage =
+  | CommonFromViewMessages
+  | OpenModelPackMessage
+  | OpenActionsLogsMessage
+  | StopEvaluationRunMessage
+  | RevealInEditorMessage;

extensions/ql-vscode/src/common/jsonl-reader.ts

@@ -10,9 +10,9 @@ import { readFile } from "fs-extra";
  * @param path The path to the file.
  * @param handler Callback to be invoked for each top-level JSON object in order.
  */
-export async function readJsonlFile(
+export async function readJsonlFile<T>(
   path: string,
-  handler: (value: any) => Promise<void>,
+  handler: (value: T) => Promise<void>,
 ): Promise<void> {
   const logSummary = await readFile(path, "utf-8");
 
@@ -20,7 +20,7 @@ export async function readJsonlFile(
   const jsonSummaryObjects: string[] = logSummary.split(/\r?\n\r?\n/g);
 
   for (const obj of jsonSummaryObjects) {
-    const jsonObj = JSON.parse(obj);
+    const jsonObj = JSON.parse(obj) as T;
     await handler(jsonObj);
   }
 }

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -1,7 +1,10 @@
-import { appendFile, ensureFile } from "fs-extra";
+import { ensureFile } from "fs-extra";
+import { open } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
 import { isAbsolute } from "path";
 import { getErrorMessage } from "../helpers-pure";
 import type { Logger, LogOptions } from "./logger";
+import type { Disposable } from "../disposable-object";
 
 /**
  * An implementation of {@link Logger} that sends the output both to another {@link Logger}
@@ -10,9 +13,10 @@ import type { Logger, LogOptions } from "./logger";
  * The first time a message is written, an additional banner is written to the underlying logger
  * pointing the user to the "side log" file.
  */
-export class TeeLogger implements Logger {
+export class TeeLogger implements Logger, Disposable {
   private emittedRedirectMessage = false;
   private error = false;
+  private fileHandle: FileHandle | undefined = undefined;
 
   public constructor(
     private readonly logger: Logger,
@@ -37,11 +41,15 @@ export class TeeLogger implements Logger {
 
     if (!this.error) {
       try {
-        const trailingNewline = options.trailingNewline ?? true;
-        await ensureFile(this.location);
+        if (!this.fileHandle) {
+          await ensureFile(this.location);
 
-        await appendFile(
-          this.location,
+          this.fileHandle = await open(this.location, "a");
+        }
+
+        const trailingNewline = options.trailingNewline ?? true;
+
+        await this.fileHandle.appendFile(
           message + (trailingNewline ? "\n" : ""),
           {
             encoding: "utf8",
@@ -50,6 +58,14 @@ export class TeeLogger implements Logger {
       } catch (e) {
         // Write an error message to the primary log, and stop trying to write to the side log.
         this.error = true;
+        try {
+          await this.fileHandle?.close();
+        } catch (e) {
+          void this.logger.log(
+            `Failed to close file handle: ${getErrorMessage(e)}`,
+          );
+        }
+        this.fileHandle = undefined;
         const errorMessage = getErrorMessage(e);
         await this.logger.log(
           `Error writing to additional log file: ${errorMessage}`,
@@ -65,4 +81,15 @@ export class TeeLogger implements Logger {
   show(preserveFocus?: boolean): void {
     this.logger.show(preserveFocus);
   }
+
+  dispose(): void {
+    try {
+      void this.fileHandle?.close();
+    } catch (e) {
+      void this.logger.log(
+        `Failed to close file handle: ${getErrorMessage(e)}`,
+      );
+    }
+    this.fileHandle = undefined;
+  }
 }

extensions/ql-vscode/src/common/logging/vscode/loggers.ts

@@ -11,7 +11,7 @@ export const extLogger = new OutputChannelLogger("CodeQL Extension Log");
 export const queryServerLogger = new OutputChannelLogger("CodeQL Query Server");
 
 // Logger for messages from the language server.
-export const ideServerLogger = new OutputChannelLogger(
+export const languageServerLogger = new OutputChannelLogger(
   "CodeQL Language Server",
 );
 

extensions/ql-vscode/src/common/memento.ts

@@ -40,5 +40,5 @@ export interface Memento {
    * @param key A string.
    * @param value A value. MUST not contain cyclic references.
    */
-  update(key: string, value: any): Thenable<void>;
+  update<T>(key: string, value: T | undefined): Thenable<void>;
 }

extensions/ql-vscode/src/common/model-pack-details.ts

@@ -0,0 +1,4 @@
+export interface ModelPackDetails {
+  name: string;
+  path: string;
+}

extensions/ql-vscode/src/common/ql.ts

@@ -1,4 +1,4 @@
-import { join } from "path";
+import { dirname, join, parse } from "path";
 import { pathExists } from "fs-extra";
 
 export const QLPACK_FILENAMES = ["qlpack.yml", "codeql-pack.yml"];
@@ -8,7 +8,13 @@ export const QLPACK_LOCK_FILENAMES = [
 ];
 export const FALLBACK_QLPACK_FILENAME = QLPACK_FILENAMES[0];
 
-export async function getQlPackPath(
+/**
+ * Gets the path to the QL pack file (a qlpack.yml or
+ * codeql-pack.yml).
+ * @param packRoot The root of the pack.
+ * @returns The path to the qlpack file, or undefined if it doesn't exist.
+ */
+export async function getQlPackFilePath(
   packRoot: string,
 ): Promise<string | undefined> {
   for (const filename of QLPACK_FILENAMES) {
@@ -21,3 +27,28 @@ export async function getQlPackPath(
 
   return undefined;
 }
+
+/**
+ * Recursively find the directory containing qlpack.yml or codeql-pack.yml. If
+ * no such directory is found, the directory containing the query file is returned.
+ * @param queryFile The query file to start from.
+ * @returns The path to the pack root or undefined if it doesn't exist.
+ */
+export async function findPackRoot(
+  queryFile: string,
+): Promise<string | undefined> {
+  let dir = dirname(queryFile);
+  while (!(await getQlPackFilePath(dir))) {
+    dir = dirname(dir);
+    if (isFileSystemRoot(dir)) {
+      return undefined;
+    }
+  }
+
+  return dir;
+}
+
+function isFileSystemRoot(dir: string): boolean {
+  const pathObj = parse(dir);
+  return pathObj.root === dir && pathObj.base === "";
+}

extensions/ql-vscode/src/common/query-metadata.ts

@@ -0,0 +1,17 @@
+export const SARIF_RESULTS_QUERY_KINDS = [
+  "problem",
+  "alert",
+  "path-problem",
+  "path-alert",
+];
+
+/**
+ * Returns whether this query kind supports producing SARIF results.
+ */
+export function isSarifResultsQueryKind(kind: string | undefined): boolean {
+  if (!kind) {
+    return false;
+  }
+
+  return SARIF_RESULTS_QUERY_KINDS.includes(kind);
+}

extensions/ql-vscode/src/common/readonly.ts

@@ -1,11 +1,12 @@
-export type DeepReadonly<T> = T extends Array<infer R>
-  ? DeepReadonlyArray<R>
-  : // eslint-disable-next-line @typescript-eslint/ban-types
-    T extends Function
-    ? T
-    : T extends object
-      ? DeepReadonlyObject<T>
-      : T;
+export type DeepReadonly<T> =
+  T extends Array<infer R>
+    ? DeepReadonlyArray<R>
+    : // eslint-disable-next-line @typescript-eslint/ban-types
+      T extends Function
+      ? T
+      : T extends object
+        ? DeepReadonlyObject<T>
+        : T;
 
 interface DeepReadonlyArray<T> extends ReadonlyArray<DeepReadonly<T>> {}
 

extensions/ql-vscode/src/common/sarif-parser.ts

@@ -1,18 +1,20 @@
-import type { Log, Tool } from "sarif";
+import type { Log } from "sarif";
 import { createReadStream } from "fs-extra";
 import { connectTo } from "stream-json/Assembler";
 import { getErrorMessage } from "./helpers-pure";
-import { withParser } from "stream-json/filters/Pick";
-
-const DUMMY_TOOL: Tool = { driver: { name: "" } };
+import { withParser } from "stream-json/filters/Ignore";
 
 export async function sarifParser(
   interpretedResultsPath: string,
 ): Promise<Log> {
   try {
-    // Parse the SARIF file into token streams, filtering out only the results array.
+    // Parse the SARIF file into token streams, filtering out some of the larger subtrees that we
+    // don't need.
     const pipeline = createReadStream(interpretedResultsPath).pipe(
-      withParser({ filter: "runs.0.results" }),
+      withParser({
+        // We don't need to run's `artifacts` property, nor the driver's `notifications` property.
+        filter: /^runs\.\d+\.(artifacts|tool\.driver\.notifications)/,
+      }),
     );
 
     // Creates JavaScript objects from the token stream
@@ -38,15 +40,17 @@ export async function sarifParser(
       });
 
       asm.on("done", (asm) => {
-        const log: Log = {
-          version: "2.1.0",
-          runs: [
-            {
-              tool: DUMMY_TOOL,
-              results: asm.current ?? [],
-            },
-          ],
-        };
+        const log = asm.current;
+
+        // Do some trivial validation. This isn't a full validation of the SARIF file, but it's at
+        // least enough to ensure that we're not trying to parse complete garbage later.
+        if (log.runs === undefined || log.runs.length < 1) {
+          reject(
+            new Error(
+              "Invalid SARIF file: expecting at least one run with result.",
+            ),
+          );
+        }
 
         resolve(log);
         alreadyDone = true;

extensions/ql-vscode/src/common/short-paths.ts

@@ -0,0 +1,117 @@
+import { platform } from "os";
+import { basename, dirname, join, normalize, resolve } from "path";
+import { lstat, readdir } from "fs/promises";
+import type { BaseLogger } from "./logging";
+
+/**
+ * Expands a path that potentially contains 8.3 short names (e.g. "C:\PROGRA~1" instead of "C:\Program Files").
+ *
+ * See https://en.wikipedia.org/wiki/8.3_filename if you're not familiar with Windows 8.3 short names.
+ *
+ * @param shortPath The path to expand.
+ * @returns A normalized, absolute path, with any short components expanded.
+ */
+export async function expandShortPaths(
+  shortPath: string,
+  logger: BaseLogger,
+): Promise<string> {
+  const absoluteShortPath = normalize(resolve(shortPath));
+  if (platform() !== "win32") {
+    // POSIX doesn't have short paths.
+    return absoluteShortPath;
+  }
+
+  void logger.log(`Expanding short paths in: ${absoluteShortPath}`);
+  // A quick check to see if there might be any short components.
+  // There might be a case where a short component doesn't contain a `~`, but if there is, I haven't
+  // found it.
+  // This may find long components that happen to have a '~', but that's OK.
+  if (absoluteShortPath.indexOf("~") < 0) {
+    // No short components to expand.
+    void logger.log(`Skipping due to no short components`);
+    return absoluteShortPath;
+  }
+
+  return await expandShortPathRecursive(absoluteShortPath, logger);
+}
+
+/**
+ * Expand a single short path component
+ * @param dir The absolute path of the directory containing the short path component.
+ * @param shortBase The shot path component to expand.
+ * @returns The expanded path component.
+ */
+async function expandShortPathComponent(
+  dir: string,
+  shortBase: string,
+  logger: BaseLogger,
+): Promise<string> {
+  void logger.log(`Expanding short path component: ${shortBase}`);
+
+  const fullPath = join(dir, shortBase);
+
+  // Use `lstat` instead of `stat` to avoid following symlinks.
+  const stats = await lstat(fullPath, { bigint: true });
+  if (stats.dev === BigInt(0) || stats.ino === BigInt(0)) {
+    // No inode info, so we won't be able to find this in the directory listing.
+    void logger.log(`No inode info available. Skipping.`);
+    return shortBase;
+  }
+  void logger.log(`dev/inode: ${stats.dev}/${stats.ino}`);
+
+  try {
+    // Enumerate the children of the parent directory, and try to find one with the same dev/inode.
+    const children = await readdir(dir);
+    for (const child of children) {
+      void logger.log(`considering child: ${child}`);
+      try {
+        const childStats = await lstat(join(dir, child), { bigint: true });
+        void logger.log(`child dev/inode: ${childStats.dev}/${childStats.ino}`);
+        if (childStats.dev === stats.dev && childStats.ino === stats.ino) {
+          // Found a match.
+          void logger.log(`Found a match: ${child}`);
+          return child;
+        }
+      } catch (e) {
+        // Can't read stats for the child, so skip it.
+        void logger.log(`Error reading stats for child: ${e}`);
+      }
+    }
+  } catch (e) {
+    // Can't read the directory, so we won't be able to find this in the directory listing.
+    void logger.log(`Error reading directory: ${e}`);
+    return shortBase;
+  }
+
+  void logger.log(`No match found. Returning original.`);
+  return shortBase;
+}
+
+/**
+ * Expand the short path components in a path, including those in ancestor directories.
+ * @param shortPath The path to expand.
+ * @returns The expanded path.
+ */
+async function expandShortPathRecursive(
+  shortPath: string,
+  logger: BaseLogger,
+): Promise<string> {
+  const shortBase = basename(shortPath);
+  if (shortBase.length === 0) {
+    // We've reached the root.
+    return shortPath;
+  }
+
+  const dir = await expandShortPathRecursive(dirname(shortPath), logger);
+  void logger.log(`dir: ${dir}`);
+  void logger.log(`base: ${shortBase}`);
+  if (shortBase.indexOf("~") < 0) {
+    // This component doesn't have a short name, so just append it to the (long) parent.
+    void logger.log(`Component is not a short name`);
+    return join(dir, shortBase);
+  }
+
+  // This component looks like it has a short name, so try to expand it.
+  const longBase = await expandShortPathComponent(dir, shortBase, logger);
+  return join(dir, longBase);
+}

extensions/ql-vscode/src/common/text-utils.ts

@@ -17,7 +17,7 @@ export function convertNonPrintableChars(label: string | undefined) {
      * If the label contains certain non-printable characters, loop through each
      * character and replace it with the cooresponding unicode control label.
      */
-    const convertedLabelArray: any[] = [];
+    const convertedLabelArray: string[] = [];
     for (let i = 0; i < label.length; i++) {
       const labelCheck = label.codePointAt(i)!;
       if (labelCheck <= CONTROL_CODE) {

extensions/ql-vscode/src/common/vscode/abstract-webview-view-provider.ts

@@ -1,9 +1,4 @@
-import type {
-  CancellationToken,
-  WebviewView,
-  WebviewViewProvider,
-  WebviewViewResolveContext,
-} from "vscode";
+import type { WebviewView, WebviewViewProvider } from "vscode";
 import { Uri } from "vscode";
 import type { WebviewKind, WebviewMessage } from "./webview-html";
 import { getHtmlForWebview } from "./webview-html";
@@ -28,11 +23,7 @@ export abstract class AbstractWebviewViewProvider<
    * This is called when a view first becomes visible. This may happen when the view is
    * first loaded or when the user hides and then shows a view again.
    */
-  public resolveWebviewView(
-    webviewView: WebviewView,
-    _context: WebviewViewResolveContext,
-    _token: CancellationToken,
-  ) {
+  public resolveWebviewView(webviewView: WebviewView) {
     webviewView.webview.options = {
       enableScripts: true,
       localResourceRoots: [Uri.file(this.app.extensionPath)],

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -13,6 +13,8 @@ import { tmpDir } from "../../tmp-dir";
 import type { WebviewMessage, WebviewKind } from "./webview-html";
 import { getHtmlForWebview } from "./webview-html";
 import type { DeepReadonly } from "../readonly";
+import { runWithErrorHandling } from "./error-handling";
+import { telemetryListener } from "./telemetry";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -117,7 +119,12 @@ export abstract class AbstractWebview<
     );
     this.push(
       panel.webview.onDidReceiveMessage(
-        async (e) => this.onMessage(e),
+        async (e) =>
+          runWithErrorHandling(
+            () => this.onMessage(e),
+            this.app.logger,
+            telemetryListener,
+          ),
         undefined,
       ),
     );

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -26,6 +26,8 @@ import {
 // All path operations in this file must be on paths *within* the zip
 // archive.
 import { posix } from "path";
+import { DatabaseEventKind } from "../../databases/local-databases/database-events";
+import type { DatabaseManager } from "../../databases/local-databases/database-manager";
 
 const path = posix;
 
@@ -242,6 +244,10 @@ export class ArchiveFileSystemProvider implements FileSystemProvider {
 
   root = new Directory("");
 
+  flushCache(zipPath: string) {
+    this.archives.delete(zipPath);
+  }
+
   // metadata
 
   async stat(uri: Uri): Promise<FileStat> {
@@ -355,15 +361,35 @@ export class ArchiveFileSystemProvider implements FileSystemProvider {
  */
 export const zipArchiveScheme = "codeql-zip-archive";
 
-export function activate(ctx: ExtensionContext) {
+export function activate(ctx: ExtensionContext, dbm?: DatabaseManager) {
+  const afsp = new ArchiveFileSystemProvider();
+  if (dbm) {
+    ctx.subscriptions.push(
+      dbm.onDidChangeDatabaseItem(async ({ kind, item: db }) => {
+        if (kind === DatabaseEventKind.Remove) {
+          if (db?.sourceArchive) {
+            afsp.flushCache(db.sourceArchive.fsPath);
+          }
+        }
+      }),
+    );
+  }
+
   ctx.subscriptions.push(
-    workspace.registerFileSystemProvider(
-      zipArchiveScheme,
-      new ArchiveFileSystemProvider(),
-      {
-        isCaseSensitive: true,
-        isReadonly: true,
-      },
-    ),
+    // When a file system archive is removed from the workspace, we should
+    // also remove it from our cache.
+    workspace.onDidChangeWorkspaceFolders((event) => {
+      for (const removed of event.removed) {
+        const zipPath = removed.uri.fsPath;
+        afsp.flushCache(zipPath);
+      }
+    }),
+  );
+
+  ctx.subscriptions.push(
+    workspace.registerFileSystemProvider(zipArchiveScheme, afsp, {
+      isCaseSensitive: true,
+      isReadonly: true,
+    }),
   );
 }

extensions/ql-vscode/src/common/vscode/authentication.ts

@@ -2,8 +2,8 @@ import { authentication } from "vscode";
 import type { Octokit } from "@octokit/rest";
 import type { Credentials } from "../authentication";
 import { AppOctokit } from "../octokit";
-
-export const GITHUB_AUTH_PROVIDER_ID = "github";
+import { hasGhecDrUri } from "../../config";
+import { getOctokitBaseUrl } from "./octokit";
 
 // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist,
 // and 'read:packages' for reading private CodeQL packages.
@@ -16,30 +16,24 @@ const SCOPES = ["repo", "gist", "read:packages"];
  */
 export class VSCodeCredentials implements Credentials {
   /**
-   * A specific octokit to return, otherwise a new authenticated octokit will be created when needed.
-   */
-  private octokit: Octokit | undefined;
-
-  /**
-   * Creates or returns an instance of Octokit.
+   * Creates or returns an instance of Octokit. The returned instance should
+   * not be stored and reused, as it may become out-of-date with the current
+   * authentication session.
    *
    * @returns An instance of Octokit.
    */
   async getOctokit(): Promise<Octokit> {
-    if (this.octokit) {
-      return this.octokit;
-    }
-
     const accessToken = await this.getAccessToken();
 
     return new AppOctokit({
       auth: accessToken,
+      baseUrl: getOctokitBaseUrl(),
     });
   }
 
   async getAccessToken(): Promise<string> {
     const session = await authentication.getSession(
-      GITHUB_AUTH_PROVIDER_ID,
+      this.authProviderId,
       SCOPES,
       { createIfNone: true },
     );
@@ -49,11 +43,18 @@ export class VSCodeCredentials implements Credentials {
 
   async getExistingAccessToken(): Promise<string | undefined> {
     const session = await authentication.getSession(
-      GITHUB_AUTH_PROVIDER_ID,
+      this.authProviderId,
       SCOPES,
       { createIfNone: false },
     );
 
     return session?.accessToken;
   }
+
+  public get authProviderId(): string {
+    if (hasGhecDrUri()) {
+      return "github-enterprise";
+    }
+    return "github";
+  }
 }

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -3,16 +3,10 @@ import { commands } from "vscode";
 import type { CommandFunction } from "../../packages/commands";
 import { CommandManager } from "../../packages/commands";
 import type { NotificationLogger } from "../logging";
-import {
-  showAndLogWarningMessage,
-  showAndLogExceptionWithTelemetry,
-} from "../logging";
 import { extLogger } from "../logging/vscode";
-import { asError, getErrorMessage } from "../../common/helpers-pure";
-import { redactableError } from "../../common/errors";
-import { UserCancellationException } from "./progress";
 import { telemetryListener } from "./telemetry";
 import type { AppTelemetry } from "../telemetry";
+import { runWithErrorHandling } from "./error-handling";
 
 /**
  * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -38,46 +32,17 @@ export function createVSCodeCommandManager<
  * @param logger The logger to use for error reporting.
  * @param telemetry The telemetry listener to use for error reporting.
  */
-export function registerCommandWithErrorHandling(
+export function registerCommandWithErrorHandling<
+  T extends (...args: unknown[]) => Promise<unknown>,
+>(
   commandId: string,
-  task: (...args: any[]) => Promise<any>,
+  task: T,
   logger: NotificationLogger = extLogger,
   telemetry: AppTelemetry | undefined = telemetryListener,
 ): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const startTime = Date.now();
-    let error: Error | undefined;
-
-    try {
-      return await task(...args);
-    } catch (e) {
-      error = asError(e);
-      const errorMessage = redactableError(error)`${
-        getErrorMessage(e) || e
-      } (${commandId})`;
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          void logger.log(errorMessage.fullMessage);
-        } else {
-          void showAndLogWarningMessage(logger, errorMessage.fullMessage);
-        }
-      } else {
-        // Include the full stack in the error log only.
-        const fullMessage = errorMessage.fullMessageWithStack;
-        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      }
-      return undefined;
-    } finally {
-      const executionTime = Date.now() - startTime;
-      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
-    }
-  });
+  return commands.registerCommand(commandId, async (...args: Parameters<T>) =>
+    runWithErrorHandling(task, logger, telemetry, commandId, ...args),
+  );
 }
 
 /**

extensions/ql-vscode/src/common/vscode/error-handling.ts

@@ -0,0 +1,81 @@
+import {
+  showAndLogWarningMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../logging";
+import type { NotificationLogger } from "../logging";
+import { extLogger } from "../logging/vscode";
+import type { AppTelemetry } from "../telemetry";
+import { telemetryListener } from "./telemetry";
+import { asError, getErrorMessage } from "../helpers-pure";
+import { redactableError } from "../errors";
+import { UserCancellationException } from "./progress";
+import { CliError } from "../../codeql-cli/cli-errors";
+import { EOL } from "os";
+
+/**
+ * Executes a task with error handling. It provides a uniform way to handle errors.
+ *
+ * @template T - A function type that takes an unknown number of arguments and returns a Promise.
+ * @param {T} task - The task to be executed.
+ * @param {NotificationLogger} [logger=extLogger] - The logger to use for error reporting.
+ * @param {AppTelemetry | undefined} [telemetry=telemetryListener] - The telemetry listener to use for error reporting.
+ * @param {string} [commandId] - The optional command id associated with the task.
+ * @param {...unknown} args - The arguments to be passed to the task.
+ * @returns {Promise<unknown>} The result of the task, or undefined if an error occurred.
+ * @throws {Error} If an error occurs during the execution of the task.
+ */
+export async function runWithErrorHandling<
+  T extends (...args: unknown[]) => Promise<unknown>,
+>(
+  task: T,
+  logger: NotificationLogger = extLogger,
+  telemetry: AppTelemetry | undefined = telemetryListener,
+  commandId?: string,
+  ...args: unknown[]
+): Promise<unknown> {
+  const startTime = Date.now();
+  let error: Error | undefined;
+
+  try {
+    return await task(...args);
+  } catch (e) {
+    error = asError(e);
+    const errorMessage = redactableError(error)`${
+      getErrorMessage(e) || e
+    }${commandId ? ` (${commandId})` : ""}`;
+
+    const extraTelemetryProperties = commandId
+      ? { command: commandId }
+      : undefined;
+
+    if (e instanceof UserCancellationException) {
+      // User has cancelled this action manually
+      if (e.silent) {
+        void logger.log(errorMessage.fullMessage);
+      } else {
+        void showAndLogWarningMessage(logger, errorMessage.fullMessage);
+      }
+    } else if (e instanceof CliError) {
+      const fullMessage = `${e.commandDescription} failed with args:${EOL}    ${e.commandArgs.join(" ")}${EOL}${
+        e.stderr ?? e.cause
+      }`;
+      void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
+        fullMessage,
+        extraTelemetryProperties,
+      });
+    } else {
+      // Include the full stack in the error log only.
+      const fullMessage = errorMessage.fullMessageWithStack;
+      void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
+        fullMessage,
+        extraTelemetryProperties,
+      });
+    }
+    return undefined;
+  } finally {
+    if (commandId) {
+      const executionTime = Date.now() - startTime;
+      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/multi-cancellation-token.ts

@@ -16,7 +16,7 @@ export class MultiCancellationToken implements CancellationToken {
     return this.tokens.some((t) => t.isCancellationRequested);
   }
 
-  onCancellationRequested<T>(listener: (e: T) => any): Disposable {
+  onCancellationRequested<T>(listener: (e: T) => void): Disposable {
     return new DisposableObject(
       ...this.tokens.map((t) => t.onCancellationRequested(listener)),
     );

extensions/ql-vscode/src/common/vscode/multi-file-system-watcher.ts

@@ -18,15 +18,11 @@ class WatcherCollection extends DisposableObject {
    *   deleted.
    * @param thisArgs The `this` argument for the event listener.
    */
-  public addWatcher(
-    pattern: GlobPattern,
-    listener: (e: Uri) => any,
-    thisArgs: any,
-  ): void {
+  public addWatcher(pattern: GlobPattern, listener: (e: Uri) => void): void {
     const watcher = workspace.createFileSystemWatcher(pattern);
-    this.push(watcher.onDidCreate(listener, thisArgs));
-    this.push(watcher.onDidChange(listener, thisArgs));
-    this.push(watcher.onDidDelete(listener, thisArgs));
+    this.push(watcher.onDidCreate(listener));
+    this.push(watcher.onDidChange(listener));
+    this.push(watcher.onDidDelete(listener));
   }
 }
 
@@ -54,7 +50,7 @@ export class MultiFileSystemWatcher extends DisposableObject {
    * @param pattern The pattern to watch.
    */
   public addWatch(pattern: GlobPattern): void {
-    this.watchers.addWatcher(pattern, this.handleDidChange, this);
+    this.watchers.addWatcher(pattern, this.handleDidChange.bind(this));
   }
 
   /**

extensions/ql-vscode/src/common/vscode/octokit.ts

@@ -0,0 +1,15 @@
+import { getGitHubInstanceApiUrl } from "../../config";
+
+/**
+ * Returns the Octokit base URL to use based on the GitHub instance URL.
+ *
+ * This is necessary because the Octokit base URL should not have a trailing
+ * slash, but this is included by default in a URL.
+ */
+export function getOctokitBaseUrl(): string {
+  let apiUrl = getGitHubInstanceApiUrl().toString();
+  if (apiUrl.endsWith("/")) {
+    apiUrl = apiUrl.slice(0, -1);
+  }
+  return apiUrl;
+}

extensions/ql-vscode/src/common/vscode/progress.ts

@@ -33,6 +33,14 @@ export interface ProgressUpdate {
   message: string;
 }
 
+export function progressUpdate(
+  step: number,
+  maxStep: number,
+  message: string,
+): ProgressUpdate {
+  return { step, maxStep, message };
+}
+
 export type ProgressCallback = (p: ProgressUpdate) => void;
 
 // Make certain properties within a type optional
@@ -48,7 +56,7 @@ type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
  * denote some progress being achieved on this task.
  * @param token a cancellation token
  */
-export type ProgressTask<R> = (
+type ProgressTask<R> = (
   progress: ProgressCallback,
   token: CancellationToken,
 ) => Thenable<R>;
@@ -85,29 +93,6 @@ export function withProgress<R>(
   );
 }
 
-export interface ProgressContext {
-  progress: ProgressCallback;
-  token: CancellationToken;
-}
-
-/**
- * Like `withProgress()`, except that the caller is not required to provide a progress context. If
- * the caller does provide one, any long-running operations performed by `task` will use the
- * supplied progress context. Otherwise, this function wraps `task` in a new progress context with
- * the supplied options.
- */
-export function withInheritedProgress<R>(
-  parent: ProgressContext | undefined,
-  task: ProgressTask<R>,
-  options: ProgressOptions,
-): Thenable<R> {
-  if (parent !== undefined) {
-    return task(parent.progress, parent.token);
-  } else {
-    return withProgress(task, options);
-  }
-}
-
 /**
  * Displays a progress monitor that indicates how much progess has been made
  * reading from a stream.

extensions/ql-vscode/src/common/vscode/selection-commands.ts

@@ -3,14 +3,10 @@ import type {
   TreeViewContextMultiSelectionCommandFunction,
   TreeViewContextSingleSelectionCommandFunction,
 } from "../commands";
+import type { NotArray } from "../helpers-pure";
 import type { NotificationLogger } from "../logging";
 import { showAndLogErrorMessage } from "../logging";
 
-// A hack to match types that are not an array, which is useful to help avoid
-// misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
-// as DatabaseItem[] instead of DatabaseItem.
-type NotArray = object & { length?: never };
-
 // A way to get the type system to help assert that one type is a supertype of another.
 type CreateSupertypeOf<Super, Sub extends Super> = Sub;
 

extensions/ql-vscode/src/common/vscode/telemetry.ts

@@ -20,6 +20,7 @@ import { showBinaryChoiceWithUrlDialog } from "./dialog";
 import type { RedactableError } from "../errors";
 import type { SemVer } from "semver";
 import type { AppTelemetry } from "../telemetry";
+import type { EnvelopeTelemetry } from "applicationinsights/out/Declarations/Contracts";
 
 // Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
 const key = "REPLACE-APP-INSIGHTS-KEY";
@@ -128,12 +129,13 @@ export class ExtensionTelemetryListener
     );
     this.push(this.reporter);
 
-    const client = (this.reporter as any).appInsightsClient as TelemetryClient;
+    // The appInsightsClient field is private but we want to access it anyway
+    const client = this.reporter["appInsightsClient"] as TelemetryClient;
     if (client) {
       // add a telemetry processor to delete unwanted properties
-      client.addTelemetryProcessor((envelope: any) => {
+      client.addTelemetryProcessor((envelope: EnvelopeTelemetry) => {
         tagsToRemove.forEach((tag) => delete envelope.tags[tag]);
-        const baseDataProperties = (envelope.data as any)?.baseData?.properties;
+        const baseDataProperties = envelope.data.baseData?.properties;
         if (baseDataProperties) {
           baseDataPropertiesToRemove.forEach(
             (prop) => delete baseDataProperties[prop],
@@ -212,6 +214,22 @@ export class ExtensionTelemetryListener
     this.reporter.sendTelemetryErrorEvent("error", properties, {});
   }
 
+  sendConfigInformation(config: Record<string, string>): void {
+    if (!this.reporter) {
+      return;
+    }
+
+    this.reporter.sendTelemetryEvent(
+      "config",
+      {
+        ...config,
+        isCanary: isCanary().toString(),
+        cliVersion: this.cliVersionStr,
+      },
+      {},
+    );
+  }
+
   /**
    * Displays a popup asking the user if they want to enable telemetry
    * for this extension.
@@ -277,7 +295,7 @@ export class ExtensionTelemetryListener
 export let telemetryListener: ExtensionTelemetryListener | undefined;
 
 export async function initializeTelemetry(
-  extension: Extension<any>,
+  extension: Extension<unknown>,
   ctx: ExtensionContext,
 ): Promise<ExtensionTelemetryListener> {
   if (telemetryListener !== undefined) {

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -10,7 +10,8 @@ export type WebviewKind =
   | "variant-analysis"
   | "data-flow-paths"
   | "model-editor"
-  | "method-modeling";
+  | "method-modeling"
+  | "model-alerts";
 
 export interface WebviewMessage {
   t: string;

extensions/ql-vscode/src/compare/compare-view.ts

@@ -33,6 +33,7 @@ import {
   getResultSetNames,
 } from "./result-set-names";
 import { compareInterpretedResults } from "./interpreted-results";
+import { isCanary } from "../config";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
@@ -116,6 +117,13 @@ export class CompareView extends AbstractWebview<
     panel.reveal(undefined, true);
     await this.waitForPanelLoaded();
 
+    await this.postMessage({
+      t: "setUserSettings",
+      userSettings: {
+        shouldShowProvenance: isCanary(),
+      },
+    });
+
     await this.postMessage({
       t: "setComparisonQueryInfo",
       stats: {

extensions/ql-vscode/src/compare/sarif-diff.ts

@@ -1,4 +1,80 @@
-import type { Result } from "sarif";
+import type { Location, Result } from "sarif";
+
+function toCanonicalLocation(location: Location): Location {
+  if (location.physicalLocation?.artifactLocation?.index !== undefined) {
+    const canonicalLocation = {
+      ...location,
+    };
+
+    canonicalLocation.physicalLocation = {
+      ...canonicalLocation.physicalLocation,
+    };
+
+    canonicalLocation.physicalLocation.artifactLocation = {
+      ...canonicalLocation.physicalLocation.artifactLocation,
+    };
+
+    // The index is dependent on the result of the SARIF file and usually doesn't really tell
+    // us anything useful, so we remove it from the comparison.
+    delete canonicalLocation.physicalLocation.artifactLocation.index;
+
+    return canonicalLocation;
+  }
+
+  // Don't create a new object if we don't need to
+  return location;
+}
+
+function toCanonicalResult(result: Result): Result {
+  const canonicalResult = {
+    ...result,
+  };
+
+  if (canonicalResult.locations) {
+    canonicalResult.locations =
+      canonicalResult.locations.map(toCanonicalLocation);
+  }
+
+  if (canonicalResult.relatedLocations) {
+    canonicalResult.relatedLocations =
+      canonicalResult.relatedLocations.map(toCanonicalLocation);
+  }
+
+  if (canonicalResult.codeFlows) {
+    canonicalResult.codeFlows = canonicalResult.codeFlows.map((codeFlow) => {
+      if (codeFlow.threadFlows) {
+        return {
+          ...codeFlow,
+          threadFlows: codeFlow.threadFlows.map((threadFlow) => {
+            if (threadFlow.locations) {
+              return {
+                ...threadFlow,
+                locations: threadFlow.locations.map((threadFlowLocation) => {
+                  if (threadFlowLocation.location) {
+                    return {
+                      ...threadFlowLocation,
+                      location: toCanonicalLocation(
+                        threadFlowLocation.location,
+                      ),
+                    };
+                  }
+
+                  return threadFlowLocation;
+                }),
+              };
+            }
+
+            return threadFlow;
+          }),
+        };
+      }
+
+      return codeFlow;
+    });
+  }
+
+  return canonicalResult;
+}
 
 /**
  * Compare the alerts of two queries. Use deep equality to determine if
@@ -25,9 +101,12 @@ export function sarifDiff(fromResults: Result[], toResults: Result[]) {
     throw new Error("CodeQL Compare: Target query has no results.");
   }
 
+  const canonicalFromResults = fromResults.map(toCanonicalResult);
+  const canonicalToResults = toResults.map(toCanonicalResult);
+
   const results = {
-    from: arrayDiff(fromResults, toResults),
-    to: arrayDiff(toResults, fromResults),
+    from: arrayDiff(canonicalFromResults, canonicalToResults),
+    to: arrayDiff(canonicalToResults, canonicalFromResults),
   };
 
   if (

extensions/ql-vscode/src/config.ts

@@ -4,7 +4,7 @@ import type {
   ConfigurationScope,
   Event,
 } from "vscode";
-import { ConfigurationTarget, EventEmitter, workspace } from "vscode";
+import { ConfigurationTarget, EventEmitter, workspace, Uri } from "vscode";
 import type { DistributionManager } from "./codeql-cli/distribution";
 import { extLogger } from "./common/logging/vscode";
 import { ONE_DAY_IN_MS } from "./common/time";
@@ -13,6 +13,8 @@ import {
   FilterKey,
   SortKey,
 } from "./variant-analysis/shared/variant-analysis-filter-sort";
+import { substituteConfigVariables } from "./common/config-template";
+import { getErrorMessage } from "./common/helpers-pure";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -68,6 +70,95 @@ export const VSCODE_SAVE_BEFORE_START_SETTING = new Setting(
   VSCODE_DEBUG_SETTING,
 );
 
+const VSCODE_GITHUB_ENTERPRISE_SETTING = new Setting(
+  "github-enterprise",
+  undefined,
+);
+export const VSCODE_GITHUB_ENTERPRISE_URI_SETTING = new Setting(
+  "uri",
+  VSCODE_GITHUB_ENTERPRISE_SETTING,
+);
+
+/**
+ * Get the value of the `github-enterprise.uri` setting, parsed as a URI.
+ * If the value is not set or cannot be parsed, return `undefined`.
+ */
+export function getEnterpriseUri(): Uri | undefined {
+  const config = VSCODE_GITHUB_ENTERPRISE_URI_SETTING.getValue<string>();
+  if (config) {
+    try {
+      let uri = Uri.parse(config, true);
+      if (uri.scheme === "http") {
+        uri = uri.with({ scheme: "https" });
+      }
+      return uri;
+    } catch (e) {
+      void extLogger.log(
+        `Failed to parse the GitHub Enterprise URI: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Is the GitHub Enterprise URI set?
+ */
+export function hasEnterpriseUri(): boolean {
+  return getEnterpriseUri() !== undefined;
+}
+
+/**
+ * Does the uri look like GHEC-DR?
+ */
+function isGhecDrUri(uri: Uri | undefined): boolean {
+  return uri !== undefined && uri.authority.toLowerCase().endsWith(".ghe.com");
+}
+
+/**
+ * Is the GitHub Enterprise URI set to something that looks like GHEC-DR?
+ */
+export function hasGhecDrUri(): boolean {
+  const uri = getEnterpriseUri();
+  return isGhecDrUri(uri);
+}
+
+/**
+ * The URI for GitHub.com.
+ */
+export const GITHUB_URL = new URL("https://github.com");
+export const GITHUB_API_URL = new URL("https://api.github.com");
+
+/**
+ * If the GitHub Enterprise URI is set to something that looks like GHEC-DR, return it.
+ */
+export function getGhecDrUri(): Uri | undefined {
+  const uri = getEnterpriseUri();
+  if (isGhecDrUri(uri)) {
+    return uri;
+  } else {
+    return undefined;
+  }
+}
+
+export function getGitHubInstanceUrl(): URL {
+  const ghecDrUri = getGhecDrUri();
+  if (ghecDrUri) {
+    return new URL(ghecDrUri.toString());
+  }
+  return GITHUB_URL;
+}
+
+export function getGitHubInstanceApiUrl(): URL {
+  const ghecDrUri = getGhecDrUri();
+  if (ghecDrUri) {
+    const url = new URL(ghecDrUri.toString());
+    url.hostname = `api.${url.hostname}`;
+    return url;
+  }
+  return GITHUB_API_URL;
+}
+
 const ROOT_SETTING = new Setting("codeQL");
 
 // Telemetry configuration
@@ -93,6 +184,10 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting(
   "personalAccessToken",
   DISTRIBUTION_SETTING,
 );
+const CLI_DOWNLOAD_TIMEOUT_SETTING = new Setting(
+  "downloadTimeout",
+  DISTRIBUTION_SETTING,
+);
 const CLI_CHANNEL_SETTING = new Setting("channel", DISTRIBUTION_SETTING);
 
 // Query History configuration
@@ -108,6 +203,7 @@ const DISTRIBUTION_CHANGE_SETTINGS = [
   CUSTOM_CODEQL_PATH_SETTING,
   INCLUDE_PRERELEASE_SETTING,
   PERSONAL_ACCESS_TOKEN_SETTING,
+  CLI_CHANNEL_SETTING,
 ];
 
 export type CLIChannel = "stable" | "nightly";
@@ -117,6 +213,7 @@ export interface DistributionConfig {
   updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
+  downloadTimeout: number;
   channel: CLIChannel;
   onDidChangeConfiguration?: Event<void>;
 }
@@ -271,6 +368,10 @@ export class DistributionConfigListener
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
+  public get downloadTimeout(): number {
+    return CLI_DOWNLOAD_TIMEOUT_SETTING.getValue() || 10;
+  }
+
   public async updateCustomCodeQlPath(newPath: string | undefined) {
     await CUSTOM_CODEQL_PATH_SETTING.updateValue(
       newPath,
@@ -511,6 +612,12 @@ export async function setRemoteControllerRepo(repo: string | undefined) {
 
 export interface VariantAnalysisConfig {
   controllerRepo: string | undefined;
+  showSystemDefinedRepositoryLists: boolean;
+  /**
+   * This uses a URL instead of a URI because the URL class is available in
+   * unit tests and is fully browser-compatible.
+   */
+  githubUrl: URL;
   onDidChangeConfiguration?: Event<void>;
 }
 
@@ -520,7 +627,7 @@ export class VariantAnalysisConfigListener
 {
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(
-      [VARIANT_ANALYSIS_SETTING],
+      [VARIANT_ANALYSIS_SETTING, VSCODE_GITHUB_ENTERPRISE_URI_SETTING],
       e,
     );
   }
@@ -528,6 +635,14 @@ export class VariantAnalysisConfigListener
   public get controllerRepo(): string | undefined {
     return getRemoteControllerRepo();
   }
+
+  public get showSystemDefinedRepositoryLists(): boolean {
+    return !hasEnterpriseUri();
+  }
+
+  public get githubUrl(): URL {
+    return getGitHubInstanceUrl();
+  }
 }
 
 const VARIANT_ANALYSIS_FILTER_RESULTS = new Setting(
@@ -643,8 +758,16 @@ const DEPRECATED_ALLOW_HTTP_SETTING = new Setting(
 
 const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING);
 
+const DOWNLOAD_TIMEOUT_SETTING = new Setting(
+  "downloadTimeout",
+  ADDING_DATABASES_SETTING,
+);
 const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING);
 
+export function downloadTimeout(): number {
+  return DOWNLOAD_TIMEOUT_SETTING.getValue<number>() || 10;
+}
+
 export function allowHttp(): boolean {
   return (
     ALLOW_HTTP_SETTING.getValue<boolean>() ||
@@ -653,7 +776,7 @@ export function allowHttp(): boolean {
   );
 }
 
-const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting(
+export const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting(
   "addDatabaseSourceToWorkspace",
   ADDING_DATABASES_SETTING,
 );
@@ -714,19 +837,33 @@ const LLM_GENERATION_DEV_ENDPOINT = new Setting(
   "llmGenerationDevEndpoint",
   MODEL_SETTING,
 );
-const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
-const ENABLE_RUBY = new Setting("enableRuby", MODEL_SETTING);
+const MODEL_EVALUATION = new Setting("evaluation", MODEL_SETTING);
+const MODEL_PACK_LOCATION = new Setting("packLocation", MODEL_SETTING);
+const MODEL_PACK_NAME = new Setting("packName", MODEL_SETTING);
+
+export type ModelConfigPackVariables = {
+  database: string;
+  owner: string;
+  name: string;
+  language: string;
+};
 
 export interface ModelConfig {
   flowGeneration: boolean;
   llmGeneration: boolean;
-  getExtensionsDirectory(languageId: string): string | undefined;
-  enableRuby: boolean;
+  getPackLocation(
+    languageId: string,
+    variables: ModelConfigPackVariables,
+  ): string;
+  getPackName(languageId: string, variables: ModelConfigPackVariables): string;
 }
 
 export class ModelConfigListener extends ConfigListener implements ModelConfig {
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
-    this.handleDidChangeConfigurationForRelevantSettings([MODEL_SETTING], e);
+    this.handleDidChangeConfigurationForRelevantSettings(
+      [MODEL_SETTING, VSCODE_GITHUB_ENTERPRISE_URI_SETTING],
+      e,
+    );
   }
 
   public get flowGeneration(): boolean {
@@ -734,7 +871,7 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
   }
 
   public get llmGeneration(): boolean {
-    return !!LLM_GENERATION.getValue<boolean>();
+    return !!LLM_GENERATION.getValue<boolean>() && !hasEnterpriseUri();
   }
 
   /**
@@ -753,14 +890,32 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return LLM_GENERATION_DEV_ENDPOINT.getValue<string | undefined>();
   }
 
-  public getExtensionsDirectory(languageId: string): string | undefined {
-    return EXTENSIONS_DIRECTORY.getValue<string>({
-      languageId,
-    });
+  public get modelEvaluation(): boolean {
+    return !!MODEL_EVALUATION.getValue<boolean>();
   }
 
-  public get enableRuby(): boolean {
-    return !!ENABLE_RUBY.getValue<boolean>();
+  public getPackLocation(
+    languageId: string,
+    variables: ModelConfigPackVariables,
+  ): string {
+    return substituteConfigVariables(
+      MODEL_PACK_LOCATION.getValue<string>({
+        languageId,
+      }),
+      variables,
+    );
+  }
+
+  public getPackName(
+    languageId: string,
+    variables: ModelConfigPackVariables,
+  ): string {
+    return substituteConfigVariables(
+      MODEL_PACK_NAME.getValue<string>({
+        languageId,
+      }),
+      variables,
+    );
   }
 }
 

extensions/ql-vscode/src/databases/README.md

@@ -1,3 +1,3 @@
 ### Databases
 
-This folder contains tests for the variant analysis repository panel.
+This folder contains code for the (local) databases panel and the variant analysis repositories panel.

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -6,6 +6,8 @@ import type { BaseLogger } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { UserCancellationException } from "../common/vscode/progress";
+import type { EndpointDefaults } from "@octokit/types";
+import { getOctokitBaseUrl } from "../common/vscode/octokit";
 
 export async function getCodeSearchRepositories(
   query: string,
@@ -53,15 +55,19 @@ async function provideOctokitWithThrottling(
 
   const octokit = new MyOctokit({
     auth,
+    baseUrl: getOctokitBaseUrl(),
     throttle: {
-      onRateLimit: (retryAfter: number, options: any): boolean => {
+      onRateLimit: (retryAfter: number, options: EndpointDefaults): boolean => {
         void logger.log(
           `Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`,
         );
 
         return true;
       },
-      onSecondaryRateLimit: (_retryAfter: number, options: any): void => {
+      onSecondaryRateLimit: (
+        _retryAfter: number,
+        options: EndpointDefaults,
+      ): void => {
         void logger.log(
           `Secondary Rate Limit detected for request ${options.method} ${options.url}`,
         );

extensions/ql-vscode/src/databases/db-manager.ts

@@ -16,6 +16,7 @@ import {
 } from "./db-item-selection";
 import { createRemoteTree } from "./db-tree-creator";
 import type { DbConfigValidationError } from "./db-validation-errors";
+import type { VariantAnalysisConfig } from "../config";
 
 export class DbManager extends DisposableObject {
   public readonly onDbItemsChanged: AppEvent<void>;
@@ -25,6 +26,7 @@ export class DbManager extends DisposableObject {
   constructor(
     private readonly app: App,
     private readonly dbConfigStore: DbConfigStore,
+    private readonly variantAnalysisConfigListener: VariantAnalysisConfig,
   ) {
     super();
 
@@ -36,6 +38,10 @@ export class DbManager extends DisposableObject {
     this.dbConfigStore.onDidChangeConfig(() => {
       this.onDbItemsChangesEventEmitter.fire();
     });
+
+    this.variantAnalysisConfigListener.onDidChangeConfiguration?.(() => {
+      this.onDbItemsChangesEventEmitter.fire();
+    });
   }
 
   public getSelectedDbItem(): DbItem | undefined {
@@ -56,7 +62,11 @@ export class DbManager extends DisposableObject {
 
     const expandedItems = this.getExpandedItems();
 
-    const remoteTree = createRemoteTree(configResult.value, expandedItems);
+    const remoteTree = createRemoteTree(
+      configResult.value,
+      this.variantAnalysisConfigListener,
+      expandedItems,
+    );
     return ValueResult.ok(remoteTree.children);
   }
 

extensions/ql-vscode/src/databases/db-module.ts

@@ -7,6 +7,7 @@ import { DbManager } from "./db-manager";
 import { DbPanel } from "./ui/db-panel";
 import { DbSelectionDecorationProvider } from "./ui/db-selection-decoration-provider";
 import type { DatabasePanelCommands } from "../common/commands";
+import { VariantAnalysisConfigListener } from "../config";
 
 export class DbModule extends DisposableObject {
   public readonly dbManager: DbManager;
@@ -17,7 +18,13 @@ export class DbModule extends DisposableObject {
     super();
 
     this.dbConfigStore = new DbConfigStore(app);
-    this.dbManager = this.push(new DbManager(app, this.dbConfigStore));
+    this.dbManager = this.push(
+      new DbManager(
+        app,
+        this.dbConfigStore,
+        new VariantAnalysisConfigListener(),
+      ),
+    );
   }
 
   public static async initialize(app: App): Promise<DbModule> {

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -1,3 +1,4 @@
+import type { VariantAnalysisConfig } from "../config";
 import type { DbConfig, RemoteRepositoryList } from "./config/db-config";
 import { SelectedDbItemKind } from "./config/db-config";
 import type {
@@ -13,13 +14,17 @@ import { ExpandedDbItemKind } from "./db-item-expansion";
 
 export function createRemoteTree(
   dbConfig: DbConfig,
+  variantAnalysisConfig: VariantAnalysisConfig,
   expandedItems: ExpandedDbItem[],
 ): RootRemoteDbItem {
-  const systemDefinedLists = [
-    createSystemDefinedList(10, dbConfig),
-    createSystemDefinedList(100, dbConfig),
-    createSystemDefinedList(1000, dbConfig),
-  ];
+  const systemDefinedLists =
+    variantAnalysisConfig.showSystemDefinedRepositoryLists
+      ? [
+          createSystemDefinedList(10, dbConfig),
+          createSystemDefinedList(100, dbConfig),
+          createSystemDefinedList(1000, dbConfig),
+        ]
+      : [];
 
   const userDefinedRepoLists =
     dbConfig.databases.variantAnalysis.repositoryLists.map((r) =>

extensions/ql-vscode/src/databases/github-databases/api.ts

@@ -3,8 +3,14 @@ import type { Octokit } from "@octokit/rest";
 import type { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods";
 import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
 import type { GitHubDatabaseConfig } from "../../config";
+import { hasGhecDrUri } from "../../config";
 import type { Credentials } from "../../common/authentication";
 import { AppOctokit } from "../../common/octokit";
+import type { ProgressCallback } from "../../common/vscode/progress";
+import { getErrorMessage } from "../../common/helpers-pure";
+import { getLanguageDisplayName } from "../../common/query-language";
+import { window } from "vscode";
+import { extLogger } from "../../common/logging/vscode";
 
 export type CodeqlDatabase =
   RestEndpointMethodTypes["codeScanning"]["listCodeqlDatabases"]["response"]["data"][number];
@@ -62,7 +68,10 @@ export async function listDatabases(
   credentials: Credentials,
   config: GitHubDatabaseConfig,
 ): Promise<ListDatabasesResult | undefined> {
-  const hasAccessToken = !!(await credentials.getExistingAccessToken());
+  // On GHEC-DR, unauthenticated requests will never work, so we should always ask
+  // for authentication.
+  const hasAccessToken =
+    !!(await credentials.getExistingAccessToken()) || hasGhecDrUri();
 
   let octokit = hasAccessToken
     ? await credentials.getOctokit()
@@ -108,3 +117,92 @@ export async function listDatabases(
     octokit,
   };
 }
+
+export async function convertGithubNwoToDatabaseUrl(
+  nwo: string,
+  octokit: Octokit,
+  progress: ProgressCallback,
+  language?: string,
+): Promise<
+  | {
+      databaseUrl: string;
+      owner: string;
+      name: string;
+      databaseId: number;
+      databaseCreatedAt: string;
+      commitOid: string | null;
+    }
+  | undefined
+> {
+  try {
+    const [owner, repo] = nwo.split("/");
+
+    const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+      owner,
+      repo,
+    });
+
+    const languages = response.data.map((db) => db.language);
+
+    if (!language || !languages.includes(language)) {
+      language = await promptForLanguage(languages, progress);
+      if (!language) {
+        return;
+      }
+    }
+
+    const databaseForLanguage = response.data.find(
+      (db) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
+    return {
+      databaseUrl: databaseForLanguage.url,
+      owner,
+      name: repo,
+      databaseId: databaseForLanguage.id,
+      databaseCreatedAt: databaseForLanguage.created_at,
+      commitOid: databaseForLanguage.commit_oid ?? null,
+    };
+  } catch (e) {
+    void extLogger.log(`Error: ${getErrorMessage(e)}`);
+    throw new Error(`Unable to get database for '${nwo}'`);
+  }
+}
+
+async function promptForLanguage(
+  languages: string[],
+  progress: ProgressCallback | undefined,
+): Promise<string | undefined> {
+  progress?.({
+    message: "Choose language",
+    step: 2,
+    maxStep: 2,
+  });
+  if (!languages.length) {
+    throw new Error("No databases found");
+  }
+  if (languages.length === 1) {
+    return languages[0];
+  }
+
+  const items = languages
+    .map((language) => ({
+      label: getLanguageDisplayName(language),
+      description: language,
+      language,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItem = await window.showQuickPick(items, {
+    placeHolder: "Select the database language to download:",
+    ignoreFocusOut: true,
+  });
+  if (!selectedItem) {
+    return undefined;
+  }
+
+  return selectedItem.language;
+}

extensions/ql-vscode/src/databases/github-databases/download.ts

@@ -2,10 +2,8 @@ import { window } from "vscode";
 import type { Octokit } from "@octokit/rest";
 import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
 import { getLanguageDisplayName } from "../../common/query-language";
-import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
+import type { DatabaseFetcher } from "../database-fetcher";
 import { withProgress } from "../../common/vscode/progress";
-import type { DatabaseManager } from "../local-databases";
-import type { CodeQLCliServer } from "../../codeql-cli/cli";
 import type { AppCommandManager } from "../../common/commands";
 import type { GitHubDatabaseConfig } from "../../config";
 import type { CodeqlDatabase } from "./api";
@@ -58,9 +56,7 @@ export async function downloadDatabaseFromGitHub(
   owner: string,
   repo: string,
   databases: CodeqlDatabase[],
-  databaseManager: DatabaseManager,
-  storagePath: string,
-  cliServer: CodeQLCliServer,
+  databaseFetcher: DatabaseFetcher,
   commandManager: AppCommandManager,
 ): Promise<void> {
   const selectedDatabases = await promptForDatabases(databases);
@@ -72,7 +68,7 @@ export async function downloadDatabaseFromGitHub(
     selectedDatabases.map((database) =>
       withProgress(
         async (progress) => {
-          await downloadGitHubDatabaseFromUrl(
+          await databaseFetcher.downloadGitHubDatabaseFromUrl(
             database.url,
             database.id,
             database.created_at,
@@ -81,9 +77,6 @@ export async function downloadDatabaseFromGitHub(
             repo,
             octokit,
             progress,
-            databaseManager,
-            storagePath,
-            cliServer,
             true,
             false,
           );

extensions/ql-vscode/src/databases/github-databases/github-databases-module.ts

@@ -14,7 +14,6 @@ import {
 } from "./download";
 import type { GitHubDatabaseConfig } from "../../config";
 import type { DatabaseManager } from "../local-databases";
-import type { CodeQLCliServer } from "../../codeql-cli/cli";
 import type { CodeqlDatabase, ListDatabasesResult } from "./api";
 import { listDatabases } from "./api";
 import type { DatabaseUpdate } from "./updates";
@@ -24,6 +23,7 @@ import {
   isNewerDatabaseAvailable,
 } from "./updates";
 import type { Octokit } from "@octokit/rest";
+import type { DatabaseFetcher } from "../database-fetcher";
 
 export class GitHubDatabasesModule extends DisposableObject {
   /**
@@ -33,8 +33,7 @@ export class GitHubDatabasesModule extends DisposableObject {
   constructor(
     private readonly app: App,
     private readonly databaseManager: DatabaseManager,
-    private readonly databaseStoragePath: string,
-    private readonly cliServer: CodeQLCliServer,
+    private readonly databaseFetcher: DatabaseFetcher,
     private readonly config: GitHubDatabaseConfig,
   ) {
     super();
@@ -43,15 +42,13 @@ export class GitHubDatabasesModule extends DisposableObject {
   public static async initialize(
     app: App,
     databaseManager: DatabaseManager,
-    databaseStoragePath: string,
-    cliServer: CodeQLCliServer,
+    databaseFetcher: DatabaseFetcher,
     config: GitHubDatabaseConfig,
   ): Promise<GitHubDatabasesModule> {
     const githubDatabasesModule = new GitHubDatabasesModule(
       app,
       databaseManager,
-      databaseStoragePath,
-      cliServer,
+      databaseFetcher,
       config,
     );
     app.subscriptions.push(githubDatabasesModule);
@@ -185,9 +182,7 @@ export class GitHubDatabasesModule extends DisposableObject {
       owner,
       repo,
       databases,
-      this.databaseManager,
-      this.databaseStoragePath,
-      this.cliServer,
+      this.databaseFetcher,
       this.app.commands,
     );
   }
@@ -212,8 +207,7 @@ export class GitHubDatabasesModule extends DisposableObject {
       repo,
       databaseUpdates,
       this.databaseManager,
-      this.databaseStoragePath,
-      this.cliServer,
+      this.databaseFetcher,
       this.app.commands,
     );
   }

extensions/ql-vscode/src/databases/github-databases/updates.ts

@@ -1,11 +1,10 @@
 import type { CodeqlDatabase } from "./api";
 import type { DatabaseItem, DatabaseManager } from "../local-databases";
 import type { Octokit } from "@octokit/rest";
-import type { CodeQLCliServer } from "../../codeql-cli/cli";
 import type { AppCommandManager } from "../../common/commands";
 import { getLanguageDisplayName } from "../../common/query-language";
 import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
-import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
+import type { DatabaseFetcher } from "../database-fetcher";
 import { withProgress } from "../../common/vscode/progress";
 import { window } from "vscode";
 import type { GitHubDatabaseConfig } from "../../config";
@@ -156,8 +155,7 @@ export async function downloadDatabaseUpdateFromGitHub(
   repo: string,
   updates: DatabaseUpdate[],
   databaseManager: DatabaseManager,
-  storagePath: string,
-  cliServer: CodeQLCliServer,
+  databaseFetcher: DatabaseFetcher,
   commandManager: AppCommandManager,
 ): Promise<void> {
   const selectedDatabases = await promptForDatabases(
@@ -179,21 +177,19 @@ export async function downloadDatabaseUpdateFromGitHub(
 
       return withProgress(
         async (progress) => {
-          const newDatabase = await downloadGitHubDatabaseFromUrl(
-            database.url,
-            database.id,
-            database.created_at,
-            database.commit_oid ?? null,
-            owner,
-            repo,
-            octokit,
-            progress,
-            databaseManager,
-            storagePath,
-            cliServer,
-            databaseManager.currentDatabaseItem === update.databaseItem,
-            update.databaseItem.hasSourceArchiveInExplorer(),
-          );
+          const newDatabase =
+            await databaseFetcher.downloadGitHubDatabaseFromUrl(
+              database.url,
+              database.id,
+              database.created_at,
+              database.commit_oid ?? null,
+              owner,
+              repo,
+              octokit,
+              progress,
+              databaseManager.currentDatabaseItem === update.databaseItem,
+              update.databaseItem.hasSourceArchiveInExplorer(),
+            );
           if (newDatabase === undefined) {
             return;
           }

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -5,6 +5,7 @@ import type {
   ProviderResult,
   TreeDataProvider,
   CancellationToken,
+  QuickPickItem,
 } from "vscode";
 import {
   EventEmitter,
@@ -15,7 +16,6 @@ import {
   ThemeIcon,
   ThemeColor,
   workspace,
-  ProgressLocation,
 } from "vscode";
 import { pathExists, stat, readdir, remove } from "fs-extra";
 
@@ -24,11 +24,11 @@ import type {
   DatabaseItem,
   DatabaseManager,
 } from "./local-databases";
-import type {
-  ProgressCallback,
-  ProgressContext,
+import type { ProgressCallback } from "../common/vscode/progress";
+import {
+  UserCancellationException,
+  withProgress,
 } from "../common/vscode/progress";
-import { withInheritedProgress, withProgress } from "../common/vscode/progress";
 import {
   isLikelyDatabaseRoot,
   isLikelyDbLanguageFolder,
@@ -37,14 +37,9 @@ import {
   showAndLogExceptionWithTelemetry,
   showAndLogErrorMessage,
 } from "../common/logging";
-import {
-  importArchiveDatabase,
-  promptImportGithubDatabase,
-  promptImportInternetDatabase,
-} from "./database-fetcher";
+import type { DatabaseFetcher } from "./database-fetcher";
 import { asError, asyncFilter, getErrorMessage } from "../common/helpers-pure";
 import type { QueryRunner } from "../query-server";
-import { isCanary } from "../config";
 import type { App } from "../common/app";
 import { redactableError } from "../common/errors";
 import type { LocalDatabasesCommands } from "../common/commands";
@@ -52,7 +47,10 @@ import {
   createMultiSelectionCommand,
   createSingleSelectionCommand,
 } from "../common/vscode/selection-commands";
-import { tryGetQueryLanguage } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  tryGetQueryLanguage,
+} from "../common/query-language";
 import type { LanguageContextStore } from "../language-context-store";
 
 enum SortOrder {
@@ -111,9 +109,8 @@ class DatabaseTreeDataProvider
     // Note that events from the database manager are instances of DatabaseChangedEvent
     // and events fired by the UI are instances of DatabaseItem
 
-    // When event.item is undefined, then the entire tree is refreshed.
-    // When event.item is a db item, then only that item is refreshed.
-    this._onDidChangeTreeData.fire(event.item);
+    // When a full refresh has occurred, then all items are refreshed by passing undefined.
+    this._onDidChangeTreeData.fire(event.fullRefresh ? undefined : event.item);
   }
 
   private handleDidChangeCurrentDatabaseItem(
@@ -138,7 +135,8 @@ class DatabaseTreeDataProvider
       item.iconPath = new ThemeIcon("error", new ThemeColor("errorForeground"));
     }
     item.tooltip = element.databaseUri.fsPath;
-    item.description = element.language;
+    item.description =
+      element.language + (element.origin?.type === "testproj" ? " (test)" : "");
     return item;
   }
 
@@ -227,12 +225,25 @@ async function chooseDatabaseDir(byFolder: boolean): Promise<Uri | undefined> {
   return getFirst(chosen);
 }
 
+export interface DatabaseSelectionQuickPickItem extends QuickPickItem {
+  databaseKind: "new" | "existing";
+}
+
+export interface DatabaseQuickPickItem extends QuickPickItem {
+  databaseItem: DatabaseItem;
+}
+
+export interface DatabaseImportQuickPickItems extends QuickPickItem {
+  importType: "URL" | "github" | "archive" | "folder";
+}
+
 export class DatabaseUI extends DisposableObject {
   private treeDataProvider: DatabaseTreeDataProvider;
 
   public constructor(
     private app: App,
     private databaseManager: DatabaseManager,
+    private readonly databaseFetcher: DatabaseFetcher,
     languageContext: LanguageContextStore,
     private readonly queryServer: QueryRunner,
     private readonly storagePath: string,
@@ -262,6 +273,7 @@ export class DatabaseUI extends DisposableObject {
         this.handleChooseDatabaseInternet.bind(this),
       "codeQL.chooseDatabaseGithub": this.handleChooseDatabaseGithub.bind(this),
       "codeQL.setCurrentDatabase": this.handleSetCurrentDatabase.bind(this),
+      "codeQL.importTestDatabase": this.handleImportTestDatabase.bind(this),
       "codeQL.setDefaultTourDatabase":
         this.handleSetDefaultTourDatabase.bind(this),
       "codeQL.upgradeCurrentDatabase":
@@ -311,10 +323,9 @@ export class DatabaseUI extends DisposableObject {
 
   private async chooseDatabaseFolder(
     progress: ProgressCallback,
-    token: CancellationToken,
   ): Promise<void> {
     try {
-      await this.chooseAndSetDatabase(true, { progress, token });
+      await this.chooseAndSetDatabase(true, progress);
     } catch (e) {
       void showAndLogExceptionWithTelemetry(
         this.app.logger,
@@ -328,8 +339,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseFolder(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseFolder(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseFolder(progress);
       },
       {
         title: "Adding database from folder",
@@ -339,8 +350,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseFolderFromPalette(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseFolder(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseFolder(progress);
       },
       {
         title: "Choose a Database from a Folder",
@@ -481,10 +492,9 @@ export class DatabaseUI extends DisposableObject {
 
   private async chooseDatabaseArchive(
     progress: ProgressCallback,
-    token: CancellationToken,
   ): Promise<void> {
     try {
-      await this.chooseAndSetDatabase(false, { progress, token });
+      await this.chooseAndSetDatabase(false, progress);
     } catch (e: unknown) {
       void showAndLogExceptionWithTelemetry(
         this.app.logger,
@@ -498,8 +508,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseArchive(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseArchive(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseArchive(progress);
       },
       {
         title: "Adding database from archive",
@@ -509,8 +519,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseArchiveFromPalette(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseArchive(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseArchive(progress);
       },
       {
         title: "Choose a Database from an Archive",
@@ -521,13 +531,7 @@ export class DatabaseUI extends DisposableObject {
   private async handleChooseDatabaseInternet(): Promise<void> {
     return withProgress(
       async (progress) => {
-        await promptImportInternetDatabase(
-          this.app.commands,
-          this.databaseManager,
-          this.storagePath,
-          progress,
-          this.queryServer.cliServer,
-        );
+        await this.databaseFetcher.promptImportInternetDatabase(progress);
       },
       {
         title: "Adding database from URL",
@@ -538,16 +542,7 @@ export class DatabaseUI extends DisposableObject {
   private async handleChooseDatabaseGithub(): Promise<void> {
     return withProgress(
       async (progress) => {
-        const credentials = isCanary() ? this.app.credentials : undefined;
-
-        await promptImportGithubDatabase(
-          this.app.commands,
-          this.databaseManager,
-          this.storagePath,
-          credentials,
-          progress,
-          this.queryServer.cliServer,
-        );
+        await this.databaseFetcher.promptImportGithubDatabase(progress);
       },
       {
         title: "Adding database from GitHub",
@@ -649,14 +644,13 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleClearCache(): Promise<void> {
     return withProgress(
-      async (_progress, token) => {
+      async () => {
         if (
           this.queryServer !== undefined &&
           this.databaseManager.currentDatabaseItem !== undefined
         ) {
           await this.queryServer.clearCacheInDatabase(
             this.databaseManager.currentDatabaseItem,
-            token,
           );
         }
       },
@@ -668,14 +662,13 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleTrimCache(): Promise<void> {
     return withProgress(
-      async (_progress, token) => {
+      async () => {
         if (
           this.queryServer !== undefined &&
           this.databaseManager.currentDatabaseItem !== undefined
         ) {
           await this.queryServer.trimCacheInDatabase(
             this.databaseManager.currentDatabaseItem,
-            token,
           );
         }
       },
@@ -696,13 +689,9 @@ export class DatabaseUI extends DisposableObject {
         try {
           // Assume user has selected an archive if the file has a .zip extension
           if (uri.path.endsWith(".zip")) {
-            await importArchiveDatabase(
-              this.app.commands,
+            await this.databaseFetcher.importLocalDatabase(
               uri.toString(true),
-              this.databaseManager,
-              this.storagePath,
               progress,
-              this.queryServer.cliServer,
             );
           } else {
             await this.databaseManager.openDatabase(uri, {
@@ -724,6 +713,56 @@ export class DatabaseUI extends DisposableObject {
     );
   }
 
+  private async handleImportTestDatabase(uri: Uri): Promise<void> {
+    return withProgress(
+      async (progress) => {
+        try {
+          if (!uri.path.endsWith(".testproj")) {
+            throw new Error(
+              "Please select a valid test database to import. Test databases end with `.testproj`.",
+            );
+          }
+
+          // Check if the database is already in the workspace. If
+          // so, delete it first before importing the new one.
+          const existingItem = this.databaseManager.findTestDatabase(uri);
+          const baseName = basename(uri.fsPath);
+          if (existingItem !== undefined) {
+            progress({
+              maxStep: 9,
+              step: 1,
+              message: `Removing existing test database ${baseName}`,
+            });
+            await this.databaseManager.removeDatabaseItem(existingItem);
+          }
+
+          await this.databaseFetcher.importLocalDatabase(
+            uri.toString(true),
+            progress,
+          );
+
+          if (existingItem !== undefined) {
+            progress({
+              maxStep: 9,
+              step: 9,
+              message: `Successfully re-imported ${baseName}`,
+            });
+          }
+        } catch (e) {
+          // rethrow and let this be handled by default error handling.
+          throw new Error(
+            `Could not set database to ${basename(
+              uri.fsPath,
+            )}. Reason: ${getErrorMessage(e)}`,
+          );
+        }
+      },
+      {
+        title: "(Re-)importing test database from directory",
+      },
+    );
+  }
+
   private async handleRemoveDatabase(
     databaseItems: DatabaseItem[],
   ): Promise<void> {
@@ -779,9 +818,8 @@ export class DatabaseUI extends DisposableObject {
    */
   public async getDatabaseItem(
     progress: ProgressCallback,
-    token: CancellationToken,
   ): Promise<DatabaseItem | undefined> {
-    return await this.getDatabaseItemInternal({ progress, token });
+    return await this.getDatabaseItemInternal(progress);
   }
 
   /**
@@ -794,13 +832,128 @@ export class DatabaseUI extends DisposableObject {
    * notification if it tries to perform any long-running operations.
    */
   private async getDatabaseItemInternal(
-    progress: ProgressContext | undefined,
+    progress: ProgressCallback | undefined,
   ): Promise<DatabaseItem | undefined> {
     if (this.databaseManager.currentDatabaseItem === undefined) {
-      await this.chooseAndSetDatabase(false, progress);
+      progress?.({
+        maxStep: 2,
+        step: 1,
+        message: "Choosing database",
+      });
+      await this.promptForDatabase();
+    }
+    return this.databaseManager.currentDatabaseItem;
+  }
+
+  private async promptForDatabase(): Promise<void> {
+    // If there aren't any existing databases,
+    // don't bother asking the user if they want to pick one.
+    if (this.databaseManager.databaseItems.length === 0) {
+      return this.importNewDatabase();
     }
 
-    return this.databaseManager.currentDatabaseItem;
+    const quickPickItems: DatabaseSelectionQuickPickItem[] = [
+      {
+        label: "$(database) Existing database",
+        detail: "Select an existing database from your workspace",
+        alwaysShow: true,
+        databaseKind: "existing",
+      },
+      {
+        label: "$(arrow-down) New database",
+        detail:
+          "Import a new database from GitHub, a URL, or your local machine...",
+        alwaysShow: true,
+        databaseKind: "new",
+      },
+    ];
+    const selectedOption =
+      await window.showQuickPick<DatabaseSelectionQuickPickItem>(
+        quickPickItems,
+        {
+          placeHolder: "Select an option",
+          ignoreFocusOut: true,
+        },
+      );
+
+    if (!selectedOption) {
+      throw new UserCancellationException("No database selected", true);
+    }
+
+    if (selectedOption.databaseKind === "existing") {
+      await this.selectExistingDatabase();
+    } else if (selectedOption.databaseKind === "new") {
+      await this.importNewDatabase();
+    }
+  }
+
+  private async selectExistingDatabase() {
+    const dbItems: DatabaseQuickPickItem[] =
+      this.databaseManager.databaseItems.map((dbItem) => ({
+        label: dbItem.name,
+        description: getLanguageDisplayName(dbItem.language),
+        databaseItem: dbItem,
+      }));
+
+    const selectedDatabase = await window.showQuickPick(dbItems, {
+      placeHolder: "Select an existing database from your workspace...",
+      ignoreFocusOut: true,
+    });
+
+    if (!selectedDatabase) {
+      throw new UserCancellationException("No database selected", true);
+    }
+
+    await this.databaseManager.setCurrentDatabaseItem(
+      selectedDatabase.databaseItem,
+    );
+  }
+
+  private async importNewDatabase() {
+    const importOptions: DatabaseImportQuickPickItems[] = [
+      {
+        label: "$(github) GitHub",
+        detail: "Import a database from a GitHub repository",
+        alwaysShow: true,
+        importType: "github",
+      },
+      {
+        label: "$(link) URL",
+        detail: "Import a database archive or folder from a remote URL",
+        alwaysShow: true,
+        importType: "URL",
+      },
+      {
+        label: "$(file-zip) Archive",
+        detail: "Import a database from a local ZIP archive",
+        alwaysShow: true,
+        importType: "archive",
+      },
+      {
+        label: "$(folder) Folder",
+        detail: "Import a database from a local folder",
+        alwaysShow: true,
+        importType: "folder",
+      },
+    ];
+    const selectedImportOption =
+      await window.showQuickPick<DatabaseImportQuickPickItems>(importOptions, {
+        placeHolder:
+          "Import a new database from GitHub, a URL, or your local machine...",
+        ignoreFocusOut: true,
+      });
+    if (!selectedImportOption) {
+      throw new UserCancellationException("No database selected", true);
+    }
+    if (selectedImportOption.importType === "github") {
+      await this.handleChooseDatabaseGithub();
+    } else if (selectedImportOption.importType === "URL") {
+      await this.handleChooseDatabaseInternet();
+    } else if (selectedImportOption.importType === "archive") {
+      await this.handleChooseDatabaseArchive();
+    } else if (selectedImportOption.importType === "folder") {
+      await this.handleChooseDatabaseFolder();
+    }
   }
 
   /**
@@ -809,41 +962,28 @@ export class DatabaseUI extends DisposableObject {
    */
   private async chooseAndSetDatabase(
     byFolder: boolean,
-    progress: ProgressContext | undefined,
+    progress: ProgressCallback,
   ): Promise<DatabaseItem | undefined> {
     const uri = await chooseDatabaseDir(byFolder);
     if (!uri) {
       return undefined;
     }
 
-    return await withInheritedProgress(
-      progress,
-      async (progress) => {
-        if (byFolder) {
-          const fixedUri = await this.fixDbUri(uri);
-          // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri, {
-            type: "folder",
-          });
-        } else {
-          // we are selecting a database archive. Must unzip into a workspace-controlled area
-          // before importing.
-          return await importArchiveDatabase(
-            this.app.commands,
-            uri.toString(true),
-            this.databaseManager,
-            this.storagePath,
-            progress,
-            this.queryServer.cliServer,
-          );
-        }
-      },
-      {
-        location: ProgressLocation.Notification,
-        cancellable: true,
-        title: "Opening database",
-      },
-    );
+    if (byFolder && !uri.fsPath.endsWith("testproj")) {
+      const fixedUri = await this.fixDbUri(uri);
+      // we are selecting a database folder
+      return await this.databaseManager.openDatabase(fixedUri, {
+        type: "folder",
+      });
+    } else {
+      // we are selecting a database archive or a testproj.
+      // Unzip archives (if an archive) and copy into a workspace-controlled area
+      // before importing.
+      return await this.databaseFetcher.importLocalDatabase(
+        uri.toString(true),
+        progress,
+      );
+    }
   }
 
   /**

extensions/ql-vscode/src/databases/local-databases/database-contents.ts

@@ -1,4 +1,7 @@
+import { pathExists, remove } from "fs-extra";
+import { join } from "path";
 import type { Uri } from "vscode";
+import { zip } from "zip-a-folder";
 
 /**
  * The layout of the database.
@@ -28,3 +31,26 @@ export interface DatabaseContents {
 export interface DatabaseContentsWithDbScheme extends DatabaseContents {
   dbSchemeUri: Uri; // Always present
 }
+
+/**
+ * Databases created by the old odasa tool will not have a zipped
+ * source location. However, this extension works better if sources
+ * are zipped.
+ *
+ * This function ensures that the source location is zipped. If the
+ * `src` folder exists and the `src.zip` file does not, the `src`
+ * folder will be zipped and then deleted.
+ *
+ * @param databasePath The full path to the unzipped database
+ */
+export async function ensureZippedSourceLocation(
+  databasePath: string,
+): Promise<void> {
+  const srcFolderPath = join(databasePath, "src");
+  const srcZipPath = `${srcFolderPath}.zip`;
+
+  if ((await pathExists(srcFolderPath)) && !(await pathExists(srcZipPath))) {
+    await zip(srcFolderPath, srcZipPath);
+    await remove(srcFolderPath);
+  }
+}

extensions/ql-vscode/src/databases/local-databases/database-events.ts

@@ -16,4 +16,8 @@ export enum DatabaseEventKind {
 export interface DatabaseChangedEvent {
   kind: DatabaseEventKind;
   item: DatabaseItem | undefined;
+  // If true, event handlers should consider the database manager
+  // to have been fully refreshed. Any state managed by the
+  // event handler should be fully refreshed as well.
+  fullRefresh: boolean;
 }

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -66,6 +66,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return this.options.origin;
   }
 
+  public get extensionManagedLocation(): string | undefined {
+    return this.options.extensionManagedLocation;
+  }
+
   public resolveSourceFile(uriStr: string | undefined): Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? Uri.parse(uriStr, true) : undefined;

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -31,6 +31,12 @@ export interface DatabaseItem {
    */
   readonly origin: DatabaseOrigin | undefined;
 
+  /**
+   * The location of the base storage location as managed by the extension, or undefined
+   * if unknown or not managed by the extension.
+   */
+  readonly extensionManagedLocation: string | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -18,7 +18,10 @@ import {
 import { join } from "path";
 import type { FullDatabaseOptions } from "./database-options";
 import { DatabaseItemImpl } from "./database-item-impl";
-import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
+import {
+  showBinaryChoiceDialog,
+  showNeverAskAgainDialog,
+} from "../../common/vscode/dialog";
 import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
@@ -32,7 +35,7 @@ import { QlPackGenerator } from "../../local-queries/qlpack-generator";
 import { asError, getErrorMessage } from "../../common/helpers-pure";
 import type { DatabaseItem, PersistedDatabaseItem } from "./database-item";
 import { redactableError } from "../../common/errors";
-import { remove } from "fs-extra";
+import { copy, remove, stat } from "fs-extra";
 import { containsPath } from "../../common/files";
 import type { DatabaseChangedEvent } from "./database-events";
 import { DatabaseEventKind } from "./database-events";
@@ -40,6 +43,7 @@ import { DatabaseResolver } from "./database-resolver";
 import { telemetryListener } from "../../common/vscode/telemetry";
 import type { LanguageContextStore } from "../../language-context-store";
 import type { DatabaseOrigin } from "./database-origin";
+import { ensureZippedSourceLocation } from "./database-contents";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -82,6 +86,10 @@ function eventFired<T>(
 }
 
 type OpenDatabaseOptions = {
+  /**
+   * A location that is managed by the extension.
+   */
+  extensionManagedLocation?: string;
   isTutorialDatabase?: boolean;
   /**
    * Whether to add a workspace folder containing the source archive to the workspace. Default is true.
@@ -116,6 +124,7 @@ export class DatabaseManager extends DisposableObject {
     super();
 
     qs.onStart(this.reregisterDatabases.bind(this));
+    qs.onQueryRunStarting(this.maybeReimportTestDatabase.bind(this));
 
     this.push(
       this.languageContext.onLanguageContextChanged(async () => {
@@ -141,6 +150,7 @@ export class DatabaseManager extends DisposableObject {
     makeSelected = true,
     displayName?: string,
     {
+      extensionManagedLocation,
       isTutorialDatabase = false,
       addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
     }: OpenDatabaseOptions = {},
@@ -149,6 +159,7 @@ export class DatabaseManager extends DisposableObject {
       uri,
       origin,
       displayName,
+      extensionManagedLocation,
     );
 
     return await this.addExistingDatabaseItem(
@@ -159,6 +170,108 @@ export class DatabaseManager extends DisposableObject {
     );
   }
 
+  /**
+   * Finds a test database that was originally imported from `uri`.
+   * A test database is creeated by the `codeql test run` command
+   * and ends with `.testproj`.
+   * @param uri The original location of the database
+   * @returns The first database item found that matches the uri
+   */
+  public findTestDatabase(uri: vscode.Uri): DatabaseItem | undefined {
+    const originPath = uri.fsPath;
+    for (const item of this._databaseItems) {
+      if (item.origin?.type === "testproj" && item.origin.path === originPath) {
+        return item;
+      }
+    }
+    return undefined;
+  }
+
+  public async maybeReimportTestDatabase(
+    databaseUri: vscode.Uri,
+    forceImport = false,
+  ): Promise<void> {
+    const res = await this.isTestDatabaseOutdated(databaseUri);
+    if (!res) {
+      return;
+    }
+    const doit =
+      forceImport ||
+      (await showBinaryChoiceDialog(
+        "This test database is outdated. Do you want to reimport it?",
+      ));
+
+    if (doit) {
+      await this.reimportTestDatabase(databaseUri);
+    }
+  }
+
+  /**
+   * Checks if the origin of the imported database is newer.
+   * The imported database must be a test database.
+   * @param databaseUri the URI of the imported database to check
+   * @returns true if both databases exist and the origin database is newer.
+   */
+  private async isTestDatabaseOutdated(
+    databaseUri: vscode.Uri,
+  ): Promise<boolean> {
+    const dbItem = this.findDatabaseItem(databaseUri);
+    if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
+      return false;
+    }
+
+    // Compare timestmps of the codeql-database.yml files of the original and the
+    // imported databases.
+    const originDbYml = join(dbItem.origin.path, "codeql-database.yml");
+    const importedDbYml = join(
+      dbItem.databaseUri.fsPath,
+      "codeql-database.yml",
+    );
+
+    let originStat;
+    try {
+      originStat = await stat(originDbYml);
+    } catch (e) {
+      // if there is an error here, assume that the origin database
+      // is no longer available. Safely ignore and do not try to re-import.
+      return false;
+    }
+
+    try {
+      const importedStat = await stat(importedDbYml);
+      return originStat.mtimeMs > importedStat.mtimeMs;
+    } catch (e) {
+      // If either of the files does not exist, we assume the origin is newer.
+      // This shouldn't happen unless the user manually deleted one of the files.
+      return true;
+    }
+  }
+
+  /**
+   * Reimport the specified imported database from its origin.
+   * The imported databsae must be a testproj database.
+   *
+   * @param databaseUri the URI of the imported database to reimport
+   */
+  private async reimportTestDatabase(databaseUri: vscode.Uri): Promise<void> {
+    const dbItem = this.findDatabaseItem(databaseUri);
+    if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
+      throw new Error(`Database ${databaseUri} is not a testproj.`);
+    }
+
+    await this.removeDatabaseItem(dbItem);
+    await copy(dbItem.origin.path, databaseUri.fsPath);
+    await ensureZippedSourceLocation(databaseUri.fsPath);
+    const newDbItem = new DatabaseItemImpl(databaseUri, dbItem.contents, {
+      dateAdded: Date.now(),
+      language: dbItem.language,
+      origin: dbItem.origin,
+      extensionManagedLocation: dbItem.extensionManagedLocation,
+    });
+    await this.addDatabaseItem(newDbItem);
+    await this.setCurrentDatabaseItem(newDbItem);
+  }
+
   /**
    * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
    * the list.
@@ -202,6 +315,7 @@ export class DatabaseManager extends DisposableObject {
     uri: vscode.Uri,
     origin: DatabaseOrigin | undefined,
     displayName: string | undefined,
+    extensionManagedLocation?: string,
   ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
     const fullOptions: FullDatabaseOptions = {
@@ -210,6 +324,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
       origin,
+      extensionManagedLocation,
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
@@ -370,6 +485,7 @@ export class DatabaseManager extends DisposableObject {
     let dateAdded = undefined;
     let language = undefined;
     let origin = undefined;
+    let extensionManagedLocation = undefined;
     if (state.options) {
       if (typeof state.options.displayName === "string") {
         displayName = state.options.displayName;
@@ -379,6 +495,7 @@ export class DatabaseManager extends DisposableObject {
       }
       language = state.options.language;
       origin = state.options.origin;
+      extensionManagedLocation = state.options.extensionManagedLocation;
     }
 
     const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -392,6 +509,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded,
       language,
       origin,
+      extensionManagedLocation,
     };
     const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 
@@ -495,6 +613,7 @@ export class DatabaseManager extends DisposableObject {
       this._onDidChangeCurrentDatabaseItem.fire({
         item,
         kind: DatabaseEventKind.Change,
+        fullRefresh: false,
       });
     }
   }
@@ -544,8 +663,9 @@ export class DatabaseManager extends DisposableObject {
     }
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
-      item: undefined,
+      item,
       kind: DatabaseEventKind.Add,
+      fullRefresh: true,
     });
   }
 
@@ -553,9 +673,9 @@ export class DatabaseManager extends DisposableObject {
     item.name = newName;
     await this.updatePersistedDatabaseList();
     this._onDidChangeDatabaseItem.fire({
-      // pass undefined so that the entire tree is rebuilt in order to re-sort
-      item: undefined,
+      item,
       kind: DatabaseEventKind.Rename,
+      fullRefresh: true,
     });
   }
 
@@ -583,24 +703,29 @@ export class DatabaseManager extends DisposableObject {
     // Remove this database item from the allow-list
     await this.deregisterDatabase(item);
 
+    // Find whether we know directly which directory we should remove
+    const directoryToRemove = item.extensionManagedLocation
+      ? vscode.Uri.file(item.extensionManagedLocation)
+      : item.databaseUri;
+
     // Delete folder from file system only if it is controlled by the extension
-    if (this.isExtensionControlledLocation(item.databaseUri)) {
+    if (this.isExtensionControlledLocation(directoryToRemove)) {
       void extLogger.log("Deleting database from filesystem.");
-      await remove(item.databaseUri.fsPath).then(
-        () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`),
+      await remove(directoryToRemove.fsPath).then(
+        () => void extLogger.log(`Deleted '${directoryToRemove.fsPath}'`),
         (e: unknown) =>
           void extLogger.log(
             `Failed to delete '${
-              item.databaseUri.fsPath
+              directoryToRemove.fsPath
             }'. Reason: ${getErrorMessage(e)}`,
           ),
       );
     }
 
-    // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
-      item: undefined,
+      item,
       kind: DatabaseEventKind.Remove,
+      fullRefresh: true,
     });
   }
 
@@ -653,6 +778,7 @@ export class DatabaseManager extends DisposableObject {
       this._onDidChangeDatabaseItem.fire({
         kind: DatabaseEventKind.Refresh,
         item: databaseItem,
+        fullRefresh: false,
       });
     }
   }

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -5,10 +5,12 @@ export interface DatabaseOptions {
   dateAdded?: number | undefined;
   language?: string;
   origin?: DatabaseOrigin;
+  extensionManagedLocation?: string;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   dateAdded: number | undefined;
   language: string | undefined;
   origin: DatabaseOrigin | undefined;
+  extensionManagedLocation: string | undefined;
 }