refactor cli

Bump the storybook group in /extensions/ql-vscode with 12 updates

.github/codeql/queries/ProgressBar.qll

@@ -0,0 +1,16 @@
+import javascript
+
+class WithProgressCall extends CallExpr {
+  WithProgressCall() { this.getCalleeName() = "withProgress" }
+
+  predicate usesToken() { exists(this.getTokenParameter()) }
+
+  Parameter getTokenParameter() { result = this.getArgument(0).(Function).getParameter(1) }
+
+  Property getCancellableProperty() { result = this.getArgument(1).(ObjectExpr).getPropertyByName("cancellable") }
+
+  predicate isCancellable() {
+    this.getCancellableProperty().getInit().(BooleanLiteral).getBoolValue() =
+      true
+  }
+}

.github/codeql/queries/progress-not-cancellable.ql

@@ -0,0 +1,20 @@
+/**
+ * @name Using token for non-cancellable progress bar
+ * @kind problem
+ * @problem.severity warning
+ * @id vscode-codeql/progress-not-cancellable
+ * @description If we call `withProgress` without `cancellable: true` then the
+ * token that is given to us should be ignored because it won't ever be cancelled.
+ * This makes the code more confusing as it tries to account for cases that can't
+ * happen. The fix is to either not use the token or make the progress bar cancellable.
+ */
+
+import javascript
+import ProgressBar
+
+from WithProgressCall t
+where not t.isCancellable() and t.usesToken()
+select t,
+  "The $@ should not be used when the progress bar is not cancellable. Either stop using the $@ or mark the progress bar as cancellable.",
+  t.getTokenParameter(), t.getTokenParameter().getName(), t.getTokenParameter(),
+  t.getTokenParameter().getName()

.github/codeql/queries/token-not-used.ql

@@ -0,0 +1,18 @@
+/**
+ * @name Don't ignore the token for a cancellable progress bar
+ * @kind problem
+ * @problem.severity warning
+ * @id vscode-codeql/token-not-used
+ * @description If we call `withProgress` with `cancellable: true` but then
+ * ignore the token that is given to us, it will lead to a poor user experience
+ * because the progress bar will appear to be canceled but it will not actually
+ * affect the background process. Either check the token and respect when it
+ * has been cancelled, or mark the progress bar as not cancellable.
+ */
+
+import javascript
+import ProgressBar
+
+from WithProgressCall t
+where t.isCancellable() and not t.usesToken()
+select t, "This progress bar is $@ but the token is not used. Either use the token or mark the progress bar as not cancellable.", t.getCancellableProperty(), "cancellable"

.github/dependabot.yml

@@ -13,6 +13,23 @@ updates:
       # are unrelated to the Node version, so we allow those.
       - dependency-name: "@types/node"
         update-types: ["version-update:semver-major", "version-update:semver-minor"]
+    groups:
+      octokit:
+        patterns:
+          - "@octokit/*"
+        update-types:
+          - "minor"
+          - "patch"
+      storybook:
+        patterns:
+          - "@storybook/*"
+          - "storybook"
+      testing-library:
+        patterns:
+          - "@testing-library/*"
+      typescript-eslint:
+        patterns:
+          - "@typescript-eslint/*"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
@@ -20,3 +37,10 @@ updates:
       day: "thursday" # Thursday is arbitrary
     labels:
       - "Update dependencies"
+  - package-ecosystem: docker
+    directory: "extensions/ql-vscode/test/e2e"
+    schedule:
+      interval: "weekly"
+      day: "thursday" # Thursday is arbitrary
+    labels:
+      - "Update dependencies"

.github/pull_request_template.md

@@ -5,8 +5,4 @@
 
 Replace this with a description of the changes your pull request makes.
 
-## Checklist
-
-- [ ] [CHANGELOG.md](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) has been updated to incorporate all user visible changes made by this pull request.
-- [ ] Issues have been created for any UI or other user-facing changes made by this pull request.
-- [ ] _[Maintainers only]_ If this pull request makes user-facing changes that require documentation changes, open a corresponding docs pull request in the [github/codeql](https://github.com/github/codeql/tree/main/docs/codeql/codeql-for-visual-studio-code) repo and add the `ready-for-doc-review` label there.
+Remember to update the [changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) if there have been user-facing changes!

.github/workflows/main.yml

@@ -144,6 +144,19 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
     steps:
+      # Enable 8.3 filename creation. This is not required to run the extension but it is required for the unit tests to pass.
+      # This feature is currently enabled by default in Windows 11 for the C: drive and therefore we must maintain support for it.
+      # This setting needs to be enabled before files are created, i.e. before we checkout the repository.
+      - name: Enable 8.3 filenames
+        shell: pwsh
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          $shortNameEnableProcess = Start-Process -FilePath fsutil.exe -ArgumentList ('8dot3name', 'set', '0') -Wait -PassThru
+          $shortNameEnableExitCode = $shortNameEnableProcess.ExitCode
+          if ($shortNameEnableExitCode -ne 0) {
+              exit $shortNameEnableExitCode
+          }
+
       - name: Checkout
         uses: actions/checkout@v4
         with:

.github/workflows/release.yml

@@ -134,7 +134,7 @@ jobs:
 
       - name: Publish to Registry
         run: |
-          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix
+          npx @vscode/vsce publish -p $VSCE_TOKEN --packagePath *.vsix
 
   open-vsx-publish:
     name: Publish to Open VSX Registry

.vscode/settings.json

@@ -111,4 +111,10 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true,
   },
+  "github.copilot.advanced": {
+    
+
+  },
+  "codeQL.variantAnalysis.enableGhecDr": true,
+  "github-enterprise.uri": "http://server:8080/"
 }

CONTRIBUTING.md

@@ -28,6 +28,7 @@ Here are a few things you can do that will increase the likelihood of your pull
   - [Integration tests that do require the VS Code API are located here](extensions/ql-vscode/src/vscode-tests).
 - Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
 - Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+- Update the [changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md) if you are making user-facing changes.
 
 ## Setting up a local build
 

docs/node-version.md

@@ -1,5 +1,8 @@
 # Node version
 
+> [!NOTE]
+> It should not be necessary to update the Node.js version manually. The [update-node-version.yml](https://github.com/github/vscode-codeql/blob/main/.github/workflows/update-node-version.yml) workflow runs daily and will open a pull request if the Node.js version needs to be updated.
+
 The CodeQL for VS Code extension defines the version of Node.js that it is intended to run with. This Node.js version is used when running most CI and unit tests.
 
 When running in production (i.e. as an extension for a VS Code application) it will use the Node.js version provided by VS Code. This can mean a different Node.js version is used by different users with different versions of VS Code.

docs/releasing.md

@@ -1,5 +1,6 @@
 # Releasing (write access required)
 
+1. Make sure the needed authentication keys are valid. Most likely the Azure DevOps PAT needs to be regenerated. See below.
 1. Determine the new version number. We default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
     - Making substantial new features available to all users. This can include lifting a feature flag.
     - Breakage in compatibility with recent versions of the CLI.
@@ -60,7 +61,7 @@
 
 ## Secrets and authentication for publishing
 
-Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token.
 
 To regenerate the Open VSX token:
 
@@ -69,4 +70,4 @@ To regenerate the Open VSX token:
 1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
 1. Update the secret in the `publish-open-vsx` environment in the project settings.
 
-To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.
+To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 7 days and must be regenerated.

docs/test-plan.md

@@ -53,18 +53,20 @@ choose to go through some of the Optional Test Cases.
 
 #### Test case 3: Running a non-problem query and viewing results
 
-1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+1. Open the [cpp HubClasses query](https://github.com/github/codeql/blob/main/cpp/ql/src/Architecture/General%20Class-Level%20Information/HubClasses.ql).
 2. Select the `google/brotli` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
+   - Check that the `#select` result set is shown
    - Check that the results table is rendered
    - Check that result locations can be clicked on
 
 #### Test case 4: Can use AST viewer
 
 1. Click on any code location from a previous query to open a source file from a database
-2. Open the AST viewing panel and click "View AST"
-3. Once the AST is computed:
+2. Select the highlighted code in the source file
+3. Open the AST viewing panel and click "View AST"
+4. Once the AST is computed:
    - Check that it can be navigated
 
 ### MRVA
@@ -143,6 +145,48 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
+#### Test Case 6: Using model packs in MRVA
+
+1. Create a model pack with mock data
+   1. Create a new directory `test-model-pack`
+   2. Create a `qlpack.yml` file in that directory with the following contents:
+
+      ```yaml
+      name: github/test-model-pack
+      version: 0.0.0
+      library: true
+      extensionTargets:
+        codeql/python-all: '*'
+      dataExtensions:
+        - extension.yml
+      ```
+
+   3. Create an `extension.yml` in the same directory with the following contents:
+
+      ```yaml
+      extensions:
+      - addsTo:
+          pack: codeql/python-all
+          extensible: sinkModel
+        data:
+         - ["vscode-codeql","Member[initialize].Argument[0]","code-injection"]
+      ```
+
+2. In a Python query pack, create the following query (e.g. `sinks.ql`):
+
+   ```ql
+   import python
+   import semmle.python.frameworks.data.internal.ApiGraphModelsExtensions
+
+   from string path, string kind
+   where sinkModel("vscode-codeql", path, kind, _)
+   select path, kind
+   ```
+
+3. Run a MRVA against a Python repository (e.g. `psf/requests`) with this query.
+4. Check that the results view contains 1 result with the values corresponding to the `extension.yml` file:
+   ![Model packs results table for `psf/requests`](images/model-pack-results-table.png)
+
 ### CodeQL Model Editor
 
 #### Test Case 1: Opening the model editor
@@ -151,7 +195,7 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
    - Check that the editor loads and shows methods to model.
    - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
-   - Check that the "Open source" link works.
+   - Check that the "Open source" link works (if you have the database source).
    - Check that the 'View' button works and the Method Usage panel highlight the correct method and usage
    - Check that the Method Modeling panel shows the correct method and modeling state
 
@@ -166,17 +210,7 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 4. Open the ".model.yml" file corresponding to the library that was changed.
    - Check that the file contains entries for the methods that were modeled.
 
-#### Test Case 3: Model with AI
-
-Note that this test requires the feature flag: `codeQL.model.llmGeneration`
-
-A package that the AI normally gives models for is `javax.servlet-api` from the `jhy/jsoup` repository.
-
-1. Click "Model with AI".
-   - Check that rows change to "Thinking".
-   - Check that results come back and rows get filled out.
-
-#### Test Case 4: Model as dependency
+#### Test Case 3: Model as dependency
 
 Note that this test requires the feature flag: `codeQL.model.flowGeneration`
 

extensions/ql-vscode/.babelrc.json

@@ -1,16 +0,0 @@
-{
-  "sourceType": "unambiguous",
-  "presets": [
-    [
-      "@babel/preset-env",
-      {
-        "targets": {
-          "chrome": 100
-        }
-      }
-    ],
-    "@babel/preset-typescript",
-    "@babel/preset-react"
-  ],
-  "plugins": []
-}

extensions/ql-vscode/.eslintrc.js

@@ -40,7 +40,7 @@ const baseConfig = {
         ignoreRestSiblings: false,
       },
     ],
-    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-explicit-any": "error",
     "@typescript-eslint/no-floating-promises": ["error", { ignoreVoid: true }],
     "@typescript-eslint/no-invalid-this": "off",
     "@typescript-eslint/no-shadow": "off",
@@ -48,7 +48,7 @@ const baseConfig = {
     "@typescript-eslint/no-throw-literal": "error",
     "@typescript-eslint/consistent-type-imports": "error",
     "import/consistent-type-specifier-style": ["error", "prefer-top-level"],
-    curly: ["error", "all"],
+    // curly: ["error", "all"],
     "escompat/no-regexp-lookbehind": "off",
     "etc/no-implicit-any-catch": "error",
     "filenames/match-regex": "off",
@@ -121,15 +121,6 @@ module.exports = {
         },
       },
     },
-    {
-      files: ["test/**/*"],
-      parserOptions: {
-        project: resolve(__dirname, "test/tsconfig.json"),
-      },
-      env: {
-        jest: true,
-      },
-    },
     {
       files: ["test/vscode-tests/**/*"],
       parserOptions: {
@@ -156,6 +147,18 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ["test/**/*"],
+      parserOptions: {
+        project: resolve(__dirname, "test/tsconfig.json"),
+      },
+      env: {
+        jest: true,
+      },
+      rules: {
+        "@typescript-eslint/no-explicit-any": "off",
+      },
+    },
     {
       files: [
         ".eslintrc.js",

extensions/ql-vscode/.nvmrc

@@ -1 +1 @@
-v18.17.1
+v20.9.0

extensions/ql-vscode/.storybook/main.ts

@@ -1,7 +1,7 @@
-import type { StorybookConfig } from "@storybook/react-webpack5";
+import type { StorybookConfig } from "@storybook/react-vite";
 
 const config: StorybookConfig = {
-  stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
+  stories: ["../src/**/*.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
   addons: [
     "@storybook/addon-links",
     "@storybook/addon-essentials",
@@ -10,7 +10,7 @@ const config: StorybookConfig = {
     "./vscode-theme-addon/preset.ts",
   ],
   framework: {
-    name: "@storybook/react-webpack5",
+    name: "@storybook/react-vite",
     options: {},
   },
   docs: {

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -5,10 +5,10 @@ import { useCallback } from "react";
 import { useGlobals } from "@storybook/manager-api";
 import {
   IconButton,
-  Icons,
   TooltipLinkList,
   WithTooltip,
 } from "@storybook/components";
+import { DashboardIcon } from "@storybook/icons";
 
 import { themeNames, VSCodeTheme } from "./theme";
 
@@ -53,7 +53,7 @@ export const ThemeSelector: FunctionComponent = () => {
         title="Change the theme of the preview"
         active={vscodeTheme !== VSCodeTheme.Dark}
       >
-        <Icons icon="dashboard" />
+        <DashboardIcon />
       </IconButton>
     </WithTooltip>
   );

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -1,3 +1,5 @@
+/// <reference types="vite/client" />
+
 import { useEffect } from "react";
 import type {
   PartialStoryFn as StoryFunction,
@@ -6,31 +8,20 @@ import type {
 
 import { VSCodeTheme } from "./theme";
 
+import darkThemeStyle from "../../src/stories/vscode-theme-dark.css?url";
+import lightThemeStyle from "../../src/stories/vscode-theme-light.css?url";
+import lightHighContrastThemeStyle from "../../src/stories/vscode-theme-light-high-contrast.css?url";
+import darkHighContrastThemeStyle from "../../src/stories/vscode-theme-dark-high-contrast.css?url";
+import githubLightDefaultThemeStyle from "../../src/stories/vscode-theme-github-light-default.css?url";
+import githubDarkDefaultThemeStyle from "../../src/stories/vscode-theme-github-dark-default.css?url";
+
 const themeFiles: { [key in VSCodeTheme]: string } = {
-  [VSCodeTheme.Dark]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-dark.css")
-      .default,
-  [VSCodeTheme.Light]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-light.css")
-      .default,
-  [VSCodeTheme.LightHighContrast]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-light-high-contrast.css")
-      .default,
-  [VSCodeTheme.DarkHighContrast]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-dark-high-contrast.css")
-      .default,
-  [VSCodeTheme.GitHubLightDefault]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-github-light-default.css")
-      .default,
-  [VSCodeTheme.GitHubDarkDefault]:
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs,import/no-webpack-loader-syntax
-    require("!file-loader?modules!../../src/stories/vscode-theme-github-dark-default.css")
-      .default,
+  [VSCodeTheme.Dark]: darkThemeStyle,
+  [VSCodeTheme.Light]: lightThemeStyle,
+  [VSCodeTheme.LightHighContrast]: lightHighContrastThemeStyle,
+  [VSCodeTheme.DarkHighContrast]: darkHighContrastThemeStyle,
+  [VSCodeTheme.GitHubLightDefault]: githubLightDefaultThemeStyle,
+  [VSCodeTheme.GitHubDarkDefault]: githubDarkDefaultThemeStyle,
 };
 
 export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,35 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## [UNRELEASED]
+
+## 1.13.1 - 29 May 2024
+
+- Fix a bug when re-importing test databases that erroneously showed old source code. [#3616](https://github.com/github/vscode-codeql/pull/3616)
+- Update the progress window with details on potentially long-running post-processing steps after running a query. [#3622](https://github.com/github/vscode-codeql/pull/3622)
+
+## 1.13.0 - 1 May 2024
+
+- Add Ruby support to the CodeQL Model Editor. [#3584](https://github.com/github/vscode-codeql/pull/3584)
+- Remove support for CodeQL CLI versions older than 2.14.6. [#3562](https://github.com/github/vscode-codeql/pull/3562)
+
+## 1.12.5 - 9 April 2024
+
+- Add new supported source and sink kinds in the CodeQL Model Editor [#3511](https://github.com/github/vscode-codeql/pull/3511)
+- Fix a bug where the test explorer wouldn't display certain tests. [#3527](https://github.com/github/vscode-codeql/pull/3527)
+- The "model dependency" operation in the model editor can now be cancelled. [#3517](https://github.com/github/vscode-codeql/pull/3517)
+
+## 1.12.4 - 20 March 2024
+
+- Don't show notification after local query cancellation. [#3489](https://github.com/github/vscode-codeql/pull/3489)
+- Databases created from [CodeQL test cases](https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/testing-custom-queries) are now copied into a shared VS Code storage location. This avoids a bug where re-running test cases would fail if the test's database is already imported into the workspace. [#3433](https://github.com/github/vscode-codeql/pull/3433)
+
+## 1.12.3 - 29 February 2024
+
+- Update variant analysis view to show when cancelation is in progress. [#3405](https://github.com/github/vscode-codeql/pull/3405)
+- Remove support for CodeQL CLI versions older than 2.13.5. [#3371](https://github.com/github/vscode-codeql/pull/3371)
+- Add a timeout to downloading databases and the CodeQL CLI. These can be changed using the `codeQL.addingDatabases.downloadTimeout` and `codeQL.cli.downloadTimeout` settings respectively. [#3373](https://github.com/github/vscode-codeql/pull/3373)
+- When downloading a CodeQL database through the model editor, only use credentials when in canary mode. [#3440](https://github.com/github/vscode-codeql/pull/3440)
+
 ## 1.12.2 - 14 February 2024
 
 - Stop allowing running variant analyses with a query outside of the workspace. [#3302](https://github.com/github/vscode-codeql/pull/3302)
@@ -474,7 +504,7 @@ No user facing changes.
 - Allow setting `codeQL.runningQueries.numberOfThreads` and `codeQL.runningTests.numberOfThreads` to 0, (which is interpreted as 'use one thread per core on the machine').
 - Clear the problems view of all CodeQL query results when a database is removed.
 - Add a `View DIL` command on query history items. This opens a text editor containing the Datalog Intermediary Language representation of the compiled query.
-- Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/exploring-the-structure-of-your-source-code.html).
+- Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/exploring-the-structure-of-your-source-code/).
 - The `codeQL.runningTests.numberOfThreads` setting is now used correctly when running tests.
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the qlpacks.
 - Ensure output of CodeQL test runs includes compilation error messages and test failure messages.

extensions/ql-vscode/README.md

@@ -9,10 +9,12 @@ This project is an extension for Visual Studio Code that adds rich language supp
 
 To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
+You can also read full documentation for the extension on the [GitHub documentation website](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension).
+
 ## Quick start overview
 
 The information in this `README` file describes the quickest way to start using CodeQL.
-For information about other configurations, see the separate [CodeQL help](https://codeql.github.com/docs/codeql-for-visual-studio-code/).
+For information about other configurations, see "[Using the advanced functionality of the CodeQL for VS Code extension](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension)" in the GitHub documentation.
 
 ### Quick start: Installing and configuring the extension
 
@@ -42,7 +44,7 @@ The CodeQL extension requires a minimum of Visual Studio Code 1.39. Older versio
 
 The extension uses the [CodeQL CLI](https://codeql.github.com/docs/codeql-cli/) to compile and run queries. The extension automatically manages access to the CLI for you by default (recommended). To check for updates to the CodeQL CLI, you can use the **CodeQL: Check for CLI Updates** command.
 
-If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see [Configuring access to the CodeQL CLI](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#configuring-access-to-the-codeql-cli).
+If you want to override the default behavior and use a CodeQL CLI that's already on your machine, see "[Configuring access to the CodeQL CLI](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension/configuring-access-to-the-codeql-cli)" in the GitHub documentation.
 
 If you have any difficulty with CodeQL CLI access, see the **CodeQL Extension Log** in the **Output** view for any error messages.
 
@@ -52,7 +54,7 @@ When you're working with CodeQL, you need access to the standard CodeQL librarie
 Initially, we recommend that you clone and use the ready-to-use [starter workspace](https://github.com/github/vscode-codeql-starter/).
 This includes libraries and queries for the main supported languages, with folders set up ready for your custom queries. After cloning the workspace (use `git clone --recursive`), you can use it in the same way as any other VS Code workspace—with the added advantage that you can easily update the CodeQL libraries.
 
-For information about configuring an existing workspace for CodeQL, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/setting-up-codeql-in-visual-studio-code/#updating-an-existing-workspace-for-codeql).
+For information about configuring an existing workspace for CodeQL, see "[Setting up a CodeQL workspace](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension/setting-up-a-codeql-workspace#option-2-updating-an-existing-workspace-for-codeql-advanced)" in the GitHub documentation.
 
 ## Upgrading CodeQL standard libraries
 
@@ -85,7 +87,7 @@ While you can use the [CodeQL CLI to create your own databases](https://codeql.g
 1. Select the language for the database you want to download (only required if the project has databases for multiple languages).
 1. Once the CodeQL database has been imported, it is displayed in the Databases view.
 
-For more information, see [Choosing a database](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#choosing-a-database) on codeql.github.com.
+For more information, see "[Managing CodeQL databases](https://docs.github.com/code-security/codeql-for-vs-code/getting-started-with-codeql-for-vs-code/managing-codeql-databases#choosing-a-database-to-analyze)" in the GitHub documentation.
 
 ### Running a query
 
@@ -106,12 +108,12 @@ If you wish to navigate the query results from your keyboard, you can bind short
 
 ## What next?
 
-For more information about the CodeQL extension, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/). Otherwise, you could:
+We recommend reading the [full documentation for the extension](https://docs.github.com/code-security/codeql-for-vs-code/) on the GitHub documentation website. You may also find the following resources useful:
 
 - [Create a database for a different codebase](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/).
-- [Try out variant analysis](https://help.semmle.com/QL/learn-ql/ql-training.html).
+- [Try out variant analysis](https://docs.github.com/code-security/codeql-for-vs-code/getting-started-with-codeql-for-vs-code/running-codeql-queries-at-scale-with-multi-repository-variant-analysis).
 - [Learn more about CodeQL](https://codeql.github.com/docs/).
-- [Read how security researchers use CodeQL to find CVEs](https://securitylab.github.com/research).
+- [Read how security researchers use CodeQL to find CVEs](https://github.blog/tag/github-security-lab/).
 
 ## License
 
@@ -119,4 +121,4 @@ The CodeQL extension for Visual Studio Code is [licensed](LICENSE.md) under the
 
 ## Data and Telemetry
 
-If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information about telemetry, [see the documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code).
+If you specifically opt-in to permit GitHub to do so, GitHub will collect usage data and metrics for the purposes of helping the core developers to improve the CodeQL extension for VS Code. This data will not be shared with any parties outside of GitHub. IP addresses and installation IDs will be retained for a maximum of 30 days. Anonymous data will be retained for a maximum of 180 days. For more information, see "[Telemetry in CodeQL for Visual Studio Code](https://docs.github.com/code-security/codeql-for-vs-code/using-the-advanced-functionality-of-the-codeql-for-vs-code-extension/telemetry-in-codeql-for-visual-studio-code)" in the GitHub documentation.

extensions/ql-vscode/gulpfile.ts/textmate-grammar.ts

@@ -0,0 +1,91 @@
+/**
+ * A subset of the standard TextMate grammar that is used by our transformation
+ * step. For a full JSON schema, see:
+ * https://github.com/martinring/tmlanguage/blob/478ad124a21933cd4b0b65f1ee7ee18ee1f87473/tmlanguage.json
+ */
+export interface TextmateGrammar {
+  patterns: Pattern[];
+  repository?: Record<string, Pattern>;
+}
+
+/**
+ * The extended TextMate grammar as used by our transformation step. This is a superset of the
+ * standard TextMate grammar, and includes additional fields that are used by our transformation
+ * step.
+ *
+ * Any comment of the form `(?#ref-id)` in a `match`, `begin`, or `end` property will be replaced
+ * with the match text of the rule named "ref-id". If the rule named "ref-id" consists of just a
+ * `patterns` property with a list of `include` directives, the replacement pattern is the
+ * disjunction of the match patterns of all of the included rules.
+ */
+export interface ExtendedTextmateGrammar<MatchType = string> {
+  /**
+   * This represents the set of regular expression options to apply to all regular
+   * expressions throughout the file.
+   */
+  regexOptions?: string;
+  /**
+   * This element defines a map of macro names to replacement text. When a `match`, `begin`, or
+   * `end` property has a value that is a single-key map, the value is replaced with the value of the
+   * macro named by the key, with any use of `(?#)` in the macro text replaced with the text of the
+   * value of the key, surrounded by a non-capturing group (`(?:)`). For example:
+   *
+   * The `beginPattern` and `endPattern` Properties
+   * A rule can have a `beginPattern` or `endPattern` property whose value is a reference to another
+   * rule (e.g. `#other-rule`). The `beginPattern` property is replaced as follows:
+   *
+   * my-rule:
+   *   beginPattern: '#other-rule'
+   *
+   * would be transformed to
+   *
+   * my-rule:
+   *   begin: '(?#other-rule)'
+   *   beginCaptures:
+   *     '0':
+   *       patterns:
+   *       - include: '#other-rule'
+   *
+   * An `endPattern` property is transformed similary.
+   *
+   * macros:
+   *   repeat: '(?#)*'
+   * repository:
+   *   multi-letter:
+   *     match:
+   *       repeat: '[A-Za-z]'
+   *     name: scope.multi-letter
+   *
+   * would be transformed to
+   *
+   * repository:
+   *   multi-letter:
+   *     match: '(?:[A-Za-z])*'
+   *     name: scope.multi-letter
+   */
+  macros?: Record<string, string>;
+
+  patterns: Array<Pattern<MatchType>>;
+  repository?: Record<string, Pattern<MatchType>>;
+}
+
+export interface Pattern<MatchType = string> {
+  include?: string;
+  match?: MatchType;
+  begin?: MatchType;
+  end?: MatchType;
+  while?: MatchType;
+  captures?: Record<string, PatternCapture>;
+  beginCaptures?: Record<string, PatternCapture>;
+  endCaptures?: Record<string, PatternCapture>;
+  patterns?: Array<Pattern<MatchType>>;
+  beginPattern?: string;
+  endPattern?: string;
+}
+
+export interface PatternCapture {
+  name?: string;
+  patterns?: Pattern[];
+}
+
+export type ExtendedMatchType = string | Record<string, string>;

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -3,6 +3,12 @@ import { load } from "js-yaml";
 import { obj } from "through2";
 import PluginError from "plugin-error";
 import type Vinyl from "vinyl";
+import type {
+  ExtendedMatchType,
+  ExtendedTextmateGrammar,
+  Pattern,
+  TextmateGrammar,
+} from "./textmate-grammar";
 
 /**
  * Replaces all rule references with the match pattern of the referenced rule.
@@ -34,7 +40,9 @@ function replaceReferencesWithStrings(
  * @param yaml The root of the YAML document.
  * @returns A map from macro name to replacement text.
  */
-function gatherMacros(yaml: any): Map<string, string> {
+function gatherMacros<T>(
+  yaml: ExtendedTextmateGrammar<T>,
+): Map<string, string> {
   const macros = new Map<string, string>();
   for (const key in yaml.macros) {
     macros.set(key, yaml.macros[key]);
@@ -51,7 +59,7 @@ function gatherMacros(yaml: any): Map<string, string> {
  * @returns The match text for the rule. This is either the value of the rule's `match` property,
  *  or the disjunction of the match text of all of the other rules `include`d by this rule.
  */
-function getNodeMatchText(rule: any): string {
+function getNodeMatchText(rule: Pattern): string {
   if (rule.match !== undefined) {
     // For a match string, just use that string as the replacement.
     return rule.match;
@@ -78,7 +86,7 @@ function getNodeMatchText(rule: any): string {
  * @returns A map whose keys are the names of rules, and whose values are the corresponding match
  *  text of each rule.
  */
-function gatherMatchTextForRules(yaml: any): Map<string, string> {
+function gatherMatchTextForRules(yaml: TextmateGrammar): Map<string, string> {
   const replacements = new Map<string, string>();
   for (const key in yaml.repository) {
     const node = yaml.repository[key];
@@ -94,9 +102,14 @@ function gatherMatchTextForRules(yaml: any): Map<string, string> {
  * @param yaml The root of the YAML document.
  * @param action Callback to invoke on each rule.
  */
-function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
+function visitAllRulesInFile<T>(
+  yaml: ExtendedTextmateGrammar<T>,
+  action: (rule: Pattern<T>) => void,
+) {
   visitAllRulesInRuleMap(yaml.patterns, action);
-  visitAllRulesInRuleMap(yaml.repository, action);
+  if (yaml.repository) {
+    visitAllRulesInRuleMap(Object.values(yaml.repository), action);
+  }
 }
 
 /**
@@ -107,9 +120,11 @@ function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
  * @param ruleMap The map or array of rules to visit.
  * @param action Callback to invoke on each rule.
  */
-function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
-  for (const key in ruleMap) {
-    const rule = ruleMap[key];
+function visitAllRulesInRuleMap<T>(
+  ruleMap: Array<Pattern<T>>,
+  action: (rule: Pattern<T>) => void,
+) {
+  for (const rule of ruleMap) {
     if (typeof rule === "object") {
       action(rule);
       if (rule.patterns !== undefined) {
@@ -125,16 +140,22 @@ function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
  * @param rule The rule whose matches are to be transformed.
  * @param action The transformation to make on each match pattern.
  */
-function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
+function visitAllMatchesInRule<T>(rule: Pattern<T>, action: (match: T) => T) {
   for (const key in rule) {
     switch (key) {
       case "begin":
       case "end":
       case "match":
-      case "while":
-        rule[key] = action(rule[key]);
-        break;
+      case "while": {
+        const ruleElement = rule[key];
 
+        if (!ruleElement) {
+          continue;
+        }
+
+        rule[key] = action(ruleElement);
+        break;
+      }
       default:
         break;
     }
@@ -148,14 +169,17 @@ function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
  * @param rule Rule to be transformed.
  * @param key Base key of the property to be transformed.
  */
-function expandPatternMatchProperties(rule: any, key: "begin" | "end") {
-  const patternKey = `${key}Pattern`;
-  const capturesKey = `${key}Captures`;
+function expandPatternMatchProperties<T>(
+  rule: Pattern<T>,
+  key: "begin" | "end",
+) {
+  const patternKey = `${key}Pattern` as const;
+  const capturesKey = `${key}Captures` as const;
   const pattern = rule[patternKey];
   if (pattern !== undefined) {
     const patterns: string[] = Array.isArray(pattern) ? pattern : [pattern];
-    rule[key] = patterns.map((p) => `((?${p}))`).join("|");
-    const captures: { [index: string]: any } = {};
+    rule[key] = patterns.map((p) => `((?${p}))`).join("|") as T;
+    const captures: Pattern["captures"] = {};
     for (const patternIndex in patterns) {
       captures[(Number(patternIndex) + 1).toString()] = {
         patterns: [
@@ -175,7 +199,7 @@ function expandPatternMatchProperties(rule: any, key: "begin" | "end") {
  *
  * @param yaml The root of the YAML document.
  */
-function transformFile(yaml: any) {
+function transformFile(yaml: ExtendedTextmateGrammar<ExtendedMatchType>) {
   const macros = gatherMacros(yaml);
   visitAllRulesInFile(yaml, (rule) => {
     expandPatternMatchProperties(rule, "begin");
@@ -198,24 +222,29 @@ function transformFile(yaml: any) {
 
   yaml.macros = undefined;
 
-  const replacements = gatherMatchTextForRules(yaml);
+  // We have removed all object match properties, so we don't have an extended match type anymore.
+  const macrolessYaml = yaml as ExtendedTextmateGrammar;
+
+  const replacements = gatherMatchTextForRules(macrolessYaml);
   // Expand references in matches.
-  visitAllRulesInFile(yaml, (rule) => {
+  visitAllRulesInFile(macrolessYaml, (rule) => {
     visitAllMatchesInRule(rule, (match) => {
       return replaceReferencesWithStrings(match, replacements);
     });
   });
 
-  if (yaml.regexOptions !== undefined) {
-    const regexOptions = `(?${yaml.regexOptions})`;
-    visitAllRulesInFile(yaml, (rule) => {
+  if (macrolessYaml.regexOptions !== undefined) {
+    const regexOptions = `(?${macrolessYaml.regexOptions})`;
+    visitAllRulesInFile(macrolessYaml, (rule) => {
       visitAllMatchesInRule(rule, (match) => {
         return regexOptions + match;
       });
     });
 
-    yaml.regexOptions = undefined;
+    macrolessYaml.regexOptions = undefined;
   }
+
+  return macrolessYaml;
 }
 
 export function transpileTextMateGrammar() {
@@ -230,8 +259,8 @@ export function transpileTextMateGrammar() {
       } else if (file.isBuffer()) {
         const buf: Buffer = file.contents;
         const yamlText: string = buf.toString("utf8");
-        const jsonData: any = load(yamlText);
-        transformFile(jsonData);
+        const yamlData = load(yamlText) as TextmateGrammar;
+        const jsonData = transformFile(yamlData);
 
         file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), "utf8");
         file.extname = ".json";

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.12.2",
+  "version": "1.13.2",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -14,7 +14,7 @@
   },
   "engines": {
     "vscode": "^1.82.0",
-    "node": "^18.17.1",
+    "node": "^20.9.0",
     "npm": ">=7.20.6"
   },
   "categories": [
@@ -53,10 +53,10 @@
     "configurationDefaults": {
       "[ql]": {
         "debug.saveBeforeStart": "nonUntitledEditorsInActiveGroup",
-        "editor.wordBasedSuggestions": false
+        "editor.wordBasedSuggestions": "off"
       },
       "[dbscheme]": {
-        "editor.wordBasedSuggestions": false
+        "editor.wordBasedSuggestions": "off"
       }
     },
     "debuggers": [
@@ -178,6 +178,11 @@
             "type": "string",
             "default": "",
             "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
+          },
+          "codeQL.cli.downloadTimeout": {
+            "type": "integer",
+            "default": 10,
+            "description": "Download timeout in seconds for downloading the CLI distribution."
           }
         }
       },
@@ -334,13 +339,6 @@
         "title": "Variant analysis",
         "order": 5,
         "properties": {
-          "codeQL.variantAnalysis.controllerRepo": {
-            "type": "string",
-            "default": "",
-            "pattern": "^$|^(?:[a-zA-Z0-9]+-)*[a-zA-Z0-9]+/[a-zA-Z0-9-_]+$",
-            "patternErrorMessage": "Please enter a valid GitHub repository",
-            "markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
-          },
           "codeQL.variantAnalysis.defaultResultsFilter": {
             "type": "string",
             "default": "all",
@@ -376,6 +374,11 @@
         "title": "Adding databases",
         "order": 6,
         "properties": {
+          "codeQL.addingDatabases.downloadTimeout": {
+            "type": "integer",
+            "default": 10,
+            "description": "Download timeout in seconds for adding a CodeQL database."
+          },
           "codeQL.addingDatabases.allowHttp": {
             "type": "boolean",
             "default": false,
@@ -453,8 +456,25 @@
       },
       {
         "type": "object",
-        "title": "Log insights",
+        "title": "Model Editor",
         "order": 9,
+        "properties": {
+          "codeQL.model.packLocation": {
+            "type": "string",
+            "default": ".github/codeql/extensions/${name}-${language}",
+            "markdownDescription": "Location for newly created CodeQL model packs. The location can be either absolute or relative. If relative, it is relative to the workspace root.\n\nThe following variables are supported:\n* **${database}** - the name of the database\n* **${language}** - the name of the language\n* **${name}** - the name of the GitHub repository, or the name of the database if the database was not downloaded from GitHub\n* **${owner}** - the owner of the GitHub repository, or an empty string if the database was not downloaded from GitHub"
+          },
+          "codeQL.model.packName": {
+            "type": "string",
+            "default": "${owner}/${name}-${language}",
+            "markdownDescription": "Name of newly created CodeQL model packs. If the result is not a valid pack name, it will be converted to a valid pack name.\n\nThe following variables are supported:\n* **${database}** - the name of the database\n* **${language}** - the name of the language\n* **${name}** - the name of the GitHub repository, or the name of the database if the database was not downloaded from GitHub\n* **${owner}** - the owner of the GitHub repository, or an empty string if the database was not downloaded from GitHub"
+          }
+        }
+      },
+      {
+        "type": "object",
+        "title": "Log insights",
+        "order": 10,
         "properties": {
           "codeQL.logInsights.joinOrderWarningThreshold": {
             "type": "number",
@@ -468,7 +488,7 @@
       {
         "type": "object",
         "title": "Telemetry",
-        "order": 10,
+        "order": 11,
         "properties": {
           "codeQL.telemetry.enableTelemetry": {
             "type": "boolean",
@@ -728,6 +748,10 @@
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
       },
+      {
+        "command": "codeQL.importTestDatabase",
+        "title": "CodeQL: (Re-)Import Test Database"
+      },
       {
         "command": "codeQL.getCurrentDatabase",
         "title": "CodeQL: Get Current Database"
@@ -1312,7 +1336,12 @@
         {
           "command": "codeQL.setCurrentDatabase",
           "group": "9_qlCommands",
-          "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
+          "when": "resourceExtname != .testproj && (resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zipz)"
+        },
+        {
+          "command": "codeQL.importTestDatabase",
+          "group": "9_qlCommands",
+          "when": "explorerResourceIsFolder && resourceExtname == .testproj"
         },
         {
           "command": "codeQL.viewAstContextExplorer",
@@ -1432,7 +1461,7 @@
         },
         {
           "command": "codeQL.quickEvalCount",
-          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+          "when": "editorLangId == ql"
         },
         {
           "command": "codeQL.quickEvalContextEditor",
@@ -1466,6 +1495,10 @@
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQL.importTestDatabase",
+          "when": "false"
+        },
         {
           "command": "codeQL.getCurrentDatabase",
           "when": "false"
@@ -1891,11 +1924,6 @@
       {
         "view": "codeQLEvalLogViewer",
         "contents": "Run the 'Show Evaluator Log (UI)' command on a CodeQL query run in the Query History view."
-      },
-      {
-        "view": "codeQLVariantAnalysisRepositories",
-        "contents": "Set up a controller repository to start using variant analysis. [Learn more](https://codeql.github.com/docs/codeql-for-visual-studio-code/running-codeql-queries-at-scale-with-mrva#controller-repository) about controller repositories. \n[Set up controller repository](command:codeQLVariantAnalysisRepositories.setupControllerRepository)",
-        "when": "!config.codeQL.variantAnalysis.controllerRepo"
       }
     ]
   },
@@ -1915,44 +1943,45 @@
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",
     "lint:markdown": "markdownlint-cli2 \"../../**/*.{md,mdx}\" \"!**/node_modules/**\" \"!**/.vscode-test/**\" \"!**/build/cli/v*/**\"",
-    "find-deadcode": "ts-node scripts/find-deadcode.ts",
+    "find-deadcode": "vite-node scripts/find-deadcode.ts",
     "format-staged": "lint-staged",
     "storybook": "storybook dev -p 6006",
     "build-storybook": "storybook build",
-    "lint:scenarios": "ts-node scripts/lint-scenarios.ts",
+    "lint:scenarios": "vite-node scripts/lint-scenarios.ts",
     "generate": "npm-run-all -p generate:*",
-    "generate:schemas": "ts-node scripts/generate-schemas.ts",
-    "generate:chromium-version": "ts-node scripts/generate-chromium-version.ts",
+    "generate:schemas": "vite-node scripts/generate-schemas.ts",
+    "generate:chromium-version": "vite-node scripts/generate-chromium-version.ts",
     "check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
     "postinstall": "patch-package",
     "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
-    "@floating-ui/react": "^0.26.9",
+    "@floating-ui/react": "^0.26.12",
     "@octokit/plugin-retry": "^6.0.1",
+    "@octokit/plugin-throttling": "^8.0.0",
     "@octokit/rest": "^20.0.2",
     "@vscode/codicons": "^0.0.35",
     "@vscode/debugadapter": "^1.59.0",
-    "@vscode/debugprotocol": "^1.59.0",
+    "@vscode/debugprotocol": "^1.65.0",
     "@vscode/webview-ui-toolkit": "^1.0.1",
     "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
-    "chokidar": "^3.5.3",
-    "d3": "^7.6.1",
+    "chokidar": "^3.6.0",
+    "d3": "^7.9.0",
     "d3-graphviz": "^5.0.2",
     "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",
-    "msw": "^2.0.13",
-    "nanoid": "^5.0.1",
+    "msw": "^2.2.13",
+    "nanoid": "^5.0.7",
     "node-fetch": "^2.6.7",
     "p-queue": "^8.0.1",
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
-    "semver": "^7.6.0",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "semver": "^7.6.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
     "stream-json": "^1.7.3",
-    "styled-components": "^6.1.8",
+    "styled-components": "^6.1.9",
     "tmp": "^0.2.1",
     "tmp-promise": "^3.0.2",
     "tree-kill": "^1.2.2",
@@ -1963,29 +1992,30 @@
     "zip-a-folder": "^3.1.6"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.13",
+    "@babel/core": "^7.24.6",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
-    "@babel/preset-env": "^7.23.9",
-    "@babel/preset-react": "^7.18.6",
+    "@babel/preset-env": "^7.24.4",
+    "@babel/preset-react": "^7.24.1",
     "@babel/preset-typescript": "^7.21.4",
-    "@faker-js/faker": "^8.0.2",
-    "@github/markdownlint-github": "^0.6.0",
-    "@octokit/plugin-throttling": "^8.0.0",
+    "@faker-js/faker": "^8.4.1",
+    "@github/markdownlint-github": "^0.6.2",
     "@playwright/test": "^1.40.1",
-    "@storybook/addon-a11y": "^7.6.13",
-    "@storybook/addon-actions": "^7.1.0",
-    "@storybook/addon-essentials": "^7.1.0",
-    "@storybook/addon-interactions": "^7.1.0",
-    "@storybook/addon-links": "^7.1.0",
-    "@storybook/components": "^7.6.7",
-    "@storybook/csf": "^0.1.1",
-    "@storybook/manager-api": "^7.6.7",
-    "@storybook/react": "^7.1.0",
-    "@storybook/react-webpack5": "^7.6.12",
-    "@storybook/theming": "^7.6.12",
-    "@testing-library/dom": "^9.3.4",
-    "@testing-library/jest-dom": "^6.2.0",
-    "@testing-library/react": "^14.0.0",
+    "@storybook/addon-a11y": "^8.1.10",
+    "@storybook/addon-actions": "^8.1.10",
+    "@storybook/addon-essentials": "^8.1.10",
+    "@storybook/addon-interactions": "^8.1.10",
+    "@storybook/addon-links": "^8.1.10",
+    "@storybook/blocks": "^8.0.2",
+    "@storybook/components": "^8.0.2",
+    "@storybook/csf": "^0.1.8",
+    "@storybook/icons": "^1.2.9",
+    "@storybook/manager-api": "^8.1.10",
+    "@storybook/react": "^8.1.10",
+    "@storybook/react-vite": "^8.1.10",
+    "@storybook/theming": "^8.1.10",
+    "@testing-library/dom": "^10.1.0",
+    "@testing-library/jest-dom": "^6.4.6",
+    "@testing-library/react": "^16.0.0",
     "@testing-library/user-event": "^14.5.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/d3": "^7.4.0",
@@ -1994,71 +2024,68 @@
     "@types/fs-extra": "^11.0.1",
     "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "^1.1.0",
-    "@types/jest": "^29.0.2",
+    "@types/jest": "^29.5.12",
     "@types/js-yaml": "^4.0.6",
     "@types/nanoid": "^3.0.0",
-    "@types/node": "18.17.*",
+    "@types/node": "20.9.*",
     "@types/node-fetch": "^2.5.2",
-    "@types/react": "^18.0.28",
-    "@types/react-dom": "^18.2.18",
+    "@types/react": "^18.3.1",
+    "@types/react-dom": "^18.3.0",
     "@types/sarif": "^2.1.2",
-    "@types/semver": "^7.2.0",
+    "@types/semver": "^7.5.8",
     "@types/stream-json": "^1.7.1",
     "@types/styled-components": "^5.1.11",
     "@types/tar-stream": "^3.1.3",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.2.6",
-    "@types/unzipper": "^0.10.1",
     "@types/vscode": "^1.82.0",
     "@types/yauzl": "^2.10.3",
-    "@typescript-eslint/eslint-plugin": "^6.19.0",
-    "@typescript-eslint/parser": "^6.16.0",
-    "@vscode/test-electron": "^2.2.0",
-    "@vscode/vsce": "^2.19.0",
+    "@typescript-eslint/eslint-plugin": "^7.5.0",
+    "@typescript-eslint/parser": "^7.5.0",
+    "@vscode/test-electron": "^2.3.9",
+    "@vscode/vsce": "^2.24.0",
     "ansi-colors": "^4.1.1",
-    "applicationinsights": "^2.9.2",
+    "applicationinsights": "^2.9.5",
     "cosmiconfig": "^9.0.0",
     "cross-env": "^7.0.3",
-    "css-loader": "^6.10.0",
     "del": "^6.0.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-deprecation": "^2.0.0",
     "eslint-plugin-etc": "^2.0.2",
-    "eslint-plugin-github": "^4.4.1",
+    "eslint-plugin-github": "^4.10.2",
     "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jest-dom": "^5.0.1",
+    "eslint-plugin-jest-dom": "^5.2.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "eslint-plugin-react": "^7.31.8",
+    "eslint-plugin-react": "^7.34.1",
     "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-storybook": "^0.6.4",
-    "file-loader": "^6.2.0",
+    "eslint-plugin-storybook": "^0.8.0",
     "glob": "^10.0.0",
     "gulp": "^4.0.2",
     "gulp-esbuild": "^0.12.0",
     "gulp-replace": "^1.1.3",
     "gulp-typescript": "^5.0.1",
-    "husky": "^8.0.0",
+    "husky": "^9.1.7",
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "^15.0.2",
-    "markdownlint-cli2": "^0.12.1",
-    "markdownlint-cli2-formatter-pretty": "^0.0.5",
-    "mini-css-extract-plugin": "^2.7.7",
+    "lint-staged": "^15.2.2",
+    "markdownlint-cli2": "^0.13.0",
+    "markdownlint-cli2-formatter-pretty": "^0.0.6",
     "npm-run-all": "^4.1.5",
     "patch-package": "^8.0.0",
     "prettier": "^3.2.5",
-    "storybook": "^7.6.10",
-    "tar-stream": "^3.0.0",
+    "storybook": "^8.1.10",
+    "tar-stream": "^3.1.7",
     "through2": "^4.0.2",
-    "ts-jest": "^29.0.1",
-    "ts-json-schema-generator": "^1.1.2",
-    "ts-loader": "^9.4.2",
-    "ts-node": "^10.7.0",
-    "ts-unused-exports": "^10.0.0",
-    "typescript": "^5.0.2"
+    "ts-jest": "^29.1.4",
+    "ts-json-schema-generator": "^2.1.1",
+    "ts-node": "^10.9.2",
+    "ts-unused-exports": "^10.1.0",
+    "typescript": "^5.0.2",
+    "vite": "^5.2.11",
+    "vite-node": "^1.5.3"
   },
   "lint-staged": {
     "./**/*.{json,css,scss}": [

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -1,3 +1,4 @@
+import type { WriteStream } from "fs";
 import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
@@ -26,6 +27,8 @@ import { unzipToDirectoryConcurrently } from "../common/unzip-concurrently";
 import { reportUnzipProgress } from "../common/vscode/unzip-progress";
 import type { Release } from "./distribution/release";
 import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";
+import { createTimeoutSignal } from "../common/fetch-stream";
+import { AbortError } from "node-fetch";
 
 /**
  * distribution.ts
@@ -384,15 +387,25 @@ class ExtensionSpecificDistributionManager {
       );
     }
 
-    const assetStream =
-      await this.createReleasesApiConsumer().streamBinaryContentOfAsset(
-        assets[0],
-      );
+    const {
+      signal,
+      onData,
+      dispose: disposeTimeout,
+    } = createTimeoutSignal(this.config.downloadTimeout);
+
     const tmpDirectory = await mkdtemp(join(tmpdir(), "vscode-codeql"));
 
+    let archiveFile: WriteStream | undefined = undefined;
+
     try {
+      const assetStream =
+        await this.createReleasesApiConsumer().streamBinaryContentOfAsset(
+          assets[0],
+          signal,
+        );
+
       const archivePath = join(tmpDirectory, "distributionDownload.zip");
-      const archiveFile = createWriteStream(archivePath);
+      archiveFile = createWriteStream(archivePath);
 
       const contentLength = assetStream.headers.get("content-length");
       const totalNumBytes = contentLength
@@ -405,12 +418,23 @@ class ExtensionSpecificDistributionManager {
         progressCallback,
       );
 
-      await new Promise((resolve, reject) =>
+      assetStream.body.on("data", onData);
+
+      await new Promise((resolve, reject) => {
+        if (!archiveFile) {
+          throw new Error("Invariant violation: archiveFile not set");
+        }
+
         assetStream.body
           .pipe(archiveFile)
           .on("finish", resolve)
-          .on("error", reject),
-      );
+          .on("error", reject);
+
+        // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
+        assetStream.body.on("error", reject);
+      });
+
+      disposeTimeout();
 
       await this.bumpDistributionFolderIndex();
 
@@ -427,7 +451,19 @@ class ExtensionSpecificDistributionManager {
             )
           : undefined,
       );
+    } catch (e) {
+      if (e instanceof AbortError) {
+        const thrownError = new AbortError("The download timed out.");
+        thrownError.stack = e.stack;
+        throw thrownError;
+      }
+
+      throw e;
     } finally {
+      disposeTimeout();
+
+      archiveFile?.close();
+
       await remove(tmpDirectory);
     }
   }

extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts

@@ -90,21 +90,28 @@ export class ReleasesApiConsumer {
 
   public async streamBinaryContentOfAsset(
     asset: ReleaseAsset,
+    signal?: AbortSignal,
   ): Promise<Response> {
     const apiPath = `/repos/${this.repositoryNwo}/releases/assets/${asset.id}`;
 
-    return await this.makeApiCall(apiPath, {
-      accept: "application/octet-stream",
-    });
+    return await this.makeApiCall(
+      apiPath,
+      {
+        accept: "application/octet-stream",
+      },
+      signal,
+    );
   }
 
   protected async makeApiCall(
     apiPath: string,
     additionalHeaders: { [key: string]: string } = {},
+    signal?: AbortSignal,
   ): Promise<Response> {
     const response = await this.makeRawRequest(
       ReleasesApiConsumer.apiBase + apiPath,
       Object.assign({}, this.defaultHeaders, additionalHeaders),
+      signal,
     );
 
     if (!response.ok) {
@@ -129,11 +136,13 @@ export class ReleasesApiConsumer {
   private async makeRawRequest(
     requestUrl: string,
     headers: { [key: string]: string },
+    signal?: AbortSignal,
     redirectCount = 0,
   ): Promise<Response> {
     const response = await fetch(requestUrl, {
       headers,
       redirect: "manual",
+      signal,
     });
 
     const redirectUrl = response.headers.get("location");
@@ -153,7 +162,12 @@ export class ReleasesApiConsumer {
         // mechanism is provided.
         delete headers["authorization"];
       }
-      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
+      return await this.makeRawRequest(
+        redirectUrl,
+        headers,
+        signal,
+        redirectCount + 1,
+      );
     }
 
     return response;

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -1,5 +1,5 @@
 import type { CodeQLCliServer } from "./cli";
-import type { Uri } from "vscode";
+import type { CancellationToken, Uri } from "vscode";
 import { window } from "vscode";
 import {
   getLanguageDisplayName,
@@ -50,6 +50,7 @@ export async function findLanguage(
 export async function askForLanguage(
   cliServer: CodeQLCliServer,
   throwOnEmpty = true,
+  token?: CancellationToken,
 ): Promise<QueryLanguage | undefined> {
   const supportedLanguages = await cliServer.getSupportedLanguages();
 
@@ -62,10 +63,14 @@ export async function askForLanguage(
     }))
     .sort((a, b) => a.label.localeCompare(b.label));
 
-  const selectedItem = await window.showQuickPick(items, {
-    placeHolder: "Select target query language",
-    ignoreFocusOut: true,
-  });
+  const selectedItem = await window.showQuickPick(
+    items,
+    {
+      placeHolder: "Select target query language",
+      ignoreFocusOut: true,
+    },
+    token,
+  );
   if (!selectedItem) {
     // This only happens if the user cancels the quick pick.
     if (throwOnEmpty) {

extensions/ql-vscode/src/common/authentication.ts

@@ -31,4 +31,9 @@ export interface Credentials {
    * @returns An OAuth access token, or undefined.
    */
   getExistingAccessToken(): Promise<string | undefined>;
+
+  /**
+   * Returns the ID of the authentication provider to use.
+   */
+  authProviderId: string;
 }

extensions/ql-vscode/src/common/commands.ts

@@ -220,6 +220,7 @@ export type LocalDatabasesCommands = {
 
   // Explorer context menu
   "codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
+  "codeQL.importTestDatabase": (uri: Uri) => Promise<void>;
 
   // Database panel view title commands
   "codeQLDatabases.chooseDatabaseFolder": () => Promise<void>;
@@ -272,6 +273,9 @@ export type VariantAnalysisCommands = {
   "codeQL.openVariantAnalysisLogs": (
     variantAnalysisId: number,
   ) => Promise<void>;
+  "codeQLModelAlerts.openVariantAnalysisLogs": (
+    variantAnalysisId: number,
+  ) => Promise<void>;
   "codeQL.openVariantAnalysisView": (
     variantAnalysisId: number,
   ) => Promise<void>;
@@ -286,7 +290,6 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openConfigFile": () => Promise<void>;
   "codeQLVariantAnalysisRepositories.addNewDatabase": () => Promise<void>;
   "codeQLVariantAnalysisRepositories.addNewList": () => Promise<void>;
-  "codeQLVariantAnalysisRepositories.setupControllerRepository": () => Promise<void>;
 
   "codeQLVariantAnalysisRepositories.setSelectedItem": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;

extensions/ql-vscode/src/common/config-template.ts

@@ -0,0 +1,57 @@
+// Based on https://github.com/microsoft/vscode/blob/edfd5b8ba54d50f3f5c2ebee877af088803def88/src/vs/base/common/labels.ts#L316C1-L400
+
+/**
+ * Helper to insert values for specific template variables into the string. E.g. "this ${is} a ${template}" can be
+ * passed to this function together with an object that maps "is" and "template" to strings to have them replaced.
+ *
+ * @param template string to which template is applied
+ * @param values the values of the templates to use
+ */
+export function substituteConfigVariables(
+  template: string,
+  values: {
+    [key: string]: string | undefined | null;
+  },
+): string {
+  const segments: string[] = [];
+
+  let inVariable = false;
+  let currentValue = "";
+  for (const char of template) {
+    // Beginning of variable
+    if (char === "$" || (inVariable && char === "{")) {
+      if (currentValue) {
+        segments.push(currentValue);
+      }
+
+      currentValue = "";
+      inVariable = true;
+    }
+
+    // End of variable
+    else if (char === "}" && inVariable) {
+      const resolved = values[currentValue];
+
+      // Variable
+      if (resolved && resolved.length > 0) {
+        segments.push(resolved);
+      }
+      // If the variable, doesn't exist, we discard it (i.e. replace it by the empty string)
+
+      currentValue = "";
+      inVariable = false;
+    }
+
+    // Text or Variable Name
+    else {
+      currentValue += char;
+    }
+  }
+
+  // Tail
+  if (currentValue && !inVariable) {
+    segments.push(currentValue);
+  }
+
+  return segments.join("");
+}

extensions/ql-vscode/src/common/fetch-stream.ts

@@ -0,0 +1,36 @@
+import { clearTimeout } from "node:timers";
+
+export function createTimeoutSignal(timeoutSeconds: number): {
+  signal: AbortSignal;
+  onData: () => void;
+  dispose: () => void;
+} {
+  const timeout = timeoutSeconds * 1000;
+
+  const abortController = new AbortController();
+
+  let timeoutId: NodeJS.Timeout;
+
+  // If we don't get any data within the timeout, abort the download
+  timeoutId = setTimeout(() => {
+    abortController.abort();
+  }, timeout);
+
+  // If we receive any data within the timeout, reset the timeout
+  const onData = () => {
+    clearTimeout(timeoutId);
+    timeoutId = setTimeout(() => {
+      abortController.abort();
+    }, timeout);
+  };
+
+  const dispose = () => {
+    clearTimeout(timeoutId);
+  };
+
+  return {
+    signal: abortController.signal,
+    onData,
+    dispose,
+  };
+}

extensions/ql-vscode/src/common/filenames.ts

@@ -0,0 +1,44 @@
+type FilenameOptions = {
+  removeDots?: boolean;
+};
+
+/**
+ * This will create a filename from an arbitrary string by removing
+ * all characters which are not allowed in filenames and making them
+ * more filesystem-friendly be replacing undesirable characters with
+ * hyphens. The result will always be lowercase ASCII.
+ *
+ * @param str The string to create a filename from
+ * @param removeDots Whether to remove dots from the filename [default: false]
+ * @returns The filename
+ */
+export function createFilenameFromString(
+  str: string,
+  { removeDots }: FilenameOptions = {},
+) {
+  let fileName = str;
+
+  // Lowercase everything
+  fileName = fileName.toLowerCase();
+
+  // Replace all spaces, underscores, slashes, and backslashes with hyphens
+  fileName = fileName.replaceAll(/[\s_/\\]+/g, "-");
+
+  // Replace all characters which are not allowed by empty strings
+  fileName = fileName.replaceAll(/[^a-z0-9.-]/g, "");
+
+  // Remove any leading or trailing hyphens or dots
+  fileName = fileName.replaceAll(/^[.-]+|[.-]+$/g, "");
+
+  // Replace dots by hyphens if dots are not allowed
+  if (removeDots) {
+    fileName = fileName.replaceAll(/\./g, "-");
+  }
+
+  // Remove any duplicate hyphens
+  fileName = fileName.replaceAll(/-{2,}/g, "-");
+  // Remove any duplicate dots
+  fileName = fileName.replaceAll(/\.{2,}/g, ".");
+
+  return fileName;
+}

extensions/ql-vscode/src/common/files.ts

@@ -176,3 +176,32 @@ export function findCommonParentDir(...paths: string[]): string {
 function isTopLevelPath(path: string): boolean {
   return dirname(path) === path;
 }
+
+/**
+ * Recursively looks for a file in a directory. If the file exists, then returns the directory containing the file.
+ *
+ * @param dir The directory to search
+ * @param toFind The file to recursively look for in this directory
+ *
+ * @returns the directory containing the file, or undefined if not found.
+ */
+export async function findDirWithFile(
+  dir: string,
+  ...toFind: string[]
+): Promise<string | undefined> {
+  if (!(await stat(dir)).isDirectory()) {
+    return;
+  }
+  const files = await readdir(dir);
+  if (toFind.some((file) => files.includes(file))) {
+    return dir;
+  }
+  for (const file of files) {
+    const newPath = join(dir, file);
+    const result = await findDirWithFile(newPath, ...toFind);
+    if (result) {
+      return result;
+    }
+  }
+  return;
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -29,37 +29,45 @@ function validGitHubNwoOrOwner(
 
 /**
  * Extracts an NWO from a GitHub URL.
- * @param githubUrl The GitHub repository URL
+ * @param repositoryUrl The GitHub repository URL
+ * @param githubUrl The URL of the GitHub instance
  * @return The corresponding NWO, or undefined if the URL is not valid
  */
-export function getNwoFromGitHubUrl(githubUrl: string): string | undefined {
-  return getNwoOrOwnerFromGitHubUrl(githubUrl, "nwo");
+export function getNwoFromGitHubUrl(
+  repositoryUrl: string,
+  githubUrl: URL,
+): string | undefined {
+  return getNwoOrOwnerFromGitHubUrl(repositoryUrl, githubUrl, "nwo");
 }
 
 /**
  * Extracts an owner from a GitHub URL.
- * @param githubUrl The GitHub repository URL
+ * @param repositoryUrl The GitHub repository URL
+ * @param githubUrl The URL of the GitHub instance
  * @return The corresponding Owner, or undefined if the URL is not valid
  */
-export function getOwnerFromGitHubUrl(githubUrl: string): string | undefined {
-  return getNwoOrOwnerFromGitHubUrl(githubUrl, "owner");
+export function getOwnerFromGitHubUrl(
+  repositoryUrl: string,
+  githubUrl: URL,
+): string | undefined {
+  return getNwoOrOwnerFromGitHubUrl(repositoryUrl, githubUrl, "owner");
 }
 
 function getNwoOrOwnerFromGitHubUrl(
-  githubUrl: string,
+  repositoryUrl: string,
+  githubUrl: URL,
   kind: "owner" | "nwo",
 ): string | undefined {
+  const validHostnames = [githubUrl.hostname, `www.${githubUrl.hostname}`];
+
   try {
     let paths: string[];
-    const urlElements = githubUrl.split("/");
-    if (
-      urlElements[0] === "github.com" ||
-      urlElements[0] === "www.github.com"
-    ) {
-      paths = githubUrl.split("/").slice(1);
+    const urlElements = repositoryUrl.split("/");
+    if (validHostnames.includes(urlElements[0])) {
+      paths = repositoryUrl.split("/").slice(1);
     } else {
-      const uri = new URL(githubUrl);
-      if (uri.hostname !== "github.com" && uri.hostname !== "www.github.com") {
+      const uri = new URL(repositoryUrl);
+      if (!validHostnames.includes(uri.hostname)) {
         return;
       }
       paths = uri.pathname.split("/").filter((segment: string) => segment);

extensions/ql-vscode/src/common/interface-types.ts

@@ -10,10 +10,11 @@ import type {
 } from "../variant-analysis/shared/variant-analysis-filter-sort";
 import type { ErrorLike } from "../common/errors";
 import type { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
-import type { Method } from "../model-editor/method";
+import type { Method, MethodSignature } from "../model-editor/method";
 import type { ModeledMethod } from "../model-editor/modeled-method";
 import type {
   MethodModelingPanelViewState,
+  ModelAlertsViewState,
   ModelEditorViewState,
 } from "../model-editor/shared/view-state";
 import type { Mode } from "../model-editor/shared/mode";
@@ -25,6 +26,7 @@ import type {
   UrlValueResolvable,
 } from "./raw-result-types";
 import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
+import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
 
 /**
  * This module contains types and code that are shared between
@@ -528,9 +530,10 @@ interface SetMethodsMessage {
   methods: Method[];
 }
 
-interface SetModeledMethodsMessage {
-  t: "setModeledMethods";
+interface SetModeledAndModifiedMethodsMessage {
+  t: "setModeledAndModifiedMethods";
   methods: Record<string, ModeledMethod[]>;
+  modifiedMethodSignatures: string[];
 }
 
 interface SetModifiedMethodsMessage {
@@ -543,6 +546,11 @@ interface SetInProgressMethodsMessage {
   methods: string[];
 }
 
+interface SetProcessedByAutoModelMethodsMessage {
+  t: "setProcessedByAutoModelMethods";
+  methods: string[];
+}
+
 interface SwitchModeMessage {
   t: "switchMode";
   mode: Mode;
@@ -585,6 +593,23 @@ interface StopGeneratingMethodsFromLlmMessage {
   packageName: string;
 }
 
+interface StartModelEvaluationMessage {
+  t: "startModelEvaluation";
+}
+
+interface StopModelEvaluationMessage {
+  t: "stopModelEvaluation";
+}
+
+interface OpenModelAlertsViewMessage {
+  t: "openModelAlertsView";
+}
+
+interface RevealInModelAlertsViewMessage {
+  t: "revealInModelAlertsView";
+  modeledMethod: ModeledMethod;
+}
+
 interface ModelDependencyMessage {
   t: "modelDependency";
 }
@@ -610,6 +635,11 @@ interface SetInProgressMessage {
   inProgress: boolean;
 }
 
+interface SetProcessedByAutoModelMessage {
+  t: "setProcessedByAutoModel";
+  processedByAutoModel: boolean;
+}
+
 interface RevealMethodMessage {
   t: "revealMethod";
   methodSignature: string;
@@ -620,14 +650,21 @@ interface SetAccessPathSuggestionsMessage {
   accessPathSuggestions: AccessPathSuggestionOptions;
 }
 
+interface SetModelEvaluationRunMessage {
+  t: "setModelEvaluationRun";
+  run: ModelEvaluationRunState | undefined;
+}
+
 export type ToModelEditorMessage =
   | SetExtensionPackStateMessage
   | SetMethodsMessage
-  | SetModeledMethodsMessage
+  | SetModeledAndModifiedMethodsMessage
   | SetModifiedMethodsMessage
   | SetInProgressMethodsMessage
+  | SetProcessedByAutoModelMethodsMessage
   | RevealMethodMessage
-  | SetAccessPathSuggestionsMessage;
+  | SetAccessPathSuggestionsMessage
+  | SetModelEvaluationRunMessage;
 
 export type FromModelEditorMessage =
   | CommonFromViewMessages
@@ -642,11 +679,15 @@ export type FromModelEditorMessage =
   | StopGeneratingMethodsFromLlmMessage
   | ModelDependencyMessage
   | HideModeledMethodsMessage
-  | SetMultipleModeledMethodsMessage;
+  | SetMultipleModeledMethodsMessage
+  | StartModelEvaluationMessage
+  | StopModelEvaluationMessage
+  | OpenModelAlertsViewMessage
+  | RevealInModelAlertsViewMessage;
 
 interface RevealInEditorMessage {
   t: "revealInModelEditor";
-  method: Method;
+  method: MethodSignature;
 }
 
 interface StartModelingMessage {
@@ -664,29 +705,67 @@ interface SetMethodModelingPanelViewStateMessage {
   viewState: MethodModelingPanelViewState;
 }
 
-interface SetMethodMessage {
-  t: "setMethod";
-  method: Method | undefined;
-}
-
 interface SetMethodModifiedMessage {
   t: "setMethodModified";
   isModified: boolean;
 }
 
+interface SetNoMethodSelectedMessage {
+  t: "setNoMethodSelected";
+}
+
 interface SetSelectedMethodMessage {
   t: "setSelectedMethod";
   method: Method;
   modeledMethods: ModeledMethod[];
   isModified: boolean;
   isInProgress: boolean;
+  processedByAutoModel: boolean;
 }
 
 export type ToMethodModelingMessage =
   | SetMethodModelingPanelViewStateMessage
-  | SetMethodMessage
   | SetMultipleModeledMethodsMessage
   | SetMethodModifiedMessage
+  | SetNoMethodSelectedMessage
   | SetSelectedMethodMessage
   | SetInModelingModeMessage
-  | SetInProgressMessage;
+  | SetInProgressMessage
+  | SetProcessedByAutoModelMessage;
+
+interface SetModelAlertsViewStateMessage {
+  t: "setModelAlertsViewState";
+  viewState: ModelAlertsViewState;
+}
+
+interface OpenModelPackMessage {
+  t: "openModelPack";
+  path: string;
+}
+
+interface OpenActionsLogsMessage {
+  t: "openActionsLogs";
+  variantAnalysisId: number;
+}
+
+interface StopEvaluationRunMessage {
+  t: "stopEvaluationRun";
+}
+
+interface RevealModelMessage {
+  t: "revealModel";
+  modeledMethod: ModeledMethod;
+}
+
+export type ToModelAlertsMessage =
+  | SetModelAlertsViewStateMessage
+  | SetVariantAnalysisMessage
+  | SetRepoResultsMessage
+  | RevealModelMessage;
+
+export type FromModelAlertsMessage =
+  | CommonFromViewMessages
+  | OpenModelPackMessage
+  | OpenActionsLogsMessage
+  | StopEvaluationRunMessage
+  | RevealInEditorMessage;

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -1,7 +1,10 @@
-import { appendFile, ensureFile } from "fs-extra";
+import { ensureFile } from "fs-extra";
+import { open } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
 import { isAbsolute } from "path";
 import { getErrorMessage } from "../helpers-pure";
 import type { Logger, LogOptions } from "./logger";
+import type { Disposable } from "../disposable-object";
 
 /**
  * An implementation of {@link Logger} that sends the output both to another {@link Logger}
@@ -10,9 +13,10 @@ import type { Logger, LogOptions } from "./logger";
  * The first time a message is written, an additional banner is written to the underlying logger
  * pointing the user to the "side log" file.
  */
-export class TeeLogger implements Logger {
+export class TeeLogger implements Logger, Disposable {
   private emittedRedirectMessage = false;
   private error = false;
+  private fileHandle: FileHandle | undefined = undefined;
 
   public constructor(
     private readonly logger: Logger,
@@ -37,11 +41,15 @@ export class TeeLogger implements Logger {
 
     if (!this.error) {
       try {
-        const trailingNewline = options.trailingNewline ?? true;
-        await ensureFile(this.location);
+        if (!this.fileHandle) {
+          await ensureFile(this.location);
 
-        await appendFile(
-          this.location,
+          this.fileHandle = await open(this.location, "a");
+        }
+
+        const trailingNewline = options.trailingNewline ?? true;
+
+        await this.fileHandle.appendFile(
           message + (trailingNewline ? "\n" : ""),
           {
             encoding: "utf8",
@@ -50,6 +58,14 @@ export class TeeLogger implements Logger {
       } catch (e) {
         // Write an error message to the primary log, and stop trying to write to the side log.
         this.error = true;
+        try {
+          await this.fileHandle?.close();
+        } catch (e) {
+          void this.logger.log(
+            `Failed to close file handle: ${getErrorMessage(e)}`,
+          );
+        }
+        this.fileHandle = undefined;
         const errorMessage = getErrorMessage(e);
         await this.logger.log(
           `Error writing to additional log file: ${errorMessage}`,
@@ -65,4 +81,15 @@ export class TeeLogger implements Logger {
   show(preserveFocus?: boolean): void {
     this.logger.show(preserveFocus);
   }
+
+  dispose(): void {
+    try {
+      void this.fileHandle?.close();
+    } catch (e) {
+      void this.logger.log(
+        `Failed to close file handle: ${getErrorMessage(e)}`,
+      );
+    }
+    this.fileHandle = undefined;
+  }
 }

extensions/ql-vscode/src/common/model-pack-details.ts

@@ -0,0 +1,4 @@
+export interface ModelPackDetails {
+  name: string;
+  path: string;
+}

extensions/ql-vscode/src/common/query-metadata.ts

@@ -0,0 +1,17 @@
+export const SARIF_RESULTS_QUERY_KINDS = [
+  "problem",
+  "alert",
+  "path-problem",
+  "path-alert",
+];
+
+/**
+ * Returns whether this query kind supports producing SARIF results.
+ */
+export function isSarifResultsQueryKind(kind: string | undefined): boolean {
+  if (!kind) {
+    return false;
+  }
+
+  return SARIF_RESULTS_QUERY_KINDS.includes(kind);
+}

extensions/ql-vscode/src/common/vscode/abstract-webview-view-provider.ts

@@ -1,9 +1,4 @@
-import type {
-  CancellationToken,
-  WebviewView,
-  WebviewViewProvider,
-  WebviewViewResolveContext,
-} from "vscode";
+import type { WebviewView, WebviewViewProvider } from "vscode";
 import { Uri } from "vscode";
 import type { WebviewKind, WebviewMessage } from "./webview-html";
 import { getHtmlForWebview } from "./webview-html";
@@ -28,11 +23,7 @@ export abstract class AbstractWebviewViewProvider<
    * This is called when a view first becomes visible. This may happen when the view is
    * first loaded or when the user hides and then shows a view again.
    */
-  public resolveWebviewView(
-    webviewView: WebviewView,
-    _context: WebviewViewResolveContext,
-    _token: CancellationToken,
-  ) {
+  public resolveWebviewView(webviewView: WebviewView) {
     webviewView.webview.options = {
       enableScripts: true,
       localResourceRoots: [Uri.file(this.app.extensionPath)],

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -13,6 +13,8 @@ import { tmpDir } from "../../tmp-dir";
 import type { WebviewMessage, WebviewKind } from "./webview-html";
 import { getHtmlForWebview } from "./webview-html";
 import type { DeepReadonly } from "../readonly";
+import { runWithErrorHandling } from "./error-handling";
+import { telemetryListener } from "./telemetry";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -117,7 +119,12 @@ export abstract class AbstractWebview<
     );
     this.push(
       panel.webview.onDidReceiveMessage(
-        async (e) => this.onMessage(e),
+        async (e) =>
+          runWithErrorHandling(
+            () => this.onMessage(e),
+            this.app.logger,
+            telemetryListener,
+          ),
         undefined,
       ),
     );

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -26,6 +26,8 @@ import {
 // All path operations in this file must be on paths *within* the zip
 // archive.
 import { posix } from "path";
+import { DatabaseEventKind } from "../../databases/local-databases/database-events";
+import type { DatabaseManager } from "../../databases/local-databases/database-manager";
 
 const path = posix;
 
@@ -242,6 +244,10 @@ export class ArchiveFileSystemProvider implements FileSystemProvider {
 
   root = new Directory("");
 
+  flushCache(zipPath: string) {
+    this.archives.delete(zipPath);
+  }
+
   // metadata
 
   async stat(uri: Uri): Promise<FileStat> {
@@ -355,15 +361,35 @@ export class ArchiveFileSystemProvider implements FileSystemProvider {
  */
 export const zipArchiveScheme = "codeql-zip-archive";
 
-export function activate(ctx: ExtensionContext) {
+export function activate(ctx: ExtensionContext, dbm?: DatabaseManager) {
+  const afsp = new ArchiveFileSystemProvider();
+  if (dbm) {
+    ctx.subscriptions.push(
+      dbm.onDidChangeDatabaseItem(async ({ kind, item: db }) => {
+        if (kind === DatabaseEventKind.Remove) {
+          if (db?.sourceArchive) {
+            afsp.flushCache(db.sourceArchive.fsPath);
+          }
+        }
+      }),
+    );
+  }
+
   ctx.subscriptions.push(
-    workspace.registerFileSystemProvider(
-      zipArchiveScheme,
-      new ArchiveFileSystemProvider(),
-      {
-        isCaseSensitive: true,
-        isReadonly: true,
-      },
-    ),
+    // When a file system archive is removed from the workspace, we should
+    // also remove it from our cache.
+    workspace.onDidChangeWorkspaceFolders((event) => {
+      for (const removed of event.removed) {
+        const zipPath = removed.uri.fsPath;
+        afsp.flushCache(zipPath);
+      }
+    }),
+  );
+
+  ctx.subscriptions.push(
+    workspace.registerFileSystemProvider(zipArchiveScheme, afsp, {
+      isCaseSensitive: true,
+      isReadonly: true,
+    }),
   );
 }

extensions/ql-vscode/src/common/vscode/authentication.ts

@@ -2,8 +2,8 @@ import { authentication } from "vscode";
 import type { Octokit } from "@octokit/rest";
 import type { Credentials } from "../authentication";
 import { AppOctokit } from "../octokit";
-
-export const GITHUB_AUTH_PROVIDER_ID = "github";
+import { hasGhecDrUri } from "../../config";
+import { getOctokitBaseUrl } from "./octokit";
 
 // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist,
 // and 'read:packages' for reading private CodeQL packages.
@@ -16,30 +16,24 @@ const SCOPES = ["repo", "gist", "read:packages"];
  */
 export class VSCodeCredentials implements Credentials {
   /**
-   * A specific octokit to return, otherwise a new authenticated octokit will be created when needed.
-   */
-  private octokit: Octokit | undefined;
-
-  /**
-   * Creates or returns an instance of Octokit.
+   * Creates or returns an instance of Octokit. The returned instance should
+   * not be stored and reused, as it may become out-of-date with the current
+   * authentication session.
    *
    * @returns An instance of Octokit.
    */
   async getOctokit(): Promise<Octokit> {
-    if (this.octokit) {
-      return this.octokit;
-    }
-
     const accessToken = await this.getAccessToken();
 
     return new AppOctokit({
       auth: accessToken,
+      baseUrl: getOctokitBaseUrl(),
     });
   }
 
   async getAccessToken(): Promise<string> {
     const session = await authentication.getSession(
-      GITHUB_AUTH_PROVIDER_ID,
+      this.authProviderId,
       SCOPES,
       { createIfNone: true },
     );
@@ -49,11 +43,18 @@ export class VSCodeCredentials implements Credentials {
 
   async getExistingAccessToken(): Promise<string | undefined> {
     const session = await authentication.getSession(
-      GITHUB_AUTH_PROVIDER_ID,
+      this.authProviderId,
       SCOPES,
       { createIfNone: false },
     );
 
     return session?.accessToken;
   }
+
+  public get authProviderId(): string {
+    if (hasGhecDrUri()) {
+      return "github-enterprise";
+    }
+    return "github";
+  }
 }

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -3,18 +3,10 @@ import { commands } from "vscode";
 import type { CommandFunction } from "../../packages/commands";
 import { CommandManager } from "../../packages/commands";
 import type { NotificationLogger } from "../logging";
-import {
-  showAndLogWarningMessage,
-  showAndLogExceptionWithTelemetry,
-} from "../logging";
 import { extLogger } from "../logging/vscode";
-import { asError, getErrorMessage } from "../../common/helpers-pure";
-import { redactableError } from "../../common/errors";
-import { UserCancellationException } from "./progress";
 import { telemetryListener } from "./telemetry";
 import type { AppTelemetry } from "../telemetry";
-import { CliError } from "../../codeql-cli/cli-errors";
-import { EOL } from "os";
+import { runWithErrorHandling } from "./error-handling";
 
 /**
  * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -48,50 +40,9 @@ export function registerCommandWithErrorHandling<
   logger: NotificationLogger = extLogger,
   telemetry: AppTelemetry | undefined = telemetryListener,
 ): Disposable {
-  return commands.registerCommand(commandId, async (...args: Parameters<T>) => {
-    const startTime = Date.now();
-    let error: Error | undefined;
-
-    try {
-      return await task(...args);
-    } catch (e) {
-      error = asError(e);
-      const errorMessage = redactableError(error)`${
-        getErrorMessage(e) || e
-      } (${commandId})`;
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          void logger.log(errorMessage.fullMessage);
-        } else {
-          void showAndLogWarningMessage(logger, errorMessage.fullMessage);
-        }
-      } else if (e instanceof CliError) {
-        const fullMessage = `${e.commandDescription} failed with args:${EOL}    ${e.commandArgs.join(" ")}${EOL}${
-          e.stderr ?? e.cause
-        }`;
-        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      } else {
-        // Include the full stack in the error log only.
-        const fullMessage = errorMessage.fullMessageWithStack;
-        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      }
-      return undefined;
-    } finally {
-      const executionTime = Date.now() - startTime;
-      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
-    }
-  });
+  return commands.registerCommand(commandId, async (...args: Parameters<T>) =>
+    runWithErrorHandling(task, logger, telemetry, commandId, ...args),
+  );
 }
 
 /**

extensions/ql-vscode/src/common/vscode/error-handling.ts

@@ -0,0 +1,81 @@
+import {
+  showAndLogWarningMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../logging";
+import type { NotificationLogger } from "../logging";
+import { extLogger } from "../logging/vscode";
+import type { AppTelemetry } from "../telemetry";
+import { telemetryListener } from "./telemetry";
+import { asError, getErrorMessage } from "../helpers-pure";
+import { redactableError } from "../errors";
+import { UserCancellationException } from "./progress";
+import { CliError } from "../../codeql-cli/cli-errors";
+import { EOL } from "os";
+
+/**
+ * Executes a task with error handling. It provides a uniform way to handle errors.
+ *
+ * @template T - A function type that takes an unknown number of arguments and returns a Promise.
+ * @param {T} task - The task to be executed.
+ * @param {NotificationLogger} [logger=extLogger] - The logger to use for error reporting.
+ * @param {AppTelemetry | undefined} [telemetry=telemetryListener] - The telemetry listener to use for error reporting.
+ * @param {string} [commandId] - The optional command id associated with the task.
+ * @param {...unknown} args - The arguments to be passed to the task.
+ * @returns {Promise<unknown>} The result of the task, or undefined if an error occurred.
+ * @throws {Error} If an error occurs during the execution of the task.
+ */
+export async function runWithErrorHandling<
+  T extends (...args: unknown[]) => Promise<unknown>,
+>(
+  task: T,
+  logger: NotificationLogger = extLogger,
+  telemetry: AppTelemetry | undefined = telemetryListener,
+  commandId?: string,
+  ...args: unknown[]
+): Promise<unknown> {
+  const startTime = Date.now();
+  let error: Error | undefined;
+
+  try {
+    return await task(...args);
+  } catch (e) {
+    error = asError(e);
+    const errorMessage = redactableError(error)`${
+      getErrorMessage(e) || e
+    }${commandId ? ` (${commandId})` : ""}`;
+
+    const extraTelemetryProperties = commandId
+      ? { command: commandId }
+      : undefined;
+
+    if (e instanceof UserCancellationException) {
+      // User has cancelled this action manually
+      if (e.silent) {
+        void logger.log(errorMessage.fullMessage);
+      } else {
+        void showAndLogWarningMessage(logger, errorMessage.fullMessage);
+      }
+    } else if (e instanceof CliError) {
+      const fullMessage = `${e.commandDescription} failed with args:${EOL}    ${e.commandArgs.join(" ")}${EOL}${
+        e.stderr ?? e.cause
+      }`;
+      void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
+        fullMessage,
+        extraTelemetryProperties,
+      });
+    } else {
+      // Include the full stack in the error log only.
+      const fullMessage = errorMessage.fullMessageWithStack;
+      void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
+        fullMessage,
+        extraTelemetryProperties,
+      });
+    }
+    return undefined;
+  } finally {
+    if (commandId) {
+      const executionTime = Date.now() - startTime;
+      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/octokit.ts

@@ -0,0 +1,15 @@
+import { getGitHubInstanceApiUrl } from "../../config";
+
+/**
+ * Returns the Octokit base URL to use based on the GitHub instance URL.
+ *
+ * This is necessary because the Octokit base URL should not have a trailing
+ * slash, but this is included by default in a URL.
+ */
+export function getOctokitBaseUrl(): string {
+  let apiUrl = getGitHubInstanceApiUrl().toString();
+  if (apiUrl.endsWith("/")) {
+    apiUrl = apiUrl.slice(0, -1);
+  }
+  return apiUrl;
+}

extensions/ql-vscode/src/common/vscode/progress.ts

@@ -33,6 +33,14 @@ export interface ProgressUpdate {
   message: string;
 }
 
+export function progressUpdate(
+  step: number,
+  maxStep: number,
+  message: string,
+): ProgressUpdate {
+  return { step, maxStep, message };
+}
+
 export type ProgressCallback = (p: ProgressUpdate) => void;
 
 // Make certain properties within a type optional
@@ -48,7 +56,7 @@ type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
  * denote some progress being achieved on this task.
  * @param token a cancellation token
  */
-export type ProgressTask<R> = (
+type ProgressTask<R> = (
   progress: ProgressCallback,
   token: CancellationToken,
 ) => Thenable<R>;
@@ -85,29 +93,6 @@ export function withProgress<R>(
   );
 }
 
-export interface ProgressContext {
-  progress: ProgressCallback;
-  token: CancellationToken;
-}
-
-/**
- * Like `withProgress()`, except that the caller is not required to provide a progress context. If
- * the caller does provide one, any long-running operations performed by `task` will use the
- * supplied progress context. Otherwise, this function wraps `task` in a new progress context with
- * the supplied options.
- */
-export function withInheritedProgress<R>(
-  parent: ProgressContext | undefined,
-  task: ProgressTask<R>,
-  options: ProgressOptions,
-): Thenable<R> {
-  if (parent !== undefined) {
-    return task(parent.progress, parent.token);
-  } else {
-    return withProgress(task, options);
-  }
-}
-
 /**
  * Displays a progress monitor that indicates how much progess has been made
  * reading from a stream.

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -10,7 +10,8 @@ export type WebviewKind =
   | "variant-analysis"
   | "data-flow-paths"
   | "model-editor"
-  | "method-modeling";
+  | "method-modeling"
+  | "model-alerts";
 
 export interface WebviewMessage {
   t: string;

extensions/ql-vscode/src/compare/sarif-diff.ts

@@ -1,4 +1,29 @@
-import type { Result } from "sarif";
+import type { Location, Result } from "sarif";
+
+function toCanonicalLocation(location: Location): Location {
+  if (location.physicalLocation?.artifactLocation?.index !== undefined) {
+    const canonicalLocation = {
+      ...location,
+    };
+
+    canonicalLocation.physicalLocation = {
+      ...canonicalLocation.physicalLocation,
+    };
+
+    canonicalLocation.physicalLocation.artifactLocation = {
+      ...canonicalLocation.physicalLocation.artifactLocation,
+    };
+
+    // The index is dependent on the result of the SARIF file and usually doesn't really tell
+    // us anything useful, so we remove it from the comparison.
+    delete canonicalLocation.physicalLocation.artifactLocation.index;
+
+    return canonicalLocation;
+  }
+
+  // Don't create a new object if we don't need to
+  return location;
+}
 
 function toCanonicalResult(result: Result): Result {
   const canonicalResult = {
@@ -6,29 +31,45 @@ function toCanonicalResult(result: Result): Result {
   };
 
   if (canonicalResult.locations) {
-    canonicalResult.locations = canonicalResult.locations.map((location) => {
-      if (location.physicalLocation?.artifactLocation?.index !== undefined) {
-        const canonicalLocation = {
-          ...location,
+    canonicalResult.locations =
+      canonicalResult.locations.map(toCanonicalLocation);
+  }
+
+  if (canonicalResult.relatedLocations) {
+    canonicalResult.relatedLocations =
+      canonicalResult.relatedLocations.map(toCanonicalLocation);
+  }
+
+  if (canonicalResult.codeFlows) {
+    canonicalResult.codeFlows = canonicalResult.codeFlows.map((codeFlow) => {
+      if (codeFlow.threadFlows) {
+        return {
+          ...codeFlow,
+          threadFlows: codeFlow.threadFlows.map((threadFlow) => {
+            if (threadFlow.locations) {
+              return {
+                ...threadFlow,
+                locations: threadFlow.locations.map((threadFlowLocation) => {
+                  if (threadFlowLocation.location) {
+                    return {
+                      ...threadFlowLocation,
+                      location: toCanonicalLocation(
+                        threadFlowLocation.location,
+                      ),
+                    };
+                  }
+
+                  return threadFlowLocation;
+                }),
+              };
+            }
+
+            return threadFlow;
+          }),
         };
-
-        canonicalLocation.physicalLocation = {
-          ...canonicalLocation.physicalLocation,
-        };
-
-        canonicalLocation.physicalLocation.artifactLocation = {
-          ...canonicalLocation.physicalLocation.artifactLocation,
-        };
-
-        // The index is dependent on the result of the SARIF file and usually doesn't really tell
-        // us anything useful, so we remove it from the comparison.
-        delete canonicalLocation.physicalLocation.artifactLocation.index;
-
-        return canonicalLocation;
       }
 
-      // Don't create a new object if we don't need to
-      return location;
+      return codeFlow;
     });
   }
 

extensions/ql-vscode/src/config.ts

@@ -4,7 +4,7 @@ import type {
   ConfigurationScope,
   Event,
 } from "vscode";
-import { ConfigurationTarget, EventEmitter, workspace } from "vscode";
+import { ConfigurationTarget, EventEmitter, workspace, Uri } from "vscode";
 import type { DistributionManager } from "./codeql-cli/distribution";
 import { extLogger } from "./common/logging/vscode";
 import { ONE_DAY_IN_MS } from "./common/time";
@@ -13,6 +13,8 @@ import {
   FilterKey,
   SortKey,
 } from "./variant-analysis/shared/variant-analysis-filter-sort";
+import { substituteConfigVariables } from "./common/config-template";
+import { getErrorMessage } from "./common/helpers-pure";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -68,6 +70,97 @@ export const VSCODE_SAVE_BEFORE_START_SETTING = new Setting(
   VSCODE_DEBUG_SETTING,
 );
 
+const VSCODE_GITHUB_ENTERPRISE_SETTING = new Setting(
+  "github-enterprise",
+  undefined,
+);
+export const VSCODE_GITHUB_ENTERPRISE_URI_SETTING = new Setting(
+  "uri",
+  VSCODE_GITHUB_ENTERPRISE_SETTING,
+);
+
+/**
+ * Get the value of the `github-enterprise.uri` setting, parsed as a URI.
+ * If the value is not set or cannot be parsed, return `undefined`.
+ */
+export function getEnterpriseUri(): Uri | undefined {
+  const config = VSCODE_GITHUB_ENTERPRISE_URI_SETTING.getValue<string>();
+  if (config) {
+    try {
+      let uri = Uri.parse(config, true);
+      if (uri.scheme === "http") {
+        uri = uri.with({ scheme: "https" });
+      }
+      return uri;
+    } catch (e) {
+      void extLogger.log(
+        `Failed to parse the GitHub Enterprise URI: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Is the GitHub Enterprise URI set?
+ */
+export function hasEnterpriseUri(): boolean {
+  return getEnterpriseUri() !== undefined;
+}
+
+/**
+ * Does the uri look like GHEC-DR?
+ */
+function isGhecDrUri(uri: Uri | undefined): boolean {
+  return (
+    uri !== undefined && !uri.authority.toLowerCase().endsWith("github.com")
+  );
+}
+
+/**
+ * Is the GitHub Enterprise URI set to something that looks like GHEC-DR?
+ */
+export function hasGhecDrUri(): boolean {
+  const uri = getEnterpriseUri();
+  return isGhecDrUri(uri);
+}
+
+/**
+ * The URI for GitHub.com.
+ */
+export const GITHUB_URL = new URL("https://github.com");
+export const GITHUB_API_URL = new URL("https://api.github.com");
+
+/**
+ * If the GitHub Enterprise URI is set to something that looks like GHEC-DR, return it.
+ */
+export function getGhecDrUri(): Uri | undefined {
+  const uri = getEnterpriseUri();
+  if (isGhecDrUri(uri)) {
+    return uri;
+  } else {
+    return undefined;
+  }
+}
+
+export function getGitHubInstanceUrl(): URL {
+  const ghecDrUri = getGhecDrUri();
+  if (ghecDrUri) {
+    return new URL(ghecDrUri.toString());
+  }
+  return GITHUB_URL;
+}
+
+export function getGitHubInstanceApiUrl(): URL {
+  const ghecDrUri = getGhecDrUri();
+  if (ghecDrUri) {
+    const url = new URL(ghecDrUri.toString());
+    url.hostname = `api.${url.hostname}`;
+    return url;
+  }
+  return GITHUB_API_URL;
+}
+
 const ROOT_SETTING = new Setting("codeQL");
 
 // Telemetry configuration
@@ -93,6 +186,10 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting(
   "personalAccessToken",
   DISTRIBUTION_SETTING,
 );
+const CLI_DOWNLOAD_TIMEOUT_SETTING = new Setting(
+  "downloadTimeout",
+  DISTRIBUTION_SETTING,
+);
 const CLI_CHANNEL_SETTING = new Setting("channel", DISTRIBUTION_SETTING);
 
 // Query History configuration
@@ -118,6 +215,7 @@ export interface DistributionConfig {
   updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
+  downloadTimeout: number;
   channel: CLIChannel;
   onDidChangeConfiguration?: Event<void>;
 }
@@ -272,6 +370,10 @@ export class DistributionConfigListener
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
+  public get downloadTimeout(): number {
+    return CLI_DOWNLOAD_TIMEOUT_SETTING.getValue() || 10;
+  }
+
   public async updateCustomCodeQlPath(newPath: string | undefined) {
     await CUSTOM_CODEQL_PATH_SETTING.updateValue(
       newPath,
@@ -491,27 +593,13 @@ export const NO_CACHE_CONTEXTUAL_QUERIES = new Setting(
 // Settings for variant analysis
 const VARIANT_ANALYSIS_SETTING = new Setting("variantAnalysis", ROOT_SETTING);
 
-/**
- * The name of the "controller" repository that you want to use with the "Run Variant Analysis" command.
- * Note: This command is only available for internal users.
- *
- * This setting should be a GitHub repository of the form `<owner>/<repo>`.
- */
-const REMOTE_CONTROLLER_REPO = new Setting(
-  "controllerRepo",
-  VARIANT_ANALYSIS_SETTING,
-);
-
-export function getRemoteControllerRepo(): string | undefined {
-  return REMOTE_CONTROLLER_REPO.getValue<string>() || undefined;
-}
-
-export async function setRemoteControllerRepo(repo: string | undefined) {
-  await REMOTE_CONTROLLER_REPO.updateValue(repo, ConfigurationTarget.Global);
-}
-
 export interface VariantAnalysisConfig {
-  controllerRepo: string | undefined;
+  showSystemDefinedRepositoryLists: boolean;
+  /**
+   * This uses a URL instead of a URI because the URL class is available in
+   * unit tests and is fully browser-compatible.
+   */
+  githubUrl: URL;
   onDidChangeConfiguration?: Event<void>;
 }
 
@@ -521,13 +609,17 @@ export class VariantAnalysisConfigListener
 {
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(
-      [VARIANT_ANALYSIS_SETTING],
+      [VARIANT_ANALYSIS_SETTING, VSCODE_GITHUB_ENTERPRISE_URI_SETTING],
       e,
     );
   }
 
-  public get controllerRepo(): string | undefined {
-    return getRemoteControllerRepo();
+  public get showSystemDefinedRepositoryLists(): boolean {
+    return !hasEnterpriseUri();
+  }
+
+  public get githubUrl(): URL {
+    return getGitHubInstanceUrl();
   }
 }
 
@@ -644,8 +736,16 @@ const DEPRECATED_ALLOW_HTTP_SETTING = new Setting(
 
 const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING);
 
+const DOWNLOAD_TIMEOUT_SETTING = new Setting(
+  "downloadTimeout",
+  ADDING_DATABASES_SETTING,
+);
 const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING);
 
+export function downloadTimeout(): number {
+  return DOWNLOAD_TIMEOUT_SETTING.getValue<number>() || 10;
+}
+
 export function allowHttp(): boolean {
   return (
     ALLOW_HTTP_SETTING.getValue<boolean>() ||
@@ -715,24 +815,35 @@ const LLM_GENERATION_DEV_ENDPOINT = new Setting(
   "llmGenerationDevEndpoint",
   MODEL_SETTING,
 );
-const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
+const MODEL_EVALUATION = new Setting("evaluation", MODEL_SETTING);
+const MODEL_PACK_LOCATION = new Setting("packLocation", MODEL_SETTING);
+const MODEL_PACK_NAME = new Setting("packName", MODEL_SETTING);
 const ENABLE_PYTHON = new Setting("enablePython", MODEL_SETTING);
-const ENABLE_ACCESS_PATH_SUGGESTIONS = new Setting(
-  "enableAccessPathSuggestions",
-  MODEL_SETTING,
-);
+
+export type ModelConfigPackVariables = {
+  database: string;
+  owner: string;
+  name: string;
+  language: string;
+};
 
 export interface ModelConfig {
   flowGeneration: boolean;
   llmGeneration: boolean;
-  getExtensionsDirectory(languageId: string): string | undefined;
+  getPackLocation(
+    languageId: string,
+    variables: ModelConfigPackVariables,
+  ): string;
+  getPackName(languageId: string, variables: ModelConfigPackVariables): string;
   enablePython: boolean;
-  enableAccessPathSuggestions: boolean;
 }
 
 export class ModelConfigListener extends ConfigListener implements ModelConfig {
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
-    this.handleDidChangeConfigurationForRelevantSettings([MODEL_SETTING], e);
+    this.handleDidChangeConfigurationForRelevantSettings(
+      [MODEL_SETTING, VSCODE_GITHUB_ENTERPRISE_URI_SETTING],
+      e,
+    );
   }
 
   public get flowGeneration(): boolean {
@@ -740,7 +851,7 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
   }
 
   public get llmGeneration(): boolean {
-    return !!LLM_GENERATION.getValue<boolean>();
+    return !!LLM_GENERATION.getValue<boolean>() && !hasEnterpriseUri();
   }
 
   /**
@@ -759,19 +870,37 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return LLM_GENERATION_DEV_ENDPOINT.getValue<string | undefined>();
   }
 
-  public getExtensionsDirectory(languageId: string): string | undefined {
-    return EXTENSIONS_DIRECTORY.getValue<string>({
-      languageId,
-    });
+  public get modelEvaluation(): boolean {
+    return !!MODEL_EVALUATION.getValue<boolean>();
+  }
+
+  public getPackLocation(
+    languageId: string,
+    variables: ModelConfigPackVariables,
+  ): string {
+    return substituteConfigVariables(
+      MODEL_PACK_LOCATION.getValue<string>({
+        languageId,
+      }),
+      variables,
+    );
+  }
+
+  public getPackName(
+    languageId: string,
+    variables: ModelConfigPackVariables,
+  ): string {
+    return substituteConfigVariables(
+      MODEL_PACK_NAME.getValue<string>({
+        languageId,
+      }),
+      variables,
+    );
   }
 
   public get enablePython(): boolean {
     return !!ENABLE_PYTHON.getValue<boolean>();
   }
-
-  public get enableAccessPathSuggestions(): boolean {
-    return !!ENABLE_ACCESS_PATH_SUGGESTIONS.getValue<boolean>();
-  }
 }
 
 const GITHUB_DATABASE_SETTING = new Setting("githubDatabase", ROOT_SETTING);

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -7,6 +7,7 @@ import { AppOctokit } from "../common/octokit";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { UserCancellationException } from "../common/vscode/progress";
 import type { EndpointDefaults } from "@octokit/types";
+import { getOctokitBaseUrl } from "../common/vscode/octokit";
 
 export async function getCodeSearchRepositories(
   query: string,
@@ -54,6 +55,7 @@ async function provideOctokitWithThrottling(
 
   const octokit = new MyOctokit({
     auth,
+    baseUrl: getOctokitBaseUrl(),
     throttle: {
       onRateLimit: (retryAfter: number, options: EndpointDefaults): boolean => {
         void logger.log(

extensions/ql-vscode/src/databases/db-manager.ts

@@ -16,6 +16,7 @@ import {
 } from "./db-item-selection";
 import { createRemoteTree } from "./db-tree-creator";
 import type { DbConfigValidationError } from "./db-validation-errors";
+import type { VariantAnalysisConfig } from "../config";
 
 export class DbManager extends DisposableObject {
   public readonly onDbItemsChanged: AppEvent<void>;
@@ -25,6 +26,7 @@ export class DbManager extends DisposableObject {
   constructor(
     private readonly app: App,
     private readonly dbConfigStore: DbConfigStore,
+    private readonly variantAnalysisConfigListener: VariantAnalysisConfig,
   ) {
     super();
 
@@ -36,6 +38,10 @@ export class DbManager extends DisposableObject {
     this.dbConfigStore.onDidChangeConfig(() => {
       this.onDbItemsChangesEventEmitter.fire();
     });
+
+    this.variantAnalysisConfigListener.onDidChangeConfiguration?.(() => {
+      this.onDbItemsChangesEventEmitter.fire();
+    });
   }
 
   public getSelectedDbItem(): DbItem | undefined {
@@ -56,7 +62,11 @@ export class DbManager extends DisposableObject {
 
     const expandedItems = this.getExpandedItems();
 
-    const remoteTree = createRemoteTree(configResult.value, expandedItems);
+    const remoteTree = createRemoteTree(
+      configResult.value,
+      this.variantAnalysisConfigListener,
+      expandedItems,
+    );
     return ValueResult.ok(remoteTree.children);
   }
 

extensions/ql-vscode/src/databases/db-module.ts

@@ -7,6 +7,7 @@ import { DbManager } from "./db-manager";
 import { DbPanel } from "./ui/db-panel";
 import { DbSelectionDecorationProvider } from "./ui/db-selection-decoration-provider";
 import type { DatabasePanelCommands } from "../common/commands";
+import { VariantAnalysisConfigListener } from "../config";
 
 export class DbModule extends DisposableObject {
   public readonly dbManager: DbManager;
@@ -17,7 +18,13 @@ export class DbModule extends DisposableObject {
     super();
 
     this.dbConfigStore = new DbConfigStore(app);
-    this.dbManager = this.push(new DbManager(app, this.dbConfigStore));
+    this.dbManager = this.push(
+      new DbManager(
+        app,
+        this.dbConfigStore,
+        new VariantAnalysisConfigListener(),
+      ),
+    );
   }
 
   public static async initialize(app: App): Promise<DbModule> {

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -1,3 +1,4 @@
+import type { VariantAnalysisConfig } from "../config";
 import type { DbConfig, RemoteRepositoryList } from "./config/db-config";
 import { SelectedDbItemKind } from "./config/db-config";
 import type {
@@ -13,13 +14,17 @@ import { ExpandedDbItemKind } from "./db-item-expansion";
 
 export function createRemoteTree(
   dbConfig: DbConfig,
+  variantAnalysisConfig: VariantAnalysisConfig,
   expandedItems: ExpandedDbItem[],
 ): RootRemoteDbItem {
-  const systemDefinedLists = [
-    createSystemDefinedList(10, dbConfig),
-    createSystemDefinedList(100, dbConfig),
-    createSystemDefinedList(1000, dbConfig),
-  ];
+  const systemDefinedLists =
+    variantAnalysisConfig.showSystemDefinedRepositoryLists
+      ? [
+          createSystemDefinedList(10, dbConfig),
+          createSystemDefinedList(100, dbConfig),
+          createSystemDefinedList(1000, dbConfig),
+        ]
+      : [];
 
   const userDefinedRepoLists =
     dbConfig.databases.variantAnalysis.repositoryLists.map((r) =>

extensions/ql-vscode/src/databases/github-databases/api.ts

@@ -3,8 +3,14 @@ import type { Octokit } from "@octokit/rest";
 import type { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods";
 import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
 import type { GitHubDatabaseConfig } from "../../config";
+import { hasGhecDrUri } from "../../config";
 import type { Credentials } from "../../common/authentication";
 import { AppOctokit } from "../../common/octokit";
+import type { ProgressCallback } from "../../common/vscode/progress";
+import { getErrorMessage } from "../../common/helpers-pure";
+import { getLanguageDisplayName } from "../../common/query-language";
+import { window } from "vscode";
+import { extLogger } from "../../common/logging/vscode";
 
 export type CodeqlDatabase =
   RestEndpointMethodTypes["codeScanning"]["listCodeqlDatabases"]["response"]["data"][number];
@@ -62,7 +68,10 @@ export async function listDatabases(
   credentials: Credentials,
   config: GitHubDatabaseConfig,
 ): Promise<ListDatabasesResult | undefined> {
-  const hasAccessToken = !!(await credentials.getExistingAccessToken());
+  // On GHEC-DR, unauthenticated requests will never work, so we should always ask
+  // for authentication.
+  const hasAccessToken =
+    !!(await credentials.getExistingAccessToken()) || hasGhecDrUri();
 
   let octokit = hasAccessToken
     ? await credentials.getOctokit()
@@ -108,3 +117,92 @@ export async function listDatabases(
     octokit,
   };
 }
+
+export async function convertGithubNwoToDatabaseUrl(
+  nwo: string,
+  octokit: Octokit,
+  progress: ProgressCallback,
+  language?: string,
+): Promise<
+  | {
+      databaseUrl: string;
+      owner: string;
+      name: string;
+      databaseId: number;
+      databaseCreatedAt: string;
+      commitOid: string | null;
+    }
+  | undefined
+> {
+  try {
+    const [owner, repo] = nwo.split("/");
+
+    const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+      owner,
+      repo,
+    });
+
+    const languages = response.data.map((db) => db.language);
+
+    if (!language || !languages.includes(language)) {
+      language = await promptForLanguage(languages, progress);
+      if (!language) {
+        return;
+      }
+    }
+
+    const databaseForLanguage = response.data.find(
+      (db) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
+    return {
+      databaseUrl: databaseForLanguage.url,
+      owner,
+      name: repo,
+      databaseId: databaseForLanguage.id,
+      databaseCreatedAt: databaseForLanguage.created_at,
+      commitOid: databaseForLanguage.commit_oid ?? null,
+    };
+  } catch (e) {
+    void extLogger.log(`Error: ${getErrorMessage(e)}`);
+    throw new Error(`Unable to get database for '${nwo}'`);
+  }
+}
+
+async function promptForLanguage(
+  languages: string[],
+  progress: ProgressCallback | undefined,
+): Promise<string | undefined> {
+  progress?.({
+    message: "Choose language",
+    step: 2,
+    maxStep: 2,
+  });
+  if (!languages.length) {
+    throw new Error("No databases found");
+  }
+  if (languages.length === 1) {
+    return languages[0];
+  }
+
+  const items = languages
+    .map((language) => ({
+      label: getLanguageDisplayName(language),
+      description: language,
+      language,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItem = await window.showQuickPick(items, {
+    placeHolder: "Select the database language to download:",
+    ignoreFocusOut: true,
+  });
+  if (!selectedItem) {
+    return undefined;
+  }
+
+  return selectedItem.language;
+}

extensions/ql-vscode/src/databases/github-databases/download.ts

@@ -2,10 +2,8 @@ import { window } from "vscode";
 import type { Octokit } from "@octokit/rest";
 import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
 import { getLanguageDisplayName } from "../../common/query-language";
-import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
+import type { DatabaseFetcher } from "../database-fetcher";
 import { withProgress } from "../../common/vscode/progress";
-import type { DatabaseManager } from "../local-databases";
-import type { CodeQLCliServer } from "../../codeql-cli/cli";
 import type { AppCommandManager } from "../../common/commands";
 import type { GitHubDatabaseConfig } from "../../config";
 import type { CodeqlDatabase } from "./api";
@@ -58,9 +56,7 @@ export async function downloadDatabaseFromGitHub(
   owner: string,
   repo: string,
   databases: CodeqlDatabase[],
-  databaseManager: DatabaseManager,
-  storagePath: string,
-  cliServer: CodeQLCliServer,
+  databaseFetcher: DatabaseFetcher,
   commandManager: AppCommandManager,
 ): Promise<void> {
   const selectedDatabases = await promptForDatabases(databases);
@@ -72,7 +68,7 @@ export async function downloadDatabaseFromGitHub(
     selectedDatabases.map((database) =>
       withProgress(
         async (progress) => {
-          await downloadGitHubDatabaseFromUrl(
+          await databaseFetcher.downloadGitHubDatabaseFromUrl(
             database.url,
             database.id,
             database.created_at,
@@ -81,9 +77,6 @@ export async function downloadDatabaseFromGitHub(
             repo,
             octokit,
             progress,
-            databaseManager,
-            storagePath,
-            cliServer,
             true,
             false,
           );

extensions/ql-vscode/src/databases/github-databases/github-databases-module.ts

@@ -14,7 +14,6 @@ import {
 } from "./download";
 import type { GitHubDatabaseConfig } from "../../config";
 import type { DatabaseManager } from "../local-databases";
-import type { CodeQLCliServer } from "../../codeql-cli/cli";
 import type { CodeqlDatabase, ListDatabasesResult } from "./api";
 import { listDatabases } from "./api";
 import type { DatabaseUpdate } from "./updates";
@@ -24,6 +23,7 @@ import {
   isNewerDatabaseAvailable,
 } from "./updates";
 import type { Octokit } from "@octokit/rest";
+import type { DatabaseFetcher } from "../database-fetcher";
 
 export class GitHubDatabasesModule extends DisposableObject {
   /**
@@ -33,8 +33,7 @@ export class GitHubDatabasesModule extends DisposableObject {
   constructor(
     private readonly app: App,
     private readonly databaseManager: DatabaseManager,
-    private readonly databaseStoragePath: string,
-    private readonly cliServer: CodeQLCliServer,
+    private readonly databaseFetcher: DatabaseFetcher,
     private readonly config: GitHubDatabaseConfig,
   ) {
     super();
@@ -43,15 +42,13 @@ export class GitHubDatabasesModule extends DisposableObject {
   public static async initialize(
     app: App,
     databaseManager: DatabaseManager,
-    databaseStoragePath: string,
-    cliServer: CodeQLCliServer,
+    databaseFetcher: DatabaseFetcher,
     config: GitHubDatabaseConfig,
   ): Promise<GitHubDatabasesModule> {
     const githubDatabasesModule = new GitHubDatabasesModule(
       app,
       databaseManager,
-      databaseStoragePath,
-      cliServer,
+      databaseFetcher,
       config,
     );
     app.subscriptions.push(githubDatabasesModule);
@@ -185,9 +182,7 @@ export class GitHubDatabasesModule extends DisposableObject {
       owner,
       repo,
       databases,
-      this.databaseManager,
-      this.databaseStoragePath,
-      this.cliServer,
+      this.databaseFetcher,
       this.app.commands,
     );
   }
@@ -212,8 +207,7 @@ export class GitHubDatabasesModule extends DisposableObject {
       repo,
       databaseUpdates,
       this.databaseManager,
-      this.databaseStoragePath,
-      this.cliServer,
+      this.databaseFetcher,
       this.app.commands,
     );
   }

extensions/ql-vscode/src/databases/github-databases/updates.ts

@@ -1,11 +1,10 @@
 import type { CodeqlDatabase } from "./api";
 import type { DatabaseItem, DatabaseManager } from "../local-databases";
 import type { Octokit } from "@octokit/rest";
-import type { CodeQLCliServer } from "../../codeql-cli/cli";
 import type { AppCommandManager } from "../../common/commands";
 import { getLanguageDisplayName } from "../../common/query-language";
 import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
-import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
+import type { DatabaseFetcher } from "../database-fetcher";
 import { withProgress } from "../../common/vscode/progress";
 import { window } from "vscode";
 import type { GitHubDatabaseConfig } from "../../config";
@@ -156,8 +155,7 @@ export async function downloadDatabaseUpdateFromGitHub(
   repo: string,
   updates: DatabaseUpdate[],
   databaseManager: DatabaseManager,
-  storagePath: string,
-  cliServer: CodeQLCliServer,
+  databaseFetcher: DatabaseFetcher,
   commandManager: AppCommandManager,
 ): Promise<void> {
   const selectedDatabases = await promptForDatabases(
@@ -179,21 +177,19 @@ export async function downloadDatabaseUpdateFromGitHub(
 
       return withProgress(
         async (progress) => {
-          const newDatabase = await downloadGitHubDatabaseFromUrl(
-            database.url,
-            database.id,
-            database.created_at,
-            database.commit_oid ?? null,
-            owner,
-            repo,
-            octokit,
-            progress,
-            databaseManager,
-            storagePath,
-            cliServer,
-            databaseManager.currentDatabaseItem === update.databaseItem,
-            update.databaseItem.hasSourceArchiveInExplorer(),
-          );
+          const newDatabase =
+            await databaseFetcher.downloadGitHubDatabaseFromUrl(
+              database.url,
+              database.id,
+              database.created_at,
+              database.commit_oid ?? null,
+              owner,
+              repo,
+              octokit,
+              progress,
+              databaseManager.currentDatabaseItem === update.databaseItem,
+              update.databaseItem.hasSourceArchiveInExplorer(),
+            );
           if (newDatabase === undefined) {
             return;
           }

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -16,7 +16,6 @@ import {
   ThemeIcon,
   ThemeColor,
   workspace,
-  ProgressLocation,
 } from "vscode";
 import { pathExists, stat, readdir, remove } from "fs-extra";
 
@@ -25,13 +24,9 @@ import type {
   DatabaseItem,
   DatabaseManager,
 } from "./local-databases";
-import type {
-  ProgressCallback,
-  ProgressContext,
-} from "../common/vscode/progress";
+import type { ProgressCallback } from "../common/vscode/progress";
 import {
   UserCancellationException,
-  withInheritedProgress,
   withProgress,
 } from "../common/vscode/progress";
 import {
@@ -42,14 +37,9 @@ import {
   showAndLogExceptionWithTelemetry,
   showAndLogErrorMessage,
 } from "../common/logging";
-import {
-  importArchiveDatabase,
-  promptImportGithubDatabase,
-  promptImportInternetDatabase,
-} from "./database-fetcher";
+import type { DatabaseFetcher } from "./database-fetcher";
 import { asError, asyncFilter, getErrorMessage } from "../common/helpers-pure";
 import type { QueryRunner } from "../query-server";
-import { isCanary } from "../config";
 import type { App } from "../common/app";
 import { redactableError } from "../common/errors";
 import type { LocalDatabasesCommands } from "../common/commands";
@@ -119,9 +109,8 @@ class DatabaseTreeDataProvider
     // Note that events from the database manager are instances of DatabaseChangedEvent
     // and events fired by the UI are instances of DatabaseItem
 
-    // When event.item is undefined, then the entire tree is refreshed.
-    // When event.item is a db item, then only that item is refreshed.
-    this._onDidChangeTreeData.fire(event.item);
+    // When a full refresh has occurred, then all items are refreshed by passing undefined.
+    this._onDidChangeTreeData.fire(event.fullRefresh ? undefined : event.item);
   }
 
   private handleDidChangeCurrentDatabaseItem(
@@ -146,7 +135,8 @@ class DatabaseTreeDataProvider
       item.iconPath = new ThemeIcon("error", new ThemeColor("errorForeground"));
     }
     item.tooltip = element.databaseUri.fsPath;
-    item.description = element.language;
+    item.description =
+      element.language + (element.origin?.type === "testproj" ? " (test)" : "");
     return item;
   }
 
@@ -253,6 +243,7 @@ export class DatabaseUI extends DisposableObject {
   public constructor(
     private app: App,
     private databaseManager: DatabaseManager,
+    private readonly databaseFetcher: DatabaseFetcher,
     languageContext: LanguageContextStore,
     private readonly queryServer: QueryRunner,
     private readonly storagePath: string,
@@ -282,6 +273,7 @@ export class DatabaseUI extends DisposableObject {
         this.handleChooseDatabaseInternet.bind(this),
       "codeQL.chooseDatabaseGithub": this.handleChooseDatabaseGithub.bind(this),
       "codeQL.setCurrentDatabase": this.handleSetCurrentDatabase.bind(this),
+      "codeQL.importTestDatabase": this.handleImportTestDatabase.bind(this),
       "codeQL.setDefaultTourDatabase":
         this.handleSetDefaultTourDatabase.bind(this),
       "codeQL.upgradeCurrentDatabase":
@@ -331,10 +323,9 @@ export class DatabaseUI extends DisposableObject {
 
   private async chooseDatabaseFolder(
     progress: ProgressCallback,
-    token: CancellationToken,
   ): Promise<void> {
     try {
-      await this.chooseAndSetDatabase(true, { progress, token });
+      await this.chooseAndSetDatabase(true, progress);
     } catch (e) {
       void showAndLogExceptionWithTelemetry(
         this.app.logger,
@@ -348,8 +339,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseFolder(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseFolder(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseFolder(progress);
       },
       {
         title: "Adding database from folder",
@@ -359,8 +350,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseFolderFromPalette(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseFolder(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseFolder(progress);
       },
       {
         title: "Choose a Database from a Folder",
@@ -501,10 +492,9 @@ export class DatabaseUI extends DisposableObject {
 
   private async chooseDatabaseArchive(
     progress: ProgressCallback,
-    token: CancellationToken,
   ): Promise<void> {
     try {
-      await this.chooseAndSetDatabase(false, { progress, token });
+      await this.chooseAndSetDatabase(false, progress);
     } catch (e: unknown) {
       void showAndLogExceptionWithTelemetry(
         this.app.logger,
@@ -518,8 +508,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseArchive(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseArchive(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseArchive(progress);
       },
       {
         title: "Adding database from archive",
@@ -529,8 +519,8 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseArchiveFromPalette(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
-        await this.chooseDatabaseArchive(progress, token);
+      async (progress) => {
+        await this.chooseDatabaseArchive(progress);
       },
       {
         title: "Choose a Database from an Archive",
@@ -541,13 +531,7 @@ export class DatabaseUI extends DisposableObject {
   private async handleChooseDatabaseInternet(): Promise<void> {
     return withProgress(
       async (progress) => {
-        await promptImportInternetDatabase(
-          this.app.commands,
-          this.databaseManager,
-          this.storagePath,
-          progress,
-          this.queryServer.cliServer,
-        );
+        await this.databaseFetcher.promptImportInternetDatabase(progress);
       },
       {
         title: "Adding database from URL",
@@ -558,16 +542,7 @@ export class DatabaseUI extends DisposableObject {
   private async handleChooseDatabaseGithub(): Promise<void> {
     return withProgress(
       async (progress) => {
-        const credentials = isCanary() ? this.app.credentials : undefined;
-
-        await promptImportGithubDatabase(
-          this.app.commands,
-          this.databaseManager,
-          this.storagePath,
-          credentials,
-          progress,
-          this.queryServer.cliServer,
-        );
+        await this.databaseFetcher.promptImportGithubDatabase(progress);
       },
       {
         title: "Adding database from GitHub",
@@ -669,14 +644,13 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleClearCache(): Promise<void> {
     return withProgress(
-      async (_progress, token) => {
+      async () => {
         if (
           this.queryServer !== undefined &&
           this.databaseManager.currentDatabaseItem !== undefined
         ) {
           await this.queryServer.clearCacheInDatabase(
             this.databaseManager.currentDatabaseItem,
-            token,
           );
         }
       },
@@ -688,14 +662,13 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleTrimCache(): Promise<void> {
     return withProgress(
-      async (_progress, token) => {
+      async () => {
         if (
           this.queryServer !== undefined &&
           this.databaseManager.currentDatabaseItem !== undefined
         ) {
           await this.queryServer.trimCacheInDatabase(
             this.databaseManager.currentDatabaseItem,
-            token,
           );
         }
       },
@@ -716,13 +689,9 @@ export class DatabaseUI extends DisposableObject {
         try {
           // Assume user has selected an archive if the file has a .zip extension
           if (uri.path.endsWith(".zip")) {
-            await importArchiveDatabase(
-              this.app.commands,
+            await this.databaseFetcher.importLocalDatabase(
               uri.toString(true),
-              this.databaseManager,
-              this.storagePath,
               progress,
-              this.queryServer.cliServer,
             );
           } else {
             await this.databaseManager.openDatabase(uri, {
@@ -744,6 +713,56 @@ export class DatabaseUI extends DisposableObject {
     );
   }
 
+  private async handleImportTestDatabase(uri: Uri): Promise<void> {
+    return withProgress(
+      async (progress) => {
+        try {
+          if (!uri.path.endsWith(".testproj")) {
+            throw new Error(
+              "Please select a valid test database to import. Test databases end with `.testproj`.",
+            );
+          }
+
+          // Check if the database is already in the workspace. If
+          // so, delete it first before importing the new one.
+          const existingItem = this.databaseManager.findTestDatabase(uri);
+          const baseName = basename(uri.fsPath);
+          if (existingItem !== undefined) {
+            progress({
+              maxStep: 9,
+              step: 1,
+              message: `Removing existing test database ${baseName}`,
+            });
+            await this.databaseManager.removeDatabaseItem(existingItem);
+          }
+
+          await this.databaseFetcher.importLocalDatabase(
+            uri.toString(true),
+            progress,
+          );
+
+          if (existingItem !== undefined) {
+            progress({
+              maxStep: 9,
+              step: 9,
+              message: `Successfully re-imported ${baseName}`,
+            });
+          }
+        } catch (e) {
+          // rethrow and let this be handled by default error handling.
+          throw new Error(
+            `Could not set database to ${basename(
+              uri.fsPath,
+            )}. Reason: ${getErrorMessage(e)}`,
+          );
+        }
+      },
+      {
+        title: "(Re-)importing test database from directory",
+      },
+    );
+  }
+
   private async handleRemoveDatabase(
     databaseItems: DatabaseItem[],
   ): Promise<void> {
@@ -799,9 +818,8 @@ export class DatabaseUI extends DisposableObject {
    */
   public async getDatabaseItem(
     progress: ProgressCallback,
-    token: CancellationToken,
   ): Promise<DatabaseItem | undefined> {
-    return await this.getDatabaseItemInternal({ progress, token });
+    return await this.getDatabaseItemInternal(progress);
   }
 
   /**
@@ -814,10 +832,10 @@ export class DatabaseUI extends DisposableObject {
    * notification if it tries to perform any long-running operations.
    */
   private async getDatabaseItemInternal(
-    progressContext: ProgressContext | undefined,
+    progress: ProgressCallback | undefined,
   ): Promise<DatabaseItem | undefined> {
     if (this.databaseManager.currentDatabaseItem === undefined) {
-      progressContext?.progress({
+      progress?.({
         maxStep: 2,
         step: 1,
         message: "Choosing database",
@@ -828,6 +846,12 @@ export class DatabaseUI extends DisposableObject {
   }
 
   private async promptForDatabase(): Promise<void> {
+    // If there aren't any existing databases,
+    // don't bother asking the user if they want to pick one.
+    if (this.databaseManager.databaseItems.length === 0) {
+      return this.importNewDatabase();
+    }
+
     const quickPickItems: DatabaseSelectionQuickPickItem[] = [
       {
         label: "$(database) Existing database",
@@ -837,7 +861,8 @@ export class DatabaseUI extends DisposableObject {
       },
       {
         label: "$(arrow-down) New database",
-        detail: "Import a new database from the cloud or your local machine",
+        detail:
+          "Import a new database from GitHub, a URL, or your local machine...",
         alwaysShow: true,
         databaseKind: "new",
       },
@@ -871,7 +896,7 @@ export class DatabaseUI extends DisposableObject {
       }));
 
     const selectedDatabase = await window.showQuickPick(dbItems, {
-      placeHolder: "Select a database",
+      placeHolder: "Select an existing database from your workspace...",
       ignoreFocusOut: true,
     });
 
@@ -913,7 +938,8 @@ export class DatabaseUI extends DisposableObject {
     ];
     const selectedImportOption =
       await window.showQuickPick<DatabaseImportQuickPickItems>(importOptions, {
-        placeHolder: "Import a database from...",
+        placeHolder:
+          "Import a new database from GitHub, a URL, or your local machine...",
         ignoreFocusOut: true,
       });
     if (!selectedImportOption) {
@@ -936,41 +962,28 @@ export class DatabaseUI extends DisposableObject {
    */
   private async chooseAndSetDatabase(
     byFolder: boolean,
-    progress: ProgressContext | undefined,
+    progress: ProgressCallback,
   ): Promise<DatabaseItem | undefined> {
     const uri = await chooseDatabaseDir(byFolder);
     if (!uri) {
       return undefined;
     }
 
-    return await withInheritedProgress(
-      progress,
-      async (progress) => {
-        if (byFolder) {
-          const fixedUri = await this.fixDbUri(uri);
-          // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri, {
-            type: "folder",
-          });
-        } else {
-          // we are selecting a database archive. Must unzip into a workspace-controlled area
-          // before importing.
-          return await importArchiveDatabase(
-            this.app.commands,
-            uri.toString(true),
-            this.databaseManager,
-            this.storagePath,
-            progress,
-            this.queryServer.cliServer,
-          );
-        }
-      },
-      {
-        location: ProgressLocation.Notification,
-        cancellable: true,
-        title: "Opening database",
-      },
-    );
+    if (byFolder && !uri.fsPath.endsWith("testproj")) {
+      const fixedUri = await this.fixDbUri(uri);
+      // we are selecting a database folder
+      return await this.databaseManager.openDatabase(fixedUri, {
+        type: "folder",
+      });
+    } else {
+      // we are selecting a database archive or a testproj.
+      // Unzip archives (if an archive) and copy into a workspace-controlled area
+      // before importing.
+      return await this.databaseFetcher.importLocalDatabase(
+        uri.toString(true),
+        progress,
+      );
+    }
   }
 
   /**

extensions/ql-vscode/src/databases/local-databases/database-contents.ts

@@ -1,4 +1,7 @@
+import { pathExists, remove } from "fs-extra";
+import { join } from "path";
 import type { Uri } from "vscode";
+import { zip } from "zip-a-folder";
 
 /**
  * The layout of the database.
@@ -28,3 +31,26 @@ export interface DatabaseContents {
 export interface DatabaseContentsWithDbScheme extends DatabaseContents {
   dbSchemeUri: Uri; // Always present
 }
+
+/**
+ * Databases created by the old odasa tool will not have a zipped
+ * source location. However, this extension works better if sources
+ * are zipped.
+ *
+ * This function ensures that the source location is zipped. If the
+ * `src` folder exists and the `src.zip` file does not, the `src`
+ * folder will be zipped and then deleted.
+ *
+ * @param databasePath The full path to the unzipped database
+ */
+export async function ensureZippedSourceLocation(
+  databasePath: string,
+): Promise<void> {
+  const srcFolderPath = join(databasePath, "src");
+  const srcZipPath = `${srcFolderPath}.zip`;
+
+  if ((await pathExists(srcFolderPath)) && !(await pathExists(srcZipPath))) {
+    await zip(srcFolderPath, srcZipPath);
+    await remove(srcFolderPath);
+  }
+}

extensions/ql-vscode/src/databases/local-databases/database-events.ts

@@ -16,4 +16,8 @@ export enum DatabaseEventKind {
 export interface DatabaseChangedEvent {
   kind: DatabaseEventKind;
   item: DatabaseItem | undefined;
+  // If true, event handlers should consider the database manager
+  // to have been fully refreshed. Any state managed by the
+  // event handler should be fully refreshed as well.
+  fullRefresh: boolean;
 }

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -66,6 +66,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return this.options.origin;
   }
 
+  public get extensionManagedLocation(): string | undefined {
+    return this.options.extensionManagedLocation;
+  }
+
   public resolveSourceFile(uriStr: string | undefined): Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? Uri.parse(uriStr, true) : undefined;

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -31,6 +31,12 @@ export interface DatabaseItem {
    */
   readonly origin: DatabaseOrigin | undefined;
 
+  /**
+   * The location of the base storage location as managed by the extension, or undefined
+   * if unknown or not managed by the extension.
+   */
+  readonly extensionManagedLocation: string | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -18,7 +18,10 @@ import {
 import { join } from "path";
 import type { FullDatabaseOptions } from "./database-options";
 import { DatabaseItemImpl } from "./database-item-impl";
-import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
+import {
+  showBinaryChoiceDialog,
+  showNeverAskAgainDialog,
+} from "../../common/vscode/dialog";
 import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
@@ -32,7 +35,7 @@ import { QlPackGenerator } from "../../local-queries/qlpack-generator";
 import { asError, getErrorMessage } from "../../common/helpers-pure";
 import type { DatabaseItem, PersistedDatabaseItem } from "./database-item";
 import { redactableError } from "../../common/errors";
-import { remove } from "fs-extra";
+import { copy, remove, stat } from "fs-extra";
 import { containsPath } from "../../common/files";
 import type { DatabaseChangedEvent } from "./database-events";
 import { DatabaseEventKind } from "./database-events";
@@ -40,6 +43,7 @@ import { DatabaseResolver } from "./database-resolver";
 import { telemetryListener } from "../../common/vscode/telemetry";
 import type { LanguageContextStore } from "../../language-context-store";
 import type { DatabaseOrigin } from "./database-origin";
+import { ensureZippedSourceLocation } from "./database-contents";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -82,6 +86,10 @@ function eventFired<T>(
 }
 
 type OpenDatabaseOptions = {
+  /**
+   * A location that is managed by the extension.
+   */
+  extensionManagedLocation?: string;
   isTutorialDatabase?: boolean;
   /**
    * Whether to add a workspace folder containing the source archive to the workspace. Default is true.
@@ -116,6 +124,7 @@ export class DatabaseManager extends DisposableObject {
     super();
 
     qs.onStart(this.reregisterDatabases.bind(this));
+    qs.onQueryRunStarting(this.maybeReimportTestDatabase.bind(this));
 
     this.push(
       this.languageContext.onLanguageContextChanged(async () => {
@@ -141,6 +150,7 @@ export class DatabaseManager extends DisposableObject {
     makeSelected = true,
     displayName?: string,
     {
+      extensionManagedLocation,
       isTutorialDatabase = false,
       addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
     }: OpenDatabaseOptions = {},
@@ -149,6 +159,7 @@ export class DatabaseManager extends DisposableObject {
       uri,
       origin,
       displayName,
+      extensionManagedLocation,
     );
 
     return await this.addExistingDatabaseItem(
@@ -159,6 +170,108 @@ export class DatabaseManager extends DisposableObject {
     );
   }
 
+  /**
+   * Finds a test database that was originally imported from `uri`.
+   * A test database is creeated by the `codeql test run` command
+   * and ends with `.testproj`.
+   * @param uri The original location of the database
+   * @returns The first database item found that matches the uri
+   */
+  public findTestDatabase(uri: vscode.Uri): DatabaseItem | undefined {
+    const originPath = uri.fsPath;
+    for (const item of this._databaseItems) {
+      if (item.origin?.type === "testproj" && item.origin.path === originPath) {
+        return item;
+      }
+    }
+    return undefined;
+  }
+
+  public async maybeReimportTestDatabase(
+    databaseUri: vscode.Uri,
+    forceImport = false,
+  ): Promise<void> {
+    const res = await this.isTestDatabaseOutdated(databaseUri);
+    if (!res) {
+      return;
+    }
+    const doit =
+      forceImport ||
+      (await showBinaryChoiceDialog(
+        "This test database is outdated. Do you want to reimport it?",
+      ));
+
+    if (doit) {
+      await this.reimportTestDatabase(databaseUri);
+    }
+  }
+
+  /**
+   * Checks if the origin of the imported database is newer.
+   * The imported database must be a test database.
+   * @param databaseUri the URI of the imported database to check
+   * @returns true if both databases exist and the origin database is newer.
+   */
+  private async isTestDatabaseOutdated(
+    databaseUri: vscode.Uri,
+  ): Promise<boolean> {
+    const dbItem = this.findDatabaseItem(databaseUri);
+    if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
+      return false;
+    }
+
+    // Compare timestmps of the codeql-database.yml files of the original and the
+    // imported databases.
+    const originDbYml = join(dbItem.origin.path, "codeql-database.yml");
+    const importedDbYml = join(
+      dbItem.databaseUri.fsPath,
+      "codeql-database.yml",
+    );
+
+    let originStat;
+    try {
+      originStat = await stat(originDbYml);
+    } catch (e) {
+      // if there is an error here, assume that the origin database
+      // is no longer available. Safely ignore and do not try to re-import.
+      return false;
+    }
+
+    try {
+      const importedStat = await stat(importedDbYml);
+      return originStat.mtimeMs > importedStat.mtimeMs;
+    } catch (e) {
+      // If either of the files does not exist, we assume the origin is newer.
+      // This shouldn't happen unless the user manually deleted one of the files.
+      return true;
+    }
+  }
+
+  /**
+   * Reimport the specified imported database from its origin.
+   * The imported databsae must be a testproj database.
+   *
+   * @param databaseUri the URI of the imported database to reimport
+   */
+  private async reimportTestDatabase(databaseUri: vscode.Uri): Promise<void> {
+    const dbItem = this.findDatabaseItem(databaseUri);
+    if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
+      throw new Error(`Database ${databaseUri} is not a testproj.`);
+    }
+
+    await this.removeDatabaseItem(dbItem);
+    await copy(dbItem.origin.path, databaseUri.fsPath);
+    await ensureZippedSourceLocation(databaseUri.fsPath);
+    const newDbItem = new DatabaseItemImpl(databaseUri, dbItem.contents, {
+      dateAdded: Date.now(),
+      language: dbItem.language,
+      origin: dbItem.origin,
+      extensionManagedLocation: dbItem.extensionManagedLocation,
+    });
+    await this.addDatabaseItem(newDbItem);
+    await this.setCurrentDatabaseItem(newDbItem);
+  }
+
   /**
    * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
    * the list.
@@ -202,6 +315,7 @@ export class DatabaseManager extends DisposableObject {
     uri: vscode.Uri,
     origin: DatabaseOrigin | undefined,
     displayName: string | undefined,
+    extensionManagedLocation?: string,
   ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
     const fullOptions: FullDatabaseOptions = {
@@ -210,6 +324,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
       origin,
+      extensionManagedLocation,
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
@@ -370,6 +485,7 @@ export class DatabaseManager extends DisposableObject {
     let dateAdded = undefined;
     let language = undefined;
     let origin = undefined;
+    let extensionManagedLocation = undefined;
     if (state.options) {
       if (typeof state.options.displayName === "string") {
         displayName = state.options.displayName;
@@ -379,6 +495,7 @@ export class DatabaseManager extends DisposableObject {
       }
       language = state.options.language;
       origin = state.options.origin;
+      extensionManagedLocation = state.options.extensionManagedLocation;
     }
 
     const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -392,6 +509,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded,
       language,
       origin,
+      extensionManagedLocation,
     };
     const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 
@@ -495,6 +613,7 @@ export class DatabaseManager extends DisposableObject {
       this._onDidChangeCurrentDatabaseItem.fire({
         item,
         kind: DatabaseEventKind.Change,
+        fullRefresh: false,
       });
     }
   }
@@ -544,8 +663,9 @@ export class DatabaseManager extends DisposableObject {
     }
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
-      item: undefined,
+      item,
       kind: DatabaseEventKind.Add,
+      fullRefresh: true,
     });
   }
 
@@ -553,9 +673,9 @@ export class DatabaseManager extends DisposableObject {
     item.name = newName;
     await this.updatePersistedDatabaseList();
     this._onDidChangeDatabaseItem.fire({
-      // pass undefined so that the entire tree is rebuilt in order to re-sort
-      item: undefined,
+      item,
       kind: DatabaseEventKind.Rename,
+      fullRefresh: true,
     });
   }
 
@@ -583,24 +703,29 @@ export class DatabaseManager extends DisposableObject {
     // Remove this database item from the allow-list
     await this.deregisterDatabase(item);
 
+    // Find whether we know directly which directory we should remove
+    const directoryToRemove = item.extensionManagedLocation
+      ? vscode.Uri.file(item.extensionManagedLocation)
+      : item.databaseUri;
+
     // Delete folder from file system only if it is controlled by the extension
-    if (this.isExtensionControlledLocation(item.databaseUri)) {
+    if (this.isExtensionControlledLocation(directoryToRemove)) {
       void extLogger.log("Deleting database from filesystem.");
-      await remove(item.databaseUri.fsPath).then(
-        () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`),
+      await remove(directoryToRemove.fsPath).then(
+        () => void extLogger.log(`Deleted '${directoryToRemove.fsPath}'`),
         (e: unknown) =>
           void extLogger.log(
             `Failed to delete '${
-              item.databaseUri.fsPath
+              directoryToRemove.fsPath
             }'. Reason: ${getErrorMessage(e)}`,
           ),
       );
     }
 
-    // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
-      item: undefined,
+      item,
       kind: DatabaseEventKind.Remove,
+      fullRefresh: true,
     });
   }
 
@@ -653,6 +778,7 @@ export class DatabaseManager extends DisposableObject {
       this._onDidChangeDatabaseItem.fire({
         kind: DatabaseEventKind.Refresh,
         item: databaseItem,
+        fullRefresh: false,
       });
     }
   }

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -5,10 +5,12 @@ export interface DatabaseOptions {
   dateAdded?: number | undefined;
   language?: string;
   origin?: DatabaseOrigin;
+  extensionManagedLocation?: string;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   dateAdded: number | undefined;
   language: string | undefined;
   origin: DatabaseOrigin | undefined;
+  extensionManagedLocation: string | undefined;
 }

extensions/ql-vscode/src/databases/local-databases/database-origin.ts

@@ -24,9 +24,15 @@ interface DatabaseOriginDebugger {
   type: "debugger";
 }
 
+interface DatabaseOriginTestProj {
+  type: "testproj";
+  path: string;
+}
+
 export type DatabaseOrigin =
   | DatabaseOriginFolder
   | DatabaseOriginArchive
   | DatabaseOriginGitHub
   | DatabaseOriginInternet
-  | DatabaseOriginDebugger;
+  | DatabaseOriginDebugger
+  | DatabaseOriginTestProj;

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -18,13 +18,12 @@ import type { DbManager } from "../db-manager";
 import { DbTreeDataProvider } from "./db-tree-data-provider";
 import type { DbTreeViewItem } from "./db-tree-view-item";
 import { getGitHubUrl } from "./db-tree-view-item-action";
-import { getControllerRepo } from "../../variant-analysis/run-remote-query";
-import { getErrorMessage } from "../../common/helpers-pure";
 import type { DatabasePanelCommands } from "../../common/commands";
 import type { App } from "../../common/app";
 import { QueryLanguage } from "../../common/query-language";
 import { getCodeSearchRepositories } from "../code-search-api";
 import { showAndLogErrorMessage } from "../../common/logging";
+import { getGitHubInstanceUrl } from "../../config";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
@@ -73,9 +72,6 @@ export class DbPanel extends DisposableObject {
         this.addNewRemoteDatabase.bind(this),
       "codeQLVariantAnalysisRepositories.addNewList":
         this.addNewList.bind(this),
-      "codeQLVariantAnalysisRepositories.setupControllerRepository":
-        this.setupControllerRepository.bind(this),
-
       "codeQLVariantAnalysisRepositories.setSelectedItem":
         this.setSelectedItem.bind(this),
       "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu":
@@ -146,16 +142,19 @@ export class DbPanel extends DisposableObject {
   }
 
   private async addNewRemoteRepo(parentList?: string): Promise<void> {
+    const instanceUrl = getGitHubInstanceUrl();
+
     const repoName = await window.showInputBox({
       title: "Add a repository",
       prompt: "Insert a GitHub repository URL or name with owner",
-      placeHolder: "<owner>/<repo> or https://github.com/<owner>/<repo>",
+      placeHolder: `<owner>/<repo> or ${new URL("/", instanceUrl).toString()}<owner>/<repo>`,
     });
     if (!repoName) {
       return;
     }
 
-    const nwo = getNwoFromGitHubUrl(repoName) || repoName;
+    const nwo =
+      getNwoFromGitHubUrl(repoName, getGitHubInstanceUrl()) || repoName;
     if (!isValidGitHubNwo(nwo)) {
       void showAndLogErrorMessage(
         this.app.logger,
@@ -176,17 +175,20 @@ export class DbPanel extends DisposableObject {
   }
 
   private async addNewRemoteOwner(): Promise<void> {
+    const instanceUrl = getGitHubInstanceUrl();
+
     const ownerName = await window.showInputBox({
       title: "Add all repositories of a GitHub org or owner",
       prompt: "Insert a GitHub organization or owner name",
-      placeHolder: "<owner> or https://github.com/<owner>",
+      placeHolder: `<owner> or ${new URL("/", instanceUrl).toString()}<owner>`,
     });
 
     if (!ownerName) {
       return;
     }
 
-    const owner = getOwnerFromGitHubUrl(ownerName) || ownerName;
+    const owner =
+      getOwnerFromGitHubUrl(ownerName, getGitHubInstanceUrl()) || ownerName;
     if (!isValidGitHubOwner(owner)) {
       void showAndLogErrorMessage(
         this.app.logger,
@@ -411,7 +413,7 @@ export class DbPanel extends DisposableObject {
     if (treeViewItem.dbItem === undefined) {
       throw new Error("Unable to open on GitHub. Please select a valid item.");
     }
-    const githubUrl = getGitHubUrl(treeViewItem.dbItem);
+    const githubUrl = getGitHubUrl(treeViewItem.dbItem, getGitHubInstanceUrl());
     if (!githubUrl) {
       throw new Error(
         "Unable to open on GitHub. Please select a variant analysis repository or owner.",
@@ -420,22 +422,4 @@ export class DbPanel extends DisposableObject {
 
     await this.app.commands.execute("vscode.open", Uri.parse(githubUrl));
   }
-
-  private async setupControllerRepository(): Promise<void> {
-    try {
-      // This will also validate that the controller repository is valid
-      await getControllerRepo(this.app.credentials);
-    } catch (e: unknown) {
-      if (e instanceof UserCancellationException) {
-        return;
-      }
-
-      void showAndLogErrorMessage(
-        this.app.logger,
-        `An error occurred while setting up the controller repository: ${getErrorMessage(
-          e,
-        )}`,
-      );
-    }
-  }
 }

extensions/ql-vscode/src/databases/ui/db-tree-data-provider.ts

@@ -83,11 +83,6 @@ export class DbTreeDataProvider
   }
 
   private createTree(): DbTreeViewItem[] {
-    // Returning an empty tree here will show the welcome view
-    if (!this.variantAnalysisConfig.controllerRepo) {
-      return [];
-    }
-
     const dbItemsResult = this.dbManager.getDbItems();
 
     if (dbItemsResult.isFailure) {

extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts

@@ -62,12 +62,15 @@ function canImportCodeSearch(dbItem: DbItem): boolean {
   return DbItemKind.RemoteUserDefinedList === dbItem.kind;
 }
 
-export function getGitHubUrl(dbItem: DbItem): string | undefined {
+export function getGitHubUrl(
+  dbItem: DbItem,
+  githubUrl: URL,
+): string | undefined {
   switch (dbItem.kind) {
     case DbItemKind.RemoteOwner:
-      return `https://github.com/${dbItem.ownerName}`;
+      return new URL(`/${dbItem.ownerName}`, githubUrl).toString();
     case DbItemKind.RemoteRepo:
-      return `https://github.com/${dbItem.repoFullName}`;
+      return new URL(`/${dbItem.repoFullName}`, githubUrl).toString();
     default:
       return undefined;
   }

extensions/ql-vscode/src/debugger/debugger-ui.ts

@@ -135,7 +135,7 @@ class QLDebugAdapterTracker
   ): Promise<void> {
     if (this.localQueryRun !== undefined) {
       const results: CoreQueryResults = body;
-      await this.localQueryRun.complete(results);
+      await this.localQueryRun.complete(results, (_) => {});
       this.localQueryRun = undefined;
     }
   }

extensions/ql-vscode/src/extension.ts

@@ -31,6 +31,7 @@ import {
   joinOrderWarningThreshold,
   QueryHistoryConfigListener,
   QueryServerConfigListener,
+  VariantAnalysisConfigListener,
 } from "./config";
 import {
   AstViewer,
@@ -109,7 +110,7 @@ import { VariantAnalysisManager } from "./variant-analysis/variant-analysis-mana
 import { createVariantAnalysisContentProvider } from "./variant-analysis/variant-analysis-content-provider";
 import { VSCodeMockGitHubApiServer } from "./common/mock-gh-api/vscode/vscode-mock-gh-api-server";
 import { VariantAnalysisResultsManager } from "./variant-analysis/variant-analysis-results-manager";
-import { ExtensionApp } from "./common/vscode/vscode-app";
+import { ExtensionApp } from "./common/vscode/extension-app";
 import { DbModule } from "./databases/db-module";
 import { redactableError } from "./common/errors";
 import { QLDebugAdapterDescriptorFactory } from "./debugger/debugger-factory";
@@ -133,6 +134,7 @@ import { OpenReferencedFileCodeLensProvider } from "./local-queries/open-referen
 import { LanguageContextStore } from "./language-context-store";
 import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
 import { GitHubDatabasesModule } from "./databases/github-databases";
+import { DatabaseFetcher } from "./databases/database-fetcher";
 
 /**
  * extension.ts
@@ -183,11 +185,11 @@ function getCommands(
 
   const restartQueryServer = async () =>
     withProgress(
-      async (progress: ProgressCallback, token: CancellationToken) => {
+      async (progress: ProgressCallback) => {
         // Restart all of the spawned servers: cli, query, and language.
         cliServer.restartCliServer();
         await Promise.all([
-          queryRunner.restartQueryServer(progress, token),
+          queryRunner.restartQueryServer(progress),
           async () => {
             if (languageClient.isRunning()) {
               await languageClient.restart();
@@ -445,8 +447,9 @@ export async function activate(
       }
 
       if (
-        CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION.compare(ver.version) <
-        0
+        CliVersionConstraint.OLDEST_SUPPORTED_CLI_VERSION.compare(
+          ver.version,
+        ) <= 0
       ) {
         return;
       }
@@ -714,12 +717,13 @@ async function installOrUpdateThenTryActivate(
   return undefined;
 }
 
-const PACK_GLOBS = [
+const CLEAR_PACK_CACHE_ON_EDIT_GLOBS = [
   "**/codeql-pack.yml",
   "**/qlpack.yml",
   "**/queries.xml",
   "**/codeql-pack.lock.yml",
   "**/qlpack.lock.yml",
+  "**/*.dbscheme",
   ".codeqlmanifest.json",
   "codeql-workspace.yml",
 ];
@@ -766,7 +770,7 @@ async function activateWithInstalledDistribution(
     ctx,
   );
 
-  for (const glob of PACK_GLOBS) {
+  for (const glob of CLEAR_PACK_CACHE_ON_EDIT_GLOBS) {
     const fsWatcher = workspace.createFileSystemWatcher(glob);
     ctx.subscriptions.push(fsWatcher);
 
@@ -799,12 +803,20 @@ async function activateWithInstalledDistribution(
   // Let this run async.
   void dbm.loadPersistedState();
 
+  const databaseFetcher = new DatabaseFetcher(
+    app,
+    dbm,
+    getContextStoragePath(ctx),
+    cliServer,
+  );
+
   ctx.subscriptions.push(dbm);
 
   void extLogger.log("Initializing database panel.");
   const databaseUI = new DatabaseUI(
     app,
     dbm,
+    databaseFetcher,
     languageContext,
     qs,
     getContextStoragePath(ctx),
@@ -855,8 +867,10 @@ async function activateWithInstalledDistribution(
     "variant-analyses",
   );
   await ensureDir(variantAnalysisStorageDir);
+  const variantAnalysisConfig = new VariantAnalysisConfigListener();
   const variantAnalysisResultsManager = new VariantAnalysisResultsManager(
     cliServer,
+    variantAnalysisConfig,
     extLogger,
   );
 
@@ -866,6 +880,7 @@ async function activateWithInstalledDistribution(
     variantAnalysisStorageDir,
     variantAnalysisResultsManager,
     dbModule.dbManager,
+    variantAnalysisConfig,
   );
   ctx.subscriptions.push(variantAnalysisManager);
   ctx.subscriptions.push(variantAnalysisResultsManager);
@@ -881,8 +896,7 @@ async function activateWithInstalledDistribution(
   await GitHubDatabasesModule.initialize(
     app,
     dbm,
-    getContextStoragePath(ctx),
-    cliServer,
+    databaseFetcher,
     githubDatabaseConfigListener,
   );
 
@@ -934,7 +948,7 @@ async function activateWithInstalledDistribution(
   ctx.subscriptions.push(compareView);
 
   void extLogger.log("Initializing source archive filesystem provider.");
-  archiveFilesystemProvider_activate(ctx);
+  archiveFilesystemProvider_activate(ctx, dbm);
 
   const qhelpTmpDir = dirSync({
     prefix: "qhelp_",
@@ -953,6 +967,7 @@ async function activateWithInstalledDistribution(
     qs,
     qhm,
     dbm,
+    databaseFetcher,
     cliServer,
     databaseUI,
     localQueryResultsView,
@@ -977,6 +992,8 @@ async function activateWithInstalledDistribution(
   const modelEditorModule = await ModelEditorModule.initialize(
     app,
     dbm,
+    databaseFetcher,
+    variantAnalysisManager,
     cliServer,
     qs,
     tmpDir.name,
@@ -1067,14 +1084,12 @@ async function activateWithInstalledDistribution(
   });
 
   // Handle visibility changes in the CodeQL language client.
-  if (await cliServer.cliConstraints.supportsVisibilityNotifications()) {
-    Window.onDidChangeVisibleTextEditors((editors) => {
-      languageClient.notifyVisibilityChange(editors);
-    });
-    // Send an inital notification to the language server
-    // to set the initial state of the visible editors.
-    languageClient.notifyVisibilityChange(Window.visibleTextEditors);
-  }
+  Window.onDidChangeVisibleTextEditors((editors) => {
+    languageClient.notifyVisibilityChange(editors);
+  });
+  // Send an inital notification to the language server
+  // to set the initial state of the visible editors.
+  languageClient.notifyVisibilityChange(Window.visibleTextEditors);
 
   // Jump-to-definition and find-references
   void extLogger.log("Registering jump-to-definition handlers.");

extensions/ql-vscode/src/language-support/contextual/query-resolver.ts

@@ -92,11 +92,12 @@ export async function runContextualQuery(
   void extLogger.log(
     `Running contextual query ${query}; results will be stored in ${queryRun.outputDir.querySaveDir}`,
   );
-  const results = await queryRun.evaluate(
-    progress,
-    token,
-    new TeeLogger(qs.logger, queryRun.outputDir.logPath),
-  );
-  await cleanup?.();
-  return results;
+  const teeLogger = new TeeLogger(qs.logger, queryRun.outputDir.logPath);
+
+  try {
+    return await queryRun.evaluate(progress, token, teeLogger);
+  } finally {
+    await cleanup?.();
+    teeLogger.dispose();
+  }
 }

extensions/ql-vscode/src/local-queries/local-queries.ts

@@ -2,7 +2,10 @@ import type {
   ProgressCallback,
   ProgressUpdate,
 } from "../common/vscode/progress";
-import { withProgress } from "../common/vscode/progress";
+import {
+  UserCancellationException,
+  withProgress,
+} from "../common/vscode/progress";
 import type { CancellationToken, Range, TabInputText } from "vscode";
 import { CancellationTokenSource, Uri, window } from "vscode";
 import {
@@ -10,7 +13,7 @@ import {
   showAndLogErrorMessage,
   showAndLogWarningMessage,
 } from "../common/logging";
-import { isCanary, MAX_QUERIES } from "../config";
+import { MAX_QUERIES } from "../config";
 import { gatherQlFiles } from "../common/files";
 import { basename } from "path";
 import { showBinaryChoiceDialog } from "../common/vscode/dialog";
@@ -41,7 +44,6 @@ import { LocalQueryInfo } from "../query-results";
 import type { WebviewReveal } from "./webview";
 import { asError, getErrorMessage } from "../common/helpers-pure";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
-import { CliVersionConstraint } from "../codeql-cli/cli";
 import type { LocalQueryCommands } from "../common/commands";
 import { DisposableObject } from "../common/disposable-object";
 import { SkeletonQueryWizard } from "./skeleton-query-wizard";
@@ -51,7 +53,8 @@ import { findLanguage } from "../codeql-cli/query-language";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
 import { tryGetQueryLanguage } from "../common/query-language";
 import type { LanguageContextStore } from "../language-context-store";
-import type { ExtensionApp } from "../common/vscode/vscode-app";
+import type { ExtensionApp } from "../common/vscode/extension-app";
+import type { DatabaseFetcher } from "../databases/database-fetcher";
 
 export enum QuickEvalType {
   None,
@@ -67,6 +70,7 @@ export class LocalQueries extends DisposableObject {
     private readonly queryRunner: QueryRunner,
     private readonly queryHistoryManager: QueryHistoryManager,
     private readonly databaseManager: DatabaseManager,
+    private readonly databaseFetcher: DatabaseFetcher,
     private readonly cliServer: CodeQLCliServer,
     private readonly databaseUI: DatabaseUI,
     private readonly localQueryResultsView: ResultsView,
@@ -256,11 +260,6 @@ export class LocalQueries extends DisposableObject {
   private async quickEvalCount(uri: Uri): Promise<void> {
     await withProgress(
       async (progress, token) => {
-        if (!(await this.cliServer.cliConstraints.supportsQuickEvalCount())) {
-          throw new Error(
-            `Quick evaluation count is only supported by CodeQL CLI v${CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT} or later.`,
-          );
-        }
         await this.compileAndRunQuery(
           QuickEvalType.QuickEvalCount,
           uri,
@@ -296,14 +295,8 @@ export class LocalQueries extends DisposableObject {
 
   private async quickQuery(): Promise<void> {
     await withProgress(
-      async (progress, token) =>
-        displayQuickQuery(
-          this.app,
-          this.cliServer,
-          this.databaseUI,
-          progress,
-          token,
-        ),
+      async (progress) =>
+        displayQuickQuery(this.app, this.cliServer, this.databaseUI, progress),
       {
         title: "Run Quick Query",
       },
@@ -328,17 +321,13 @@ export class LocalQueries extends DisposableObject {
   private async createSkeletonQuery(): Promise<void> {
     await withProgress(
       async (progress: ProgressCallback) => {
-        const credentials = isCanary() ? this.app.credentials : undefined;
-        const contextStoragePath =
-          this.app.workspaceStoragePath || this.app.globalStoragePath;
         const language = this.languageContextStore.selectedLanguage;
         const skeletonQueryWizard = new SkeletonQueryWizard(
           this.cliServer,
           progress,
-          credentials,
           this.app,
           this.databaseManager,
-          contextStoragePath,
+          this.databaseFetcher,
           this.selectedQueryTreeViewItems,
           language,
         );
@@ -453,7 +442,7 @@ export class LocalQueries extends DisposableObject {
 
     // If no databaseItem is specified, use the database currently selected in the Databases UI
     databaseItem =
-      databaseItem ?? (await this.databaseUI.getDatabaseItem(progress, token));
+      databaseItem ?? (await this.databaseUI.getDatabaseItem(progress));
     if (databaseItem === undefined) {
       throw new Error("Can't run query without a selected database");
     }
@@ -496,7 +485,7 @@ export class LocalQueries extends DisposableObject {
           localQueryRun.logger,
         );
 
-        await localQueryRun.complete(results);
+        await localQueryRun.complete(results, progress);
 
         return results;
       } catch (e) {
@@ -506,7 +495,12 @@ export class LocalQueries extends DisposableObject {
         // to unify both error handling paths.
         const err = asError(e);
         await localQueryRun.fail(err);
-        throw e;
+
+        if (token.isCancellationRequested) {
+          throw new UserCancellationException(err.message, true);
+        } else {
+          throw e;
+        }
       }
     } finally {
       source.dispose();
@@ -594,7 +588,7 @@ export class LocalQueries extends DisposableObject {
   public async getDefaultExtensionPacks(
     additionalPacks: string[],
   ): Promise<string[]> {
-    return (await this.cliServer.useExtensionPacks())
+    return this.cliServer.useExtensionPacks()
       ? Object.keys(await this.cliServer.resolveQlpacks(additionalPacks, true))
       : [];
   }

extensions/ql-vscode/src/local-queries/local-query-run.ts

@@ -25,6 +25,9 @@ import { redactableError } from "../common/errors";
 import type { LocalQueries } from "./local-queries";
 import { tryGetQueryMetadata } from "../codeql-cli/query-metadata";
 import { telemetryListener } from "../common/vscode/telemetry";
+import type { Disposable } from "../common/disposable-object";
+import type { ProgressCallback } from "../common/vscode/progress";
+import { progressUpdate } from "../common/vscode/progress";
 
 function formatResultMessage(result: CoreQueryResults): string {
   switch (result.resultType) {
@@ -61,7 +64,11 @@ export class LocalQueryRun {
     private readonly localQueries: LocalQueries,
     private readonly queryInfo: LocalQueryInfo,
     private readonly dbItem: DatabaseItem,
-    public readonly logger: Logger, // Public so that other clients, like the debug adapter, know where to send log output
+    /**
+     * The logger is only available while the query is running and will be disposed of when the
+     * query completes.
+     */
+    public readonly logger: Logger & Disposable, // Public so that other clients, like the debug adapter, know where to send log output
     private readonly queryHistoryManager: QueryHistoryManager,
     private readonly cliServer: CodeQLCliServer,
   ) {}
@@ -74,24 +81,34 @@ export class LocalQueryRun {
    * This function must be called when the evaluation completes, whether the evaluation was
    * successful or not.
    * */
-  public async complete(results: CoreQueryResults): Promise<void> {
+  public async complete(
+    results: CoreQueryResults,
+    progress: ProgressCallback,
+  ): Promise<void> {
     const evalLogPaths = await this.summarizeEvalLog(
       results.resultType,
       this.outputDir,
       this.logger,
+      progress,
     );
     if (evalLogPaths !== undefined) {
       this.queryInfo.setEvaluatorLogPaths(evalLogPaths);
     }
+    progress(progressUpdate(1, 4, "Getting completed query info"));
     const queryWithResults = await this.getCompletedQueryInfo(results);
+    progress(progressUpdate(2, 4, "Updating query history"));
     this.queryHistoryManager.completeQuery(this.queryInfo, queryWithResults);
+    progress(progressUpdate(3, 4, "Showing results"));
     await this.localQueries.showResultsForCompletedQuery(
       this.queryInfo as CompletedLocalQueryInfo,
       WebviewReveal.Forced,
     );
     // Note we must update the query history view after showing results as the
     // display and sorting might depend on the number of results
+    progress(progressUpdate(4, 4, "Updating query history"));
     await this.queryHistoryManager.refreshTreeView();
+
+    this.logger.dispose();
   }
 
   /**
@@ -102,6 +119,7 @@ export class LocalQueryRun {
       QueryResultType.OTHER_ERROR,
       this.outputDir,
       this.logger,
+      (_) => {},
     );
     if (evalLogPaths !== undefined) {
       this.queryInfo.setEvaluatorLogPaths(evalLogPaths);
@@ -110,6 +128,8 @@ export class LocalQueryRun {
     err.message = `Error running query: ${err.message}`;
     this.queryInfo.failureReason = err.message;
     await this.queryHistoryManager.refreshTreeView();
+
+    this.logger.dispose();
   }
 
   /**
@@ -119,10 +139,12 @@ export class LocalQueryRun {
     resultType: QueryResultType,
     outputDir: QueryOutputDir,
     logger: BaseLogger,
+    progress: ProgressCallback,
   ): Promise<EvaluatorLogPaths | undefined> {
     const evalLogPaths = await generateEvalLogSummaries(
       this.cliServer,
       outputDir,
+      progress,
     );
     if (evalLogPaths !== undefined) {
       if (evalLogPaths.endSummary !== undefined) {

extensions/ql-vscode/src/local-queries/quick-query.ts

@@ -1,7 +1,6 @@
 import { ensureDir, writeFile, pathExists, readFile } from "fs-extra";
 import { dump, load } from "js-yaml";
 import { basename, join } from "path";
-import type { CancellationToken } from "vscode";
 import { window as Window, workspace, Uri } from "vscode";
 import { LSPErrorCodes, ResponseError } from "vscode-languageclient";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
@@ -13,7 +12,7 @@ import { UserCancellationException } from "../common/vscode/progress";
 import { getErrorMessage } from "../common/helpers-pure";
 import { FALLBACK_QLPACK_FILENAME, getQlPackFilePath } from "../common/ql";
 import type { App } from "../common/app";
-import type { ExtensionApp } from "../common/vscode/vscode-app";
+import type { ExtensionApp } from "../common/vscode/extension-app";
 import type { QlPackFile } from "../packaging/qlpack-file";
 
 const QUICK_QUERIES_DIR_NAME = "quick-queries";
@@ -56,7 +55,6 @@ export async function displayQuickQuery(
   cliServer: CodeQLCliServer,
   databaseUI: DatabaseUI,
   progress: ProgressCallback,
-  token: CancellationToken,
 ) {
   try {
     // If there is already a quick query open, don't clobber it, just
@@ -111,7 +109,7 @@ export async function displayQuickQuery(
     }
 
     // We're going to infer which qlpack to use from the current database
-    const dbItem = await databaseUI.getDatabaseItem(progress, token);
+    const dbItem = await databaseUI.getDatabaseItem(progress);
     if (dbItem === undefined) {
       throw new Error("Can't start quick query without a selected database");
     }

extensions/ql-vscode/src/local-queries/run-query.ts

@@ -47,21 +47,27 @@ export async function runQuery({
     undefined,
   );
 
-  const completedQuery = await queryRun.evaluate(
-    progress,
-    token,
-    new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
+  const teeLogger = new TeeLogger(
+    queryRunner.logger,
+    queryRun.outputDir.logPath,
   );
 
-  if (completedQuery.resultType !== QueryResultType.SUCCESS) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Failed to run ${basename(queryPath)} query: ${
-        completedQuery.message ?? "No message"
-      }`,
-    );
-    return;
+  try {
+    const completedQuery = await queryRun.evaluate(progress, token, teeLogger);
+
+    if (completedQuery.resultType !== QueryResultType.SUCCESS) {
+      void showAndLogExceptionWithTelemetry(
+        extLogger,
+        telemetryListener,
+        redactableError`Failed to run ${basename(queryPath)} query: ${
+          completedQuery.message ?? "No message"
+        }`,
+      );
+      return;
+    }
+
+    return completedQuery;
+  } finally {
+    teeLogger.dispose();
   }
-  return completedQuery;
 }

extensions/ql-vscode/src/local-queries/skeleton-query-wizard.ts

@@ -2,7 +2,6 @@ import { dirname, join } from "path";
 import { Uri, window, window as Window, workspace } from "vscode";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import { showAndLogExceptionWithTelemetry } from "../common/logging";
-import type { Credentials } from "../common/authentication";
 import type { QueryLanguage } from "../common/query-language";
 import { getLanguageDisplayName } from "../common/query-language";
 import {
@@ -20,10 +19,7 @@ import {
   UserCancellationException,
   withProgress,
 } from "../common/vscode/progress";
-import {
-  askForGitHubRepo,
-  downloadGitHubDatabase,
-} from "../databases/database-fetcher";
+import type { DatabaseFetcher } from "../databases/database-fetcher";
 import {
   getQlPackLocation,
   isCodespacesTemplate,
@@ -61,10 +57,9 @@ export class SkeletonQueryWizard {
   constructor(
     private readonly cliServer: CodeQLCliServer,
     private readonly progress: ProgressCallback,
-    private readonly credentials: Credentials | undefined,
     private readonly app: App,
     private readonly databaseManager: DatabaseManager,
-    private readonly databaseStoragePath: string | undefined,
+    private readonly databaseFetcher: DatabaseFetcher,
     private readonly selectedItems: readonly QueryTreeViewItem[],
     private language: QueryLanguage | undefined = undefined,
   ) {}
@@ -365,10 +360,6 @@ export class SkeletonQueryWizard {
   }
 
   private async downloadDatabase(progress: ProgressCallback) {
-    if (this.databaseStoragePath === undefined) {
-      throw new Error("Database storage path is undefined");
-    }
-
     if (this.language === undefined) {
       throw new Error("Language is undefined");
     }
@@ -380,20 +371,10 @@ export class SkeletonQueryWizard {
     });
 
     const githubRepoNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[this.language];
-    const chosenRepo = await askForGitHubRepo(undefined, githubRepoNwo);
-
-    if (!chosenRepo) {
-      throw new UserCancellationException("No GitHub repository provided");
-    }
-
-    await downloadGitHubDatabase(
-      chosenRepo,
-      this.databaseManager,
-      this.databaseStoragePath,
-      this.credentials,
+    await this.databaseFetcher.promptImportGithubDatabase(
       progress,
-      this.cliServer,
       this.language,
+      githubRepoNwo,
     );
   }
 

extensions/ql-vscode/src/model-editor/auto-model.ts

@@ -5,53 +5,6 @@ import type { AutoModelQueriesResult } from "./auto-model-codeml-queries";
 import { assertNever } from "../common/helpers-pure";
 import type { Log } from "sarif";
 import { gzipEncode } from "../common/zlib";
-import type { Method, MethodSignature } from "./method";
-import type { ModeledMethod } from "./modeled-method";
-import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
-
-/**
- * Return the candidates that the model should be run on. This includes limiting the number of
- * candidates to the candidate limit and filtering out anything that is already modeled and respecting
- * the order in the UI.
- * @param mode Whether it is application or framework mode.
- * @param methods all methods.
- * @param modeledMethodsBySignature the currently modeled methods.
- * @returns list of modeled methods that are candidates for modeling.
- */
-export function getCandidates(
-  mode: Mode,
-  methods: readonly Method[],
-  modeledMethodsBySignature: Record<string, readonly ModeledMethod[]>,
-): MethodSignature[] {
-  // Sort the same way as the UI so we send the first ones listed in the UI first
-  const grouped = groupMethods(methods, mode);
-  const sortedGroupNames = sortGroupNames(grouped);
-  const sortedMethods = sortedGroupNames.flatMap((name) =>
-    sortMethods(grouped[name]),
-  );
-
-  const candidates: MethodSignature[] = [];
-
-  for (const method of sortedMethods) {
-    const modeledMethods: ModeledMethod[] = [
-      ...(modeledMethodsBySignature[method.signature] ?? []),
-    ];
-
-    // Anything that is modeled is not a candidate
-    if (modeledMethods.some((m) => m.type !== "none")) {
-      continue;
-    }
-
-    // A method that is supported is modeled outside of the model file, so it is not a candidate.
-    if (method.supported) {
-      continue;
-    }
-
-    // The rest are candidates
-    candidates.push(method);
-  }
-  return candidates;
-}
 
 /**
  * Encode a SARIF log to the format expected by the server: JSON, GZIP-compressed, base64-encoded

extensions/ql-vscode/src/model-editor/auto-modeler.ts

@@ -3,7 +3,8 @@ import type { ModeledMethod } from "./modeled-method";
 import { load as loadYaml } from "js-yaml";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { withProgress } from "../common/vscode/progress";
-import { createAutoModelRequest, getCandidates } from "./auto-model";
+import { createAutoModelRequest } from "./auto-model";
+import { getCandidates } from "./shared/auto-model-candidates";
 import { runAutoModelQueries } from "./auto-model-codeml-queries";
 import { loadDataExtensionYaml } from "./yaml";
 import type { ModelRequest, ModelResponse } from "./auto-model-api";
@@ -58,6 +59,7 @@ export class AutoModeler {
     packageName: string,
     methods: readonly Method[],
     modeledMethods: Record<string, readonly ModeledMethod[]>,
+    processedByAutoModelMethods: Set<string>,
     mode: Mode,
   ): Promise<void> {
     if (this.jobs.has(packageName)) {
@@ -72,6 +74,7 @@ export class AutoModeler {
         packageName,
         methods,
         modeledMethods,
+        processedByAutoModelMethods,
         mode,
         cancellationTokenSource,
       );
@@ -105,6 +108,7 @@ export class AutoModeler {
     packageName: string,
     methods: readonly Method[],
     modeledMethods: Record<string, readonly ModeledMethod[]>,
+    processedByAutoModelMethods: Set<string>,
     mode: Mode,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
@@ -114,7 +118,12 @@ export class AutoModeler {
 
     await withProgress(async (progress) => {
       // Fetch the candidates to send to the model
-      const allCandidateMethods = getCandidates(mode, methods, modeledMethods);
+      const allCandidateMethods = getCandidates(
+        mode,
+        methods,
+        modeledMethods,
+        processedByAutoModelMethods,
+      );
 
       // If there are no candidates, there is nothing to model and we just return
       if (allCandidateMethods.length === 0) {
@@ -159,6 +168,12 @@ export class AutoModeler {
             this.databaseItem,
             candidateSignatures,
           );
+
+          // Let the UI know which methods have been sent to the LLM
+          this.modelingStore.addProcessedByAutoModelMethods(
+            this.databaseItem,
+            candidateSignatures,
+          );
         }
       } finally {
         // Clear out in progress methods in case anything went wrong
@@ -223,7 +238,7 @@ export class AutoModeler {
         void showAndLogExceptionWithTelemetry(
           this.app.logger,
           this.app.telemetry,
-          redactableError(e)`Rate limit hit, please try again soon.`,
+          redactableError`Rate limit hit, please try again soon.`,
         );
         return null;
       } else {

extensions/ql-vscode/src/model-editor/extension-pack-metadata.schema.json

@@ -67,6 +67,9 @@
                 "$ref": "#/definitions/SuiteInstruction"
               }
             },
+            {
+              "$ref": "#/definitions/SuiteInstruction"
+            },
             {
               "type": "null"
             }
@@ -93,24 +96,41 @@
         "include": {
           "type": "object",
           "additionalProperties": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            }
+            "anyOf": [
+              {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              {
+                "type": "string"
+              }
+            ]
           }
         },
         "exclude": {
           "type": "object",
           "additionalProperties": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            }
+            "anyOf": [
+              {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              {
+                "type": "string"
+              }
+            ]
           }
         },
         "description": {
           "type": "string"
         },
+        "import": {
+          "type": "string"
+        },
         "from": {
           "type": "string"
         }

extensions/ql-vscode/src/model-editor/extension-pack-name.ts

@@ -1,3 +1,5 @@
+import { createFilenameFromString } from "../common/filenames";
+
 const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
 const packNameRegex = new RegExp(
   `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
@@ -13,17 +15,29 @@ export function formatPackName(packName: ExtensionPackName): string {
   return `${packName.scope}/${packName.name}`;
 }
 
-export function autoNameExtensionPack(
-  name: string,
-  language: string,
-): ExtensionPackName | undefined {
-  let packName = `${name}-${language}`;
+export function sanitizePackName(userPackName: string): ExtensionPackName {
+  let packName = userPackName;
+
+  packName = packName.trim();
+
+  while (packName.startsWith("/")) {
+    packName = packName.slice(1);
+  }
+
+  while (packName.endsWith("/")) {
+    packName = packName.slice(0, -1);
+  }
+
   if (!packName.includes("/")) {
     packName = `pack/${packName}`;
   }
 
   const parts = packName.split("/");
-  const sanitizedParts = parts.map((part) => sanitizeExtensionPackName(part));
+  const sanitizedParts = parts.map((part) =>
+    createFilenameFromString(part, {
+      removeDots: true,
+    }),
+  );
 
   // If the scope is empty (e.g. if the given name is "-/b"), then we need to still set a scope
   if (sanitizedParts[0].length === 0) {
@@ -37,25 +51,6 @@ export function autoNameExtensionPack(
   };
 }
 
-function sanitizeExtensionPackName(name: string) {
-  // Lowercase everything
-  name = name.toLowerCase();
-
-  // Replace all spaces, dots, and underscores with hyphens
-  name = name.replaceAll(/[\s._]+/g, "-");
-
-  // Replace all characters which are not allowed by empty strings
-  name = name.replaceAll(/[^a-z0-9-]/g, "");
-
-  // Remove any leading or trailing hyphens
-  name = name.replaceAll(/^-|-$/g, "");
-
-  // Remove any duplicate hyphens
-  name = name.replaceAll(/-{2,}/g, "-");
-
-  return name;
-}
-
 export function parsePackName(packName: string): ExtensionPackName | undefined {
   const matches = packNameRegex.exec(packName);
   if (!matches?.groups) {

extensions/ql-vscode/src/model-editor/extension-pack-picker.ts

@@ -2,7 +2,6 @@ import { join } from "path";
 import { outputFile, pathExists, readFile } from "fs-extra";
 import { dump as dumpYaml, load as loadYaml } from "js-yaml";
 import type { CancellationToken } from "vscode";
-import { Uri } from "vscode";
 import Ajv from "ajv";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
@@ -14,10 +13,17 @@ import { getErrorMessage } from "../common/helpers-pure";
 import type { ExtensionPack } from "./shared/extension-pack";
 import type { NotificationLogger } from "../common/logging";
 import { showAndLogErrorMessage } from "../common/logging";
-import type { ModelConfig } from "../config";
+import type { ModelConfig, ModelConfigPackVariables } from "../config";
 import type { ExtensionPackName } from "./extension-pack-name";
-import { autoNameExtensionPack, formatPackName } from "./extension-pack-name";
-import { autoPickExtensionsDirectory } from "./extensions-workspace-folder";
+import {
+  validatePackName,
+  sanitizePackName,
+  formatPackName,
+} from "./extension-pack-name";
+import {
+  ensurePackLocationIsInWorkspaceFolder,
+  packLocationToAbsolute,
+} from "./extensions-workspace-folder";
 
 import type { ExtensionPackMetadata } from "./extension-pack-metadata";
 import extensionPackMetadataSchemaJson from "./extension-pack-metadata.schema.json";
@@ -27,7 +33,7 @@ const extensionPackValidate = ajv.compile(extensionPackMetadataSchemaJson);
 
 export async function pickExtensionPack(
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
-  databaseItem: Pick<DatabaseItem, "name" | "language">,
+  databaseItem: Pick<DatabaseItem, "name" | "language" | "origin">,
   modelConfig: ModelConfig,
   logger: NotificationLogger,
   progress: ProgressCallback,
@@ -64,29 +70,34 @@ export async function pickExtensionPack(
     maxStep,
   });
 
-  // Get the `codeQL.model.extensionsDirectory` setting for the language
-  const userExtensionsDirectory = modelConfig.getExtensionsDirectory(
-    databaseItem.language,
+  // The default is .github/codeql/extensions/${name}-${language}
+  const packPath = await packLocationToAbsolute(
+    modelConfig.getPackLocation(
+      databaseItem.language,
+      getModelConfigPackVariables(databaseItem),
+    ),
+    logger,
   );
-
-  // If the setting is not set, automatically pick a suitable directory
-  const extensionsDirectory = userExtensionsDirectory
-    ? Uri.file(userExtensionsDirectory)
-    : await autoPickExtensionsDirectory(logger);
-
-  if (!extensionsDirectory) {
+  if (!packPath) {
     return undefined;
   }
 
-  // Generate the name of the extension pack
-  const packName = autoNameExtensionPack(
-    databaseItem.name,
+  await ensurePackLocationIsInWorkspaceFolder(packPath, modelConfig, logger);
+
+  const userPackName = modelConfig.getPackName(
     databaseItem.language,
+    getModelConfigPackVariables(databaseItem),
   );
-  if (!packName) {
+
+  // Generate the name of the extension pack
+  const packName = sanitizePackName(userPackName);
+
+  // Validate that the name isn't too long etc.
+  const packNameError = validatePackName(formatPackName(packName));
+  if (packNameError) {
     void showAndLogErrorMessage(
       logger,
-      `Could not automatically name extension pack for database ${databaseItem.name}`,
+      `Invalid model pack name '${formatPackName(packName)}' for database ${databaseItem.name}: ${packNameError}`,
     );
 
     return undefined;
@@ -139,14 +150,12 @@ export async function pickExtensionPack(
     return undefined;
   }
 
-  const packPath = join(extensionsDirectory.fsPath, packName.name);
-
   if (await pathExists(packPath)) {
     void showAndLogErrorMessage(
       logger,
       `Directory ${packPath} already exists for extension pack ${formatPackName(
         packName,
-      )}`,
+      )}, but wasn't returned by codeql resolve qlpacks --kind extension --no-recursive`,
     );
 
     return undefined;
@@ -155,6 +164,26 @@ export async function pickExtensionPack(
   return writeExtensionPack(packPath, packName, databaseItem.language);
 }
 
+function getModelConfigPackVariables(
+  databaseItem: Pick<DatabaseItem, "name" | "language" | "origin">,
+): ModelConfigPackVariables {
+  const database = databaseItem.name;
+  const language = databaseItem.language;
+  let name = databaseItem.name;
+  let owner = "";
+
+  if (databaseItem.origin?.type === "github") {
+    [owner, name] = databaseItem.origin.repository.split("/");
+  }
+
+  return {
+    database,
+    language,
+    name,
+    owner,
+  };
+}
+
 async function writeExtensionPack(
   packPath: string,
   packName: ExtensionPackName,

extensions/ql-vscode/src/model-editor/extensions-workspace-folder.ts

@@ -1,9 +1,12 @@
 import type { WorkspaceFolder } from "vscode";
 import { FileType, Uri, workspace } from "vscode";
 import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
-import { tmpdir } from "../common/files";
+import { containsPath, tmpdir } from "../common/files";
 import type { NotificationLogger } from "../common/logging";
 import { showAndLogErrorMessage } from "../common/logging";
+import { isAbsolute, normalize, resolve } from "path";
+import { nanoid } from "nanoid";
+import type { ModelConfig } from "../config";
 
 /**
  * Returns the ancestors of this path in order from furthest to closest (i.e. root of filesystem to parent directory)
@@ -22,26 +25,17 @@ function getAncestors(uri: Uri): Uri[] {
   return ancestors;
 }
 
-async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
-  // If there is a valid workspace file, just use its directory as the directory for the extensions
-  const workspaceFile = workspace.workspaceFile;
-  if (workspaceFile?.scheme === "file") {
-    return Uri.joinPath(workspaceFile, "..");
+function findCommonAncestor(uris: Uri[]): Uri | undefined {
+  if (uris.length === 0) {
+    return undefined;
   }
 
-  const allWorkspaceFolders = getOnDiskWorkspaceFoldersObjects();
-
-  // Get the system temp directory and convert it to a URI so it's normalized
-  const systemTmpdir = Uri.file(tmpdir());
-
-  const workspaceFolders = allWorkspaceFolders.filter((folder) => {
-    // Never use a workspace folder that is in the system temp directory
-    return !folder.uri.fsPath.startsWith(systemTmpdir.fsPath);
-  });
+  if (uris.length === 1) {
+    return uris[0];
+  }
 
   // Find the common root directory of all workspace folders by finding the longest common prefix
-  const commonRoot = workspaceFolders.reduce((commonRoot, folder) => {
-    const folderUri = folder.uri;
+  const commonRoot = uris.reduce((commonRoot, folderUri) => {
     const ancestors = getAncestors(folderUri);
 
     const minLength = Math.min(commonRoot.length, ancestors.length);
@@ -55,10 +49,10 @@ async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
     }
 
     return commonRoot.slice(0, commonLength);
-  }, getAncestors(workspaceFolders[0].uri));
+  }, getAncestors(uris[0]));
 
   if (commonRoot.length === 0) {
-    return await findGitFolder(workspaceFolders);
+    return undefined;
   }
 
   // The path closest to the workspace folders is the last element of the common root
@@ -67,6 +61,54 @@ async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
   // If we are at the root of the filesystem, we can't go up any further and there's something
   // wrong, so just return undefined
   if (commonRootUri.fsPath === Uri.joinPath(commonRootUri, "..").fsPath) {
+    return undefined;
+  }
+
+  return commonRootUri;
+}
+
+/**
+ * Finds the root directory of this workspace. It is determined
+ * heuristically based on the on-disk workspace folders.
+ *
+ * The heuristic is as follows:
+ * 1. If there is a workspace file (`<basename>.code-workspace`), use the directory containing that file
+ * 2. If there is only 1 workspace folder, use that folder
+ * 3. If there is a common root directory for all workspace folders, use that directory
+ *   - Workspace folders in the system temp directory are ignored
+ *   - If the common root directory is the root of the filesystem, then it's not used
+ * 4. If there is a .git directory in any workspace folder, use the directory containing that .git directory
+ *    for which the .git directory is closest to a workspace folder
+ * 5. If none of the above apply, return `undefined`
+ */
+export async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
+  // If there is a valid workspace file, just use its directory as the directory for the extensions
+  const workspaceFile = workspace.workspaceFile;
+  if (workspaceFile?.scheme === "file") {
+    return Uri.joinPath(workspaceFile, "..");
+  }
+
+  const allWorkspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  if (allWorkspaceFolders.length === 1) {
+    return allWorkspaceFolders[0].uri;
+  }
+
+  // Get the system temp directory and convert it to a URI so it's normalized
+  const systemTmpdir = Uri.file(tmpdir());
+
+  const workspaceFolders = allWorkspaceFolders.filter((folder) => {
+    // Never use a workspace folder that is in the system temp directory
+    return !folder.uri.fsPath.startsWith(systemTmpdir.fsPath);
+  });
+
+  // The path closest to the workspace folders is the last element of the common root
+  const commonRootUri = findCommonAncestor(
+    workspaceFolders.map((folder) => folder.uri),
+  );
+
+  // If there is no common root URI, try to find a .git folder in the workspace folders
+  if (commonRootUri === undefined) {
     return await findGitFolder(workspaceFolders);
   }
 
@@ -126,90 +168,130 @@ async function findGitFolder(
   return closestFolder?.[1];
 }
 
-/**
- * Finds a suitable directory for extension packs to be created in. This will
- * always be a path ending in `.github/codeql/extensions`. The parent directory
- * will be determined heuristically based on the on-disk workspace folders.
- *
- * The heuristic is as follows (`.github/codeql/extensions` is added automatically unless
- * otherwise specified):
- * 1. If there is only 1 workspace folder, use that folder
- * 2. If there is a workspace folder for which the path ends in `.github/codeql/extensions`, use that folder
- *   - If there are multiple such folders, use the first one
- *   - Does not append `.github/codeql/extensions` to the path
- * 3. If there is a workspace file (`<basename>.code-workspace`), use the directory containing that file
- * 4. If there is a common root directory for all workspace folders, use that directory
- *   - Workspace folders in the system temp directory are ignored
- *   - If the common root directory is the root of the filesystem, then it's not used
- * 5. If there is a .git directory in any workspace folder, use the directory containing that .git directory
- *    for which the .git directory is closest to a workspace folder
- * 6. If none of the above apply, return `undefined`
- */
-export async function autoPickExtensionsDirectory(
+export async function packLocationToAbsolute(
+  packLocation: string,
   logger: NotificationLogger,
-): Promise<Uri | undefined> {
-  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+): Promise<string | undefined> {
+  let userPackLocation = packLocation.trim();
 
-  // If there are no on-disk workspace folders, we can't do anything
-  if (workspaceFolders.length === 0) {
+  if (!isAbsolute(userPackLocation)) {
+    const rootDirectory = await getRootWorkspaceDirectory();
+    if (!rootDirectory) {
+      void logger.log("Unable to determine root workspace directory");
+
+      return undefined;
+    }
+
+    userPackLocation = resolve(rootDirectory.fsPath, userPackLocation);
+  }
+
+  userPackLocation = normalize(userPackLocation);
+
+  if (!isAbsolute(userPackLocation)) {
+    // This shouldn't happen, but just in case
     void showAndLogErrorMessage(
       logger,
-      `Could not find any on-disk workspace folders. Please ensure that you have opened a folder or workspace.`,
+      `Invalid pack location: ${userPackLocation}`,
     );
-    return undefined;
-  }
-
-  // If there's only 1 workspace folder, use the `.github/codeql/extensions` directory in that folder
-  if (workspaceFolders.length === 1) {
-    return Uri.joinPath(
-      workspaceFolders[0].uri,
-      ".github",
-      "codeql",
-      "extensions",
-    );
-  }
-
-  // Now try to find a workspace folder for which the path ends in `.github/codeql/extensions`
-  const workspaceFolderForExtensions = workspaceFolders.find((folder) =>
-    // Using path instead of fsPath because path always uses forward slashes
-    folder.uri.path.endsWith(".github/codeql/extensions"),
-  );
-  if (workspaceFolderForExtensions) {
-    return workspaceFolderForExtensions.uri;
-  }
-
-  // Get the root workspace directory, i.e. the common root directory of all workspace folders
-  const rootDirectory = await getRootWorkspaceDirectory();
-  if (!rootDirectory) {
-    void logger.log("Unable to determine root workspace directory");
 
     return undefined;
   }
 
-  // We'll create a new workspace folder for the extensions in the root workspace directory
-  // at `.github/codeql/extensions`
-  const extensionsUri = Uri.joinPath(
-    rootDirectory,
-    ".github",
-    "codeql",
-    "extensions",
+  // If we are at the root of the filesystem, then something is wrong since
+  // this should never be the location of a pack
+  if (userPackLocation === resolve(userPackLocation, "..")) {
+    void showAndLogErrorMessage(
+      logger,
+      `Invalid pack location: ${userPackLocation}`,
+    );
+
+    return undefined;
+  }
+
+  return userPackLocation;
+}
+
+/**
+ * This function will try to add the pack location as a workspace folder if it's not already in a
+ * workspace folder and the workspace is a multi-root workspace.
+ */
+export async function ensurePackLocationIsInWorkspaceFolder(
+  packLocation: string,
+  modelConfig: ModelConfig,
+  logger: NotificationLogger,
+): Promise<void> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  const existsInWorkspaceFolder = workspaceFolders.some((folder) =>
+    containsPath(folder.uri.fsPath, packLocation),
   );
 
+  if (existsInWorkspaceFolder) {
+    // If the pack location is already in a workspace folder, we don't need to do anything
+    return;
+  }
+
+  if (workspace.workspaceFile === undefined) {
+    // If we're not in a workspace, we can't add a workspace folder without reloading the window,
+    // so we'll not do anything
+    return;
+  }
+
+  // To find the "correct" directory to add as a workspace folder, we'll generate a few different
+  // pack locations and find the common ancestor of the directories. This is the directory that
+  // we'll add as a workspace folder.
+
+  // Generate a few different pack locations to get an accurate common ancestor
+  const otherPackLocations = await Promise.all(
+    Array.from({ length: 3 }).map(() =>
+      packLocationToAbsolute(
+        modelConfig.getPackLocation(nanoid(), {
+          database: nanoid(),
+          language: nanoid(),
+          name: nanoid(),
+          owner: nanoid(),
+        }),
+        logger,
+      ),
+    ),
+  );
+
+  const otherPackLocationUris = otherPackLocations
+    .filter((loc): loc is string => loc !== undefined)
+    .map((loc) => Uri.file(loc));
+
+  if (otherPackLocationUris.length === 0) {
+    void logger.log(
+      `Failed to generate different pack locations, not adding workspace folder.`,
+    );
+    return;
+  }
+
+  const commonRootUri = findCommonAncestor([
+    Uri.file(packLocation),
+    ...otherPackLocationUris,
+  ]);
+
+  if (commonRootUri === undefined) {
+    void logger.log(
+      `Failed to find common ancestor for ${packLocation} and ${otherPackLocationUris[0].fsPath}, not adding workspace folder.`,
+    );
+    return;
+  }
+
   if (
     !workspace.updateWorkspaceFolders(
       workspace.workspaceFolders?.length ?? 0,
       0,
       {
         name: "CodeQL Extension Packs",
-        uri: extensionsUri,
+        uri: commonRootUri,
       },
     )
   ) {
     void logger.log(
-      `Failed to add workspace folder for extensions at ${extensionsUri.fsPath}`,
+      `Failed to add workspace folder for extensions at ${commonRootUri.fsPath}`,
     );
-    return undefined;
+    return;
   }
-
-  return extensionsUri;
 }

extensions/ql-vscode/src/model-editor/generate.ts

@@ -5,19 +5,15 @@ import type { QueryRunner } from "../query-server";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import type { ModeledMethod } from "./modeled-method";
 import { runQuery } from "../local-queries/run-query";
 import type { QueryConstraints } from "../local-queries";
 import { resolveQueries } from "../local-queries";
 import type { DecodedBqrs } from "../common/bqrs-cli-types";
+
 type GenerateQueriesOptions = {
   queryConstraints: QueryConstraints;
   filterQueries?: (queryPath: string) => boolean;
-  parseResults: (
-    queryPath: string,
-    results: DecodedBqrs,
-  ) => ModeledMethod[] | Promise<ModeledMethod[]>;
-  onResults: (results: ModeledMethod[]) => void | Promise<void>;
+  onResults: (queryPath: string, results: DecodedBqrs) => void | Promise<void>;
 
   cliServer: CodeQLCliServer;
   queryRunner: QueryRunner;
@@ -28,7 +24,7 @@ type GenerateQueriesOptions = {
 };
 
 export async function runGenerateQueries(options: GenerateQueriesOptions) {
-  const { queryConstraints, filterQueries, parseResults, onResults } = options;
+  const { queryConstraints, filterQueries, onResults } = options;
 
   options.progress({
     message: "Resolving queries",
@@ -55,7 +51,7 @@ export async function runGenerateQueries(options: GenerateQueriesOptions) {
 
     const bqrs = await runSingleGenerateQuery(queryPath, i, maxStep, options);
     if (bqrs) {
-      await onResults(await parseResults(queryPath, bqrs));
+      await onResults(queryPath, bqrs);
     }
   }
 }

extensions/ql-vscode/src/model-editor/languages/csharp/index.ts

@@ -0,0 +1,21 @@
+import type { ModelsAsDataLanguage } from "../models-as-data";
+import { staticLanguage } from "../static";
+
+export const csharp: ModelsAsDataLanguage = {
+  ...staticLanguage,
+  predicates: {
+    ...staticLanguage.predicates,
+    sink: {
+      ...staticLanguage.predicates.sink,
+    },
+    source: {
+      ...staticLanguage.predicates.source,
+      supportedKinds: [
+        ...staticLanguage.predicates.source.supportedKinds,
+        // https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L122-L123
+        "file-write",
+        "windows-registry",
+      ],
+    },
+  },
+};

extensions/ql-vscode/src/model-editor/languages/java/index.ts

@@ -0,0 +1,42 @@
+import type { ModelsAsDataLanguage } from "../models-as-data";
+import { staticLanguage } from "../static";
+
+export const java: ModelsAsDataLanguage = {
+  ...staticLanguage,
+  predicates: {
+    ...staticLanguage.predicates,
+    sink: {
+      ...staticLanguage.predicates.sink,
+      supportedKinds: [
+        ...staticLanguage.predicates.sink.supportedKinds,
+        // https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L32-L37
+        "bean-validation",
+        "fragment-injection",
+        "groovy-injection",
+        "hostname-verification",
+        "information-leak",
+        "intent-redirection",
+        "jexl-injection",
+        "jndi-injection",
+        "mvel-injection",
+        "notification",
+        "ognl-injection",
+        "pending-intents",
+        "response-splitting",
+        "trust-boundary-violation",
+        "template-injection",
+        "xpath-injection",
+        "xslt-injection",
+      ],
+    },
+    source: {
+      ...staticLanguage.predicates.source,
+      supportedKinds: [
+        ...staticLanguage.predicates.source.supportedKinds,
+        // https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L120-L121
+        "android-external-storage-dir",
+        "contentprovider",
+      ],
+    },
+  },
+};

extensions/ql-vscode/src/model-editor/languages/languages.ts

@@ -3,12 +3,15 @@ import type {
   ModelsAsDataLanguage,
   ModelsAsDataLanguagePredicates,
 } from "./models-as-data";
+import { csharp } from "./csharp";
+import { java } from "./java";
+import { python } from "./python";
 import { ruby } from "./ruby";
-import { staticLanguage } from "./static";
 
 const languages: Partial<Record<QueryLanguage, ModelsAsDataLanguage>> = {
-  [QueryLanguage.CSharp]: staticLanguage,
-  [QueryLanguage.Java]: staticLanguage,
+  [QueryLanguage.CSharp]: csharp,
+  [QueryLanguage.Java]: java,
+  [QueryLanguage.Python]: python,
   [QueryLanguage.Ruby]: ruby,
 };
 

extensions/ql-vscode/src/model-editor/languages/models-as-data.ts

@@ -7,7 +7,7 @@ import type {
   SummaryModeledMethod,
   TypeModeledMethod,
 } from "../modeled-method";
-import type { DataTuple } from "../model-extension-file";
+import type { DataTuple, ModelExtension } from "../model-extension-file";
 import type { Mode } from "../shared/mode";
 import type { QueryConstraints } from "../../local-queries/query-constraints";
 import type {
@@ -17,9 +17,37 @@ import type {
 import type { BaseLogger } from "../../common/logging";
 import type { AccessPathSuggestionRow } from "../suggestions";
 
+// This is a subset of the model config that doesn't import the vscode module.
+// It only includes settings that are actually used.
+export type ModelConfig = {
+  flowGeneration: boolean;
+};
+
+/**
+ * This function creates a new model config object from the given model config object.
+ * The new model config object is a deep copy of the given model config object.
+ *
+ * @param modelConfig The model config object to create a new model config object from.
+ *                    In most cases, this is a `ModelConfigListener`.
+ */
+export function createModelConfig(modelConfig: ModelConfig): ModelConfig {
+  return {
+    flowGeneration: modelConfig.flowGeneration,
+  };
+}
+
+export const defaultModelConfig: ModelConfig = {
+  flowGeneration: false,
+};
+
 type GenerateMethodDefinition<T> = (method: T) => DataTuple[];
 type ReadModeledMethod = (row: DataTuple[]) => ModeledMethod;
 
+type IsHiddenContext = {
+  method: MethodDefinition;
+  config: ModelConfig;
+};
+
 export type ModelsAsDataLanguagePredicate<T> = {
   extensiblePredicate: string;
   supportedKinds?: string[];
@@ -30,22 +58,81 @@ export type ModelsAsDataLanguagePredicate<T> = {
   supportedEndpointTypes?: EndpointType[];
   generateMethodDefinition: GenerateMethodDefinition<T>;
   readModeledMethod: ReadModeledMethod;
+
+  /**
+   * Controls whether this predicate is hidden for a certain method. This only applies to the UI.
+   * If not specified, the predicate is visible for all methods.
+   *
+   * @param method The method to check if the predicate is hidden for.
+   */
+  isHidden?: (context: IsHiddenContext) => boolean;
 };
 
+export type GenerationContext = {
+  mode: Mode;
+  config: ModelConfig;
+};
+
+type ParseGenerationResults = (
+  // The path to the query that generated the results.
+  queryPath: string,
+  // The results of the query.
+  bqrs: DecodedBqrs,
+  // The language-specific predicate that was used to generate the results. This is passed to allow
+  // sharing of code between different languages.
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  // The logger to use for logging.
+  logger: BaseLogger,
+  // Context about this invocation of the generation.
+  context: GenerationContext,
+) => ModeledMethod[];
+
 type ModelsAsDataLanguageModelGeneration = {
-  queryConstraints: QueryConstraints;
+  queryConstraints: (mode: Mode) => QueryConstraints;
   filterQueries?: (queryPath: string) => boolean;
-  parseResults: (
-    // The path to the query that generated the results.
-    queryPath: string,
-    // The results of the query.
-    bqrs: DecodedBqrs,
-    // The language-specific predicate that was used to generate the results. This is passed to allow
-    // sharing of code between different languages.
-    modelsAsDataLanguage: ModelsAsDataLanguage,
-    // The logger to use for logging.
-    logger: BaseLogger,
-  ) => ModeledMethod[];
+  parseResults: ParseGenerationResults;
+};
+
+type ParseResultsToYaml = (
+  // The path to the query that generated the results.
+  queryPath: string,
+  // The results of the query.
+  bqrs: DecodedBqrs,
+  // The language-specific predicate that was used to generate the results. This is passed to allow
+  // sharing of code between different languages.
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  // The logger to use for logging.
+  logger: BaseLogger,
+) => ModelExtension[];
+
+export enum AutoModelGenerationType {
+  /**
+   * Auto model generation is disabled and will not be run.
+   */
+  Disabled = "disabled",
+  /**
+   * The models are generated to a separate file (suffixed with .model.generated.yml).
+   */
+  SeparateFile = "separateFile",
+  /**
+   * The models are added as a model in the model editor, but are not automatically saved.
+   * The user can view them and choose to save them.
+   */
+  Models = "models",
+}
+
+type ModelsAsDataLanguageAutoModelGeneration = {
+  queryConstraints: (mode: Mode) => QueryConstraints;
+  filterQueries?: (queryPath: string) => boolean;
+  /**
+   * This function is only used when type is `separateFile`.
+   */
+  parseResultsToYaml: ParseResultsToYaml;
+  /**
+   * This function is only used when type is `models`.
+   */
+  parseResults: ParseGenerationResults;
+  type: (context: GenerationContext) => AutoModelGenerationType;
 };
 
 type ModelsAsDataLanguageAccessPathSuggestions = {
@@ -95,6 +182,7 @@ export type ModelsAsDataLanguage = {
   ) => EndpointType | undefined;
   predicates: ModelsAsDataLanguagePredicates;
   modelGeneration?: ModelsAsDataLanguageModelGeneration;
+  autoModelGeneration?: ModelsAsDataLanguageAutoModelGeneration;
   accessPathSuggestions?: ModelsAsDataLanguageAccessPathSuggestions;
   /**
    * Returns the list of valid arguments that can be selected for the given method.

extensions/ql-vscode/src/model-editor/languages/python/access-paths.ts

@@ -0,0 +1,122 @@
+import { parseAccessPathTokens } from "../../shared/access-paths";
+import type { MethodDefinition } from "../../method";
+import { EndpointType } from "../../method";
+
+const memberTokenRegex = /^Member\[(.+)]$/;
+
+export function parsePythonAccessPath(path: string): {
+  typeName: string;
+  methodName: string;
+  endpointType: EndpointType;
+  path: string;
+} {
+  const tokens = parseAccessPathTokens(path);
+
+  if (tokens.length === 0) {
+    return {
+      typeName: "",
+      methodName: "",
+      endpointType: EndpointType.Method,
+      path: "",
+    };
+  }
+
+  const typeParts = [];
+  let endpointType = EndpointType.Function;
+
+  let remainingTokens: typeof tokens = [];
+
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    const memberMatch = token.text.match(memberTokenRegex);
+    if (memberMatch) {
+      typeParts.push(memberMatch[1]);
+    } else if (token.text === "Instance") {
+      endpointType = EndpointType.Method;
+    } else {
+      remainingTokens = tokens.slice(i);
+      break;
+    }
+  }
+
+  const methodName = typeParts.pop() ?? "";
+  const typeName = typeParts.join(".");
+  const remainingPath = remainingTokens.map((token) => token.text).join(".");
+
+  return {
+    methodName,
+    typeName,
+    endpointType,
+    path: remainingPath,
+  };
+}
+
+export function pythonMethodSignature(typeName: string, methodName: string) {
+  return `${typeName}#${methodName}`;
+}
+
+function pythonTypePath(typeName: string) {
+  if (typeName === "") {
+    return "";
+  }
+
+  return typeName
+    .split(".")
+    .map((part) => `Member[${part}]`)
+    .join(".");
+}
+
+export function pythonMethodPath(
+  typeName: string,
+  methodName: string,
+  endpointType: EndpointType,
+) {
+  if (methodName === "") {
+    return pythonTypePath(typeName);
+  }
+
+  const typePath = pythonTypePath(typeName);
+
+  let result = typePath;
+  if (typePath !== "" && endpointType === EndpointType.Method) {
+    result += ".Instance";
+  }
+
+  if (result !== "") {
+    result += ".";
+  }
+
+  result += `Member[${methodName}]`;
+
+  return result;
+}
+
+export function pythonPath(
+  typeName: string,
+  methodName: string,
+  endpointType: EndpointType,
+  path: string,
+) {
+  const methodPath = pythonMethodPath(typeName, methodName, endpointType);
+  if (methodPath === "") {
+    return path;
+  }
+
+  if (path === "") {
+    return methodPath;
+  }
+
+  return `${methodPath}.${path}`;
+}
+
+export function pythonEndpointType(
+  method: Omit<MethodDefinition, "endpointType">,
+): EndpointType {
+  if (
+    method.methodParameters.startsWith("(self,") ||
+    method.methodParameters === "(self)"
+  ) {
+    return EndpointType.Method;
+  }
+  return EndpointType.Function;
+}

extensions/ql-vscode/src/model-editor/languages/python/index.ts

@@ -0,0 +1,228 @@
+import type { ModelsAsDataLanguage } from "../models-as-data";
+import { sharedExtensiblePredicates, sharedKinds } from "../shared";
+import { Mode } from "../../shared/mode";
+import type { MethodArgument } from "../../method";
+import { EndpointType, getArgumentsList } from "../../method";
+import {
+  parsePythonAccessPath,
+  pythonEndpointType,
+  pythonMethodPath,
+  pythonMethodSignature,
+  pythonPath,
+} from "./access-paths";
+
+export const python: ModelsAsDataLanguage = {
+  availableModes: [Mode.Framework],
+  createMethodSignature: ({ typeName, methodName }) =>
+    `${typeName}#${methodName}`,
+  endpointTypeForEndpoint: (method) => pythonEndpointType(method),
+  predicates: {
+    source: {
+      extensiblePredicate: sharedExtensiblePredicates.source,
+      supportedKinds: sharedKinds.source,
+      supportedEndpointTypes: [EndpointType.Method, EndpointType.Function],
+      // extensible predicate sourceModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        pythonPath(
+          method.typeName,
+          method.methodName,
+          method.endpointType,
+          method.output,
+        ),
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const {
+          typeName,
+          methodName,
+          endpointType,
+          path: output,
+        } = parsePythonAccessPath(row[1] as string);
+        return {
+          type: "source",
+          output,
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    sink: {
+      extensiblePredicate: sharedExtensiblePredicates.sink,
+      supportedKinds: sharedKinds.sink,
+      supportedEndpointTypes: [EndpointType.Method, EndpointType.Function],
+      // extensible predicate sinkModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => {
+        return [
+          method.packageName,
+          pythonPath(
+            method.typeName,
+            method.methodName,
+            method.endpointType,
+            method.input,
+          ),
+          method.kind,
+        ];
+      },
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const {
+          typeName,
+          methodName,
+          endpointType,
+          path: input,
+        } = parsePythonAccessPath(row[1] as string);
+        return {
+          type: "sink",
+          input,
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    summary: {
+      extensiblePredicate: sharedExtensiblePredicates.summary,
+      supportedKinds: sharedKinds.summary,
+      supportedEndpointTypes: [EndpointType.Method, EndpointType.Function],
+      // extensible predicate summaryModel(
+      //   string type, string path, string input, string output, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        pythonMethodPath(
+          method.typeName,
+          method.methodName,
+          method.endpointType,
+        ),
+        method.input,
+        method.output,
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const { typeName, methodName, endpointType, path } =
+          parsePythonAccessPath(row[1] as string);
+        if (path !== "") {
+          throw new Error("Summary path must be a method");
+        }
+        return {
+          type: "summary",
+          input: row[2] as string,
+          output: row[3] as string,
+          kind: row[4] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    neutral: {
+      extensiblePredicate: sharedExtensiblePredicates.neutral,
+      supportedKinds: sharedKinds.neutral,
+      // extensible predicate neutralModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        pythonMethodPath(
+          method.typeName,
+          method.methodName,
+          method.endpointType,
+        ),
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const { typeName, methodName, endpointType, path } =
+          parsePythonAccessPath(row[1] as string);
+        if (path !== "") {
+          throw new Error("Neutral path must be a method");
+        }
+        return {
+          type: "neutral",
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+  },
+  getArgumentOptions: (method) => {
+    // Argument and Parameter are equivalent in Python, but we'll use Argument in the model editor
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => {
+        if (
+          method.endpointType === EndpointType.Method &&
+          argument === "self" &&
+          index === 0
+        ) {
+          return {
+            path: "Argument[self]",
+            label: "Argument[self]: self",
+          };
+        }
+
+        // If this is a method, self does not count as an argument index, so we
+        // should start at 0 for the second argument
+        if (method.endpointType === EndpointType.Method) {
+          index -= 1;
+        }
+
+        // Keyword-only arguments end with `:` in the query
+        if (argument.endsWith(":")) {
+          return {
+            path: `Argument[${argument}]`,
+            label: `Argument[${argument}]: ${argument.substring(0, argument.length - 1)}`,
+          };
+        }
+
+        // Positional-only arguments end with `/` in the query
+        if (argument.endsWith("/")) {
+          return {
+            path: `Argument[${index}]`,
+            label: `Argument[${index}]: ${argument.substring(0, argument.length - 1)}`,
+          };
+        }
+
+        // All other arguments are both keyword and positional
+        return {
+          path: `Argument[${index},${argument}:]`,
+          label: `Argument[${index},${argument}:]: ${argument}`,
+        };
+      },
+    );
+
+    return {
+      options: argumentsList,
+      // If there are no arguments, we will default to "Argument[self]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[self]",
+    };
+  },
+};

extensions/ql-vscode/src/model-editor/languages/ruby/index.ts

@@ -1,4 +1,5 @@
 import type { ModelsAsDataLanguage } from "../models-as-data";
+import { AutoModelGenerationType } from "../models-as-data";
 import { sharedExtensiblePredicates, sharedKinds } from "../shared";
 import { Mode } from "../../shared/mode";
 import { parseGenerateModelResults } from "./generate";
@@ -172,11 +173,69 @@ export const ruby: ModelsAsDataLanguage = {
     },
   },
   modelGeneration: {
-    queryConstraints: {
-      "query path": "queries/modeling/GenerateModel.ql",
-    },
+    queryConstraints: (mode) => ({
+      kind: "table",
+      "tags contain all": ["modeleditor", "generate-model", modeTag(mode)],
+    }),
     parseResults: parseGenerateModelResults,
   },
+  autoModelGeneration: {
+    queryConstraints: (mode) => ({
+      kind: "table",
+      "tags contain all": ["modeleditor", "generate-model", modeTag(mode)],
+    }),
+    parseResultsToYaml: (_queryPath, bqrs, modelsAsDataLanguage) => {
+      const typePredicate = modelsAsDataLanguage.predicates.type;
+      if (!typePredicate) {
+        throw new Error("Type predicate not found");
+      }
+
+      const typeTuples = bqrs[typePredicate.extensiblePredicate];
+      if (!typeTuples) {
+        return [];
+      }
+
+      return [
+        {
+          addsTo: {
+            pack: "codeql/ruby-all",
+            extensible: typePredicate.extensiblePredicate,
+          },
+          data: typeTuples.tuples.filter((tuple): tuple is string[] => {
+            return (
+              tuple.filter((x) => typeof x === "string").length === tuple.length
+            );
+          }),
+        },
+      ];
+    },
+    parseResults: (queryPath, bqrs, modelsAsDataLanguage, logger) => {
+      // Only parse type models when automatically generating models
+      const typePredicate = modelsAsDataLanguage.predicates.type;
+      if (!typePredicate) {
+        throw new Error("Type predicate not found");
+      }
+
+      const typeTuples = bqrs[typePredicate.extensiblePredicate];
+      if (!typeTuples) {
+        return [];
+      }
+
+      return parseGenerateModelResults(
+        queryPath,
+        {
+          [typePredicate.extensiblePredicate]: typeTuples,
+        },
+        modelsAsDataLanguage,
+        logger,
+      );
+    },
+    // Only enabled for framework mode when type models are hidden
+    type: ({ mode }) =>
+      mode === Mode.Framework
+        ? AutoModelGenerationType.Models
+        : AutoModelGenerationType.Disabled,
+  },
   accessPathSuggestions: {
     queryConstraints: (mode) => ({
       kind: "table",

extensions/ql-vscode/src/model-editor/languages/shared.ts

@@ -6,10 +6,13 @@ export const sharedExtensiblePredicates = {
 };
 
 export const sharedKinds = {
-  source: ["local", "remote"],
+  // https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L118-L119
+  source: ["local", "remote", "file", "commandargs", "database", "environment"],
+  // Bhttps://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L28-L31
   sink: [
     "code-injection",
     "command-injection",
+    "environment-injection",
     "file-content-store",
     "html-injection",
     "js-injection",
@@ -20,6 +23,8 @@ export const sharedKinds = {
     "sql-injection",
     "url-redirection",
   ],
+  // https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L142-L143
   summary: ["taint", "value"],
+  // https://github.com/github/codeql/blob/0c5ea975a4c4dc5c439b908c006e440cb9bdf926/shared/mad/codeql/mad/ModelValidation.qll#L155-L156
   neutral: ["summary", "source", "sink"],
 };

extensions/ql-vscode/src/model-editor/languages/static/index.ts

@@ -10,7 +10,7 @@ function readRowToMethod(row: DataTuple[]): string {
   return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
 }
 
-export const staticLanguage: ModelsAsDataLanguage = {
+export const staticLanguage = {
   createMethodSignature: ({
     packageName,
     typeName,
@@ -141,9 +141,9 @@ export const staticLanguage: ModelsAsDataLanguage = {
     },
   },
   modelGeneration: {
-    queryConstraints: {
+    queryConstraints: () => ({
       "tags contain": ["modelgenerator"],
-    },
+    }),
     filterQueries: filterFlowModelQueries,
     parseResults: parseFlowModelResults,
   },
@@ -168,4 +168,4 @@ export const staticLanguage: ModelsAsDataLanguage = {
         argumentsList.length > 0 ? argumentsList[0].path : "Argument[this]",
     };
   },
-};
+} satisfies ModelsAsDataLanguage;

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts

@@ -2,10 +2,8 @@ import { window } from "vscode";
 import type { App } from "../../common/app";
 import { DisposableObject } from "../../common/disposable-object";
 import { MethodModelingViewProvider } from "./method-modeling-view-provider";
-import type { Method } from "../method";
 import type { ModelingStore } from "../modeling-store";
 import { ModelConfigListener } from "../../config";
-import type { DatabaseItem } from "../../databases/local-databases";
 import type { ModelingEvents } from "../modeling-events";
 
 export class MethodModelingPanel extends DisposableObject {
@@ -36,11 +34,4 @@ export class MethodModelingPanel extends DisposableObject {
       ),
     );
   }
-
-  public async setMethod(
-    databaseItem: DatabaseItem,
-    method: Method,
-  ): Promise<void> {
-    await this.provider.setMethod(databaseItem, method);
-  }
 }

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts

@@ -6,7 +6,7 @@ import { telemetryListener } from "../../common/vscode/telemetry";
 import { showAndLogExceptionWithTelemetry } from "../../common/logging/notifications";
 import type { App } from "../../common/app";
 import { redactableError } from "../../common/errors";
-import type { Method } from "../method";
+import type { Method, MethodSignature } from "../method";
 import type { ModelingStore } from "../modeling-store";
 import { AbstractWebviewViewProvider } from "../../common/vscode/abstract-webview-view-provider";
 import { assertNever } from "../../common/helpers-pure";
@@ -15,6 +15,8 @@ import type { DatabaseItem } from "../../databases/local-databases";
 import type { ModelingEvents } from "../modeling-events";
 import type { QueryLanguage } from "../../common/query-language";
 import { tryGetQueryLanguage } from "../../common/query-language";
+import { createModelConfig } from "../languages";
+import type { ModeledMethod } from "../modeled-method";
 
 export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   ToMethodModelingMessage,
@@ -36,7 +38,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   }
 
   protected override async onWebViewLoaded(): Promise<void> {
-    await Promise.all([this.setViewState(), this.setInitialState()]);
+    await this.setInitialState();
     this.registerToModelingEvents();
     this.registerToModelConfigEvents();
   }
@@ -46,49 +48,65 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
       t: "setMethodModelingPanelViewState",
       viewState: {
         language: this.language,
+        modelConfig: createModelConfig(this.modelConfig),
       },
     });
   }
 
-  public async setMethod(
-    databaseItem: DatabaseItem | undefined,
-    method: Method | undefined,
+  private async setDatabaseItem(databaseItem: DatabaseItem): Promise<void> {
+    this.databaseItem = databaseItem;
+
+    await this.postMessage({
+      t: "setInModelingMode",
+      inModelingMode: true,
+    });
+
+    this.language = tryGetQueryLanguage(databaseItem.language);
+    await this.setViewState();
+  }
+
+  private async setSelectedMethod(
+    databaseItem: DatabaseItem,
+    method: Method,
+    modeledMethods: readonly ModeledMethod[],
+    isModified: boolean,
+    isInProgress: boolean,
+    processedByAutoModel: boolean,
   ): Promise<void> {
     this.method = method;
     this.databaseItem = databaseItem;
-    this.language = databaseItem && tryGetQueryLanguage(databaseItem.language);
+    this.language = tryGetQueryLanguage(databaseItem.language);
 
-    if (this.isShowingView) {
-      await this.postMessage({
-        t: "setMethod",
-        method,
-      });
-    }
+    await this.postMessage({
+      t: "setSelectedMethod",
+      method,
+      modeledMethods,
+      isModified,
+      isInProgress,
+      processedByAutoModel,
+    });
   }
 
   private async setInitialState(): Promise<void> {
-    if (this.modelingStore.hasStateForActiveDb()) {
-      const selectedMethod = this.modelingStore.getSelectedMethodDetails();
-      if (selectedMethod) {
-        this.databaseItem = selectedMethod.databaseItem;
-        this.language = tryGetQueryLanguage(
-          selectedMethod.databaseItem.language,
-        );
-        this.method = selectedMethod.method;
+    await this.setViewState();
 
-        await this.postMessage({
-          t: "setSelectedMethod",
-          method: selectedMethod.method,
-          modeledMethods: selectedMethod.modeledMethods,
-          isModified: selectedMethod.isModified,
-          isInProgress: selectedMethod.isInProgress,
-        });
-      }
+    const stateForActiveDb = this.modelingStore.getStateForActiveDb();
+    if (!stateForActiveDb) {
+      return;
+    }
 
-      await this.postMessage({
-        t: "setInModelingMode",
-        inModelingMode: true,
-      });
+    await this.setDatabaseItem(stateForActiveDb.databaseItem);
+
+    const selectedMethod = this.modelingStore.getSelectedMethodDetails();
+    if (selectedMethod) {
+      await this.setSelectedMethod(
+        stateForActiveDb.databaseItem,
+        selectedMethod.method,
+        selectedMethod.modeledMethods,
+        selectedMethod.isModified,
+        selectedMethod.isInProgress,
+        selectedMethod.processedByAutoModel,
+      );
     }
   }
 
@@ -123,10 +141,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
           this.databaseItem,
           msg.methodSignature,
           msg.modeledMethods,
-        );
-        this.modelingStore.addModifiedMethod(
-          this.databaseItem,
-          msg.methodSignature,
+          true,
         );
         break;
       }
@@ -148,7 +163,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     }
   }
 
-  private async revealInModelEditor(method: Method): Promise<void> {
+  private async revealInModelEditor(method: MethodSignature): Promise<void> {
     if (!this.databaseItem) {
       return;
     }
@@ -161,7 +176,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
   private registerToModelingEvents(): void {
     this.push(
-      this.modelingEvents.onModeledMethodsChanged(async (e) => {
+      this.modelingEvents.onModeledAndModifiedMethodsChanged(async (e) => {
         if (this.webviewView && e.isActiveDb && this.method) {
           const modeledMethods = e.modeledMethods[this.method.signature];
           if (modeledMethods) {
@@ -171,17 +186,10 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
               modeledMethods,
             });
           }
-        }
-      }),
-    );
 
-    this.push(
-      this.modelingEvents.onModifiedMethodsChanged(async (e) => {
-        if (this.webviewView && e.isActiveDb && this.method) {
-          const isModified = e.modifiedMethods.has(this.method.signature);
           await this.postMessage({
             t: "setMethodModified",
-            isModified,
+            isModified: e.modifiedMethodSignatures.has(this.method.signature),
           });
         }
       }),
@@ -190,32 +198,21 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     this.push(
       this.modelingEvents.onSelectedMethodChanged(async (e) => {
         if (this.webviewView) {
-          this.method = e.method;
-          this.databaseItem = e.databaseItem;
-          this.language = tryGetQueryLanguage(e.databaseItem.language);
-
-          await this.postMessage({
-            t: "setSelectedMethod",
-            method: e.method,
-            modeledMethods: e.modeledMethods,
-            isModified: e.isModified,
-            isInProgress: e.isInProgress,
-          });
+          await this.setSelectedMethod(
+            e.databaseItem,
+            e.method,
+            e.modeledMethods,
+            e.isModified,
+            e.isInProgress,
+            e.processedByAutoModel,
+          );
         }
       }),
     );
 
     this.push(
       this.modelingEvents.onDbOpened(async (databaseItem) => {
-        this.databaseItem = databaseItem;
-
-        await this.postMessage({
-          t: "setInModelingMode",
-          inModelingMode: true,
-        });
-
-        this.language = tryGetQueryLanguage(databaseItem.language);
-        await this.setViewState();
+        await this.setDatabaseItem(databaseItem);
       }),
     );
 
@@ -229,7 +226,9 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         }
 
         if (dbUri === this.databaseItem?.databaseUri.toString()) {
-          await this.setMethod(undefined, undefined);
+          await this.postMessage({
+            t: "setNoMethodSelected",
+          });
         }
       }),
     );
@@ -248,6 +247,21 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         }
       }),
     );
+
+    this.push(
+      this.modelingEvents.onProcessedByAutoModelMethodsChanged(async (e) => {
+        if (this.method && this.databaseItem) {
+          const dbUri = this.databaseItem.databaseUri.toString();
+          if (e.dbUri === dbUri) {
+            const processedByAutoModel = e.methods.has(this.method.signature);
+            await this.postMessage({
+              t: "setProcessedByAutoModel",
+              processedByAutoModel,
+            });
+          }
+        }
+      }),
+    );
   }
 
   private registerToModelConfigEvents(): void {

extensions/ql-vscode/src/model-editor/method.ts

@@ -28,6 +28,7 @@ export enum EndpointType {
   Class = "class",
   Method = "method",
   Constructor = "constructor",
+  Function = "function",
 }
 
 export interface MethodDefinition {

extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-data-provider.ts

@@ -16,7 +16,7 @@ import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "../shared/hide-modeled-metho
 import { getModelingStatus } from "../shared/modeling-status";
 import { assertNever } from "../../common/helpers-pure";
 import type { ModeledMethod } from "../modeled-method";
-import { groupMethods, sortGroupNames, sortMethods } from "../shared/sorting";
+import { groupMethods, sortGroupNames } from "../shared/sorting";
 import type { Mode } from "../shared/mode";
 import { INITIAL_MODE } from "../shared/mode";
 import type { UrlValueResolvable } from "../../common/raw-result-types";
@@ -73,9 +73,7 @@ export class MethodsUsageDataProvider
       this.modifiedMethodSignatures !== modifiedMethodSignatures
     ) {
       this.methods = methods;
-      this.sortedTreeItems = createTreeItems(
-        sortMethodsInGroups(methods, mode),
-      );
+      this.sortedTreeItems = createTreeItems(createGroups(methods, mode));
       this.databaseItem = databaseItem;
       this.sourceLocationPrefix =
         await this.databaseItem.getSourceLocationPrefix(this.cliServer);
@@ -246,16 +244,9 @@ function urlValueResolvablesAreEqual(
   return false;
 }
 
-function sortMethodsInGroups(methods: readonly Method[], mode: Mode): Method[] {
+function createGroups(methods: readonly Method[], mode: Mode): Method[] {
   const grouped = groupMethods(methods, mode);
-
-  const sortedGroupNames = sortGroupNames(grouped);
-
-  return sortedGroupNames.flatMap((groupName) => {
-    const group = grouped[groupName];
-
-    return sortMethods(group);
-  });
+  return sortGroupNames(grouped).flatMap((groupName) => grouped[groupName]);
 }
 
 function createTreeItems(methods: readonly Method[]): MethodTreeViewItem[] {