v1.12.3

Show test results for tests with warnings

docs/test-plan.md

@@ -57,14 +57,16 @@ choose to go through some of the Optional Test Cases.
 2. Select the `google/brotli` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
+   - Chose the `#select` result set from the drop-down
    - Check that the results table is rendered
    - Check that result locations can be clicked on
 
 #### Test case 4: Can use AST viewer
 
 1. Click on any code location from a previous query to open a source file from a database
-2. Open the AST viewing panel and click "View AST"
-3. Once the AST is computed:
+2. Select the highlighted code in the source file
+3. Open the AST viewing panel and click "View AST"
+4. Once the AST is computed:
    - Check that it can be navigated
 
 ### MRVA
@@ -143,6 +145,48 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
+#### Test Case 6: Using model packs in MRVA
+
+1. Create a model pack with mock data
+   1. Create a new directory `test-model-pack`
+   2. Create a `qlpack.yml` file in that directory with the following contents:
+
+      ```yaml
+      name: github/test-model-pack
+      version: 0.0.0
+      library: true
+      extensionTargets:
+        codeql/python-all: '*'
+      dataExtensions:
+        - extension.yml
+      ```
+
+   3. Create an `extension.yml` in the same directory with the following contents:
+
+      ```yaml
+      extensions:
+      - addsTo:
+          pack: codeql/python-all
+          extensible: sinkModel
+        data:
+         - ["vscode-codeql","Member[initialize].Argument[0]","code-injection"]
+      ```
+
+2. In a Python query pack, create the following query (e.g. `sinks.ql`):
+
+   ```ql
+   import python
+   import semmle.python.frameworks.data.internal.ApiGraphModelsExtensions
+
+   from string path, string kind
+   where sinkModel("vscode-codeql", path, kind)
+   select path, kind
+   ```
+
+3. Run a MRVA against a Python repository (e.g. `psf/requests`) with this query.
+4. Check that the results view contains 1 result with the values corresponding to the `extension.yml` file:
+   ![Model packs results table for `psf/requests`](images/model-pack-results-table.png)
+
 ### CodeQL Model Editor
 
 #### Test Case 1: Opening the model editor
@@ -151,7 +195,7 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
    - Check that the editor loads and shows methods to model.
    - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
-   - Check that the "Open source" link works.
+   - Check that the "Open source" link works (if you have the database source).
    - Check that the 'View' button works and the Method Usage panel highlight the correct method and usage
    - Check that the Method Modeling panel shows the correct method and modeling state
 

extensions/ql-vscode/.eslintrc.js

@@ -40,7 +40,7 @@ const baseConfig = {
         ignoreRestSiblings: false,
       },
     ],
-    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-explicit-any": "error",
     "@typescript-eslint/no-floating-promises": ["error", { ignoreVoid: true }],
     "@typescript-eslint/no-invalid-this": "off",
     "@typescript-eslint/no-shadow": "off",
@@ -121,15 +121,6 @@ module.exports = {
         },
       },
     },
-    {
-      files: ["test/**/*"],
-      parserOptions: {
-        project: resolve(__dirname, "test/tsconfig.json"),
-      },
-      env: {
-        jest: true,
-      },
-    },
     {
       files: ["test/vscode-tests/**/*"],
       parserOptions: {
@@ -156,6 +147,18 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ["test/**/*"],
+      parserOptions: {
+        project: resolve(__dirname, "test/tsconfig.json"),
+      },
+      env: {
+        jest: true,
+      },
+      rules: {
+        "@typescript-eslint/no-explicit-any": "off",
+      },
+    },
     {
       files: [
         ".eslintrc.js",

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.12.3 - 29 February 2024
+
+- Update variant analysis view to show when cancelation is in progress. [#3405](https://github.com/github/vscode-codeql/pull/3405)
+- Remove support for CodeQL CLI versions older than 2.13.5. [#3371](https://github.com/github/vscode-codeql/pull/3371)
+- Add a timeout to downloading databases and the CodeQL CLI. These can be changed using the `codeQL.addingDatabases.downloadTimeout` and `codeQL.cli.downloadTimeout` settings respectively. [#3373](https://github.com/github/vscode-codeql/pull/3373)
+
 ## 1.12.2 - 14 February 2024
 
 - Stop allowing running variant analyses with a query outside of the workspace. [#3302](https://github.com/github/vscode-codeql/pull/3302)

extensions/ql-vscode/gulpfile.ts/textmate-grammar.ts

@@ -0,0 +1,91 @@
+/**
+ * A subset of the standard TextMate grammar that is used by our transformation
+ * step. For a full JSON schema, see:
+ * https://github.com/martinring/tmlanguage/blob/478ad124a21933cd4b0b65f1ee7ee18ee1f87473/tmlanguage.json
+ */
+export interface TextmateGrammar {
+  patterns: Pattern[];
+  repository?: Record<string, Pattern>;
+}
+
+/**
+ * The extended TextMate grammar as used by our transformation step. This is a superset of the
+ * standard TextMate grammar, and includes additional fields that are used by our transformation
+ * step.
+ *
+ * Any comment of the form `(?#ref-id)` in a `match`, `begin`, or `end` property will be replaced
+ * with the match text of the rule named "ref-id". If the rule named "ref-id" consists of just a
+ * `patterns` property with a list of `include` directives, the replacement pattern is the
+ * disjunction of the match patterns of all of the included rules.
+ */
+export interface ExtendedTextmateGrammar<MatchType = string> {
+  /**
+   * This represents the set of regular expression options to apply to all regular
+   * expressions throughout the file.
+   */
+  regexOptions?: string;
+  /**
+   * This element defines a map of macro names to replacement text. When a `match`, `begin`, or
+   * `end` property has a value that is a single-key map, the value is replaced with the value of the
+   * macro named by the key, with any use of `(?#)` in the macro text replaced with the text of the
+   * value of the key, surrounded by a non-capturing group (`(?:)`). For example:
+   *
+   * The `beginPattern` and `endPattern` Properties
+   * A rule can have a `beginPattern` or `endPattern` property whose value is a reference to another
+   * rule (e.g. `#other-rule`). The `beginPattern` property is replaced as follows:
+   *
+   * my-rule:
+   *   beginPattern: '#other-rule'
+   *
+   * would be transformed to
+   *
+   * my-rule:
+   *   begin: '(?#other-rule)'
+   *   beginCaptures:
+   *     '0':
+   *       patterns:
+   *       - include: '#other-rule'
+   *
+   * An `endPattern` property is transformed similary.
+   *
+   * macros:
+   *   repeat: '(?#)*'
+   * repository:
+   *   multi-letter:
+   *     match:
+   *       repeat: '[A-Za-z]'
+   *     name: scope.multi-letter
+   *
+   * would be transformed to
+   *
+   * repository:
+   *   multi-letter:
+   *     match: '(?:[A-Za-z])*'
+   *     name: scope.multi-letter
+   */
+  macros?: Record<string, string>;
+
+  patterns: Array<Pattern<MatchType>>;
+  repository?: Record<string, Pattern<MatchType>>;
+}
+
+export interface Pattern<MatchType = string> {
+  include?: string;
+  match?: MatchType;
+  begin?: MatchType;
+  end?: MatchType;
+  while?: MatchType;
+  captures?: Record<string, PatternCapture>;
+  beginCaptures?: Record<string, PatternCapture>;
+  endCaptures?: Record<string, PatternCapture>;
+  patterns?: Array<Pattern<MatchType>>;
+  beginPattern?: string;
+  endPattern?: string;
+}
+
+export interface PatternCapture {
+  name?: string;
+  patterns?: Pattern[];
+}
+
+export type ExtendedMatchType = string | Record<string, string>;

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -3,6 +3,12 @@ import { load } from "js-yaml";
 import { obj } from "through2";
 import PluginError from "plugin-error";
 import type Vinyl from "vinyl";
+import type {
+  ExtendedMatchType,
+  ExtendedTextmateGrammar,
+  Pattern,
+  TextmateGrammar,
+} from "./textmate-grammar";
 
 /**
  * Replaces all rule references with the match pattern of the referenced rule.
@@ -34,7 +40,9 @@ function replaceReferencesWithStrings(
  * @param yaml The root of the YAML document.
  * @returns A map from macro name to replacement text.
  */
-function gatherMacros(yaml: any): Map<string, string> {
+function gatherMacros<T>(
+  yaml: ExtendedTextmateGrammar<T>,
+): Map<string, string> {
   const macros = new Map<string, string>();
   for (const key in yaml.macros) {
     macros.set(key, yaml.macros[key]);
@@ -51,7 +59,7 @@ function gatherMacros(yaml: any): Map<string, string> {
  * @returns The match text for the rule. This is either the value of the rule's `match` property,
  *  or the disjunction of the match text of all of the other rules `include`d by this rule.
  */
-function getNodeMatchText(rule: any): string {
+function getNodeMatchText(rule: Pattern): string {
   if (rule.match !== undefined) {
     // For a match string, just use that string as the replacement.
     return rule.match;
@@ -78,7 +86,7 @@ function getNodeMatchText(rule: any): string {
  * @returns A map whose keys are the names of rules, and whose values are the corresponding match
  *  text of each rule.
  */
-function gatherMatchTextForRules(yaml: any): Map<string, string> {
+function gatherMatchTextForRules(yaml: TextmateGrammar): Map<string, string> {
   const replacements = new Map<string, string>();
   for (const key in yaml.repository) {
     const node = yaml.repository[key];
@@ -94,9 +102,14 @@ function gatherMatchTextForRules(yaml: any): Map<string, string> {
  * @param yaml The root of the YAML document.
  * @param action Callback to invoke on each rule.
  */
-function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
+function visitAllRulesInFile<T>(
+  yaml: ExtendedTextmateGrammar<T>,
+  action: (rule: Pattern<T>) => void,
+) {
   visitAllRulesInRuleMap(yaml.patterns, action);
-  visitAllRulesInRuleMap(yaml.repository, action);
+  if (yaml.repository) {
+    visitAllRulesInRuleMap(Object.values(yaml.repository), action);
+  }
 }
 
 /**
@@ -107,9 +120,11 @@ function visitAllRulesInFile(yaml: any, action: (rule: any) => void) {
  * @param ruleMap The map or array of rules to visit.
  * @param action Callback to invoke on each rule.
  */
-function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
-  for (const key in ruleMap) {
-    const rule = ruleMap[key];
+function visitAllRulesInRuleMap<T>(
+  ruleMap: Array<Pattern<T>>,
+  action: (rule: Pattern<T>) => void,
+) {
+  for (const rule of ruleMap) {
     if (typeof rule === "object") {
       action(rule);
       if (rule.patterns !== undefined) {
@@ -125,16 +140,22 @@ function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) {
  * @param rule The rule whose matches are to be transformed.
  * @param action The transformation to make on each match pattern.
  */
-function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
+function visitAllMatchesInRule<T>(rule: Pattern<T>, action: (match: T) => T) {
   for (const key in rule) {
     switch (key) {
       case "begin":
       case "end":
       case "match":
-      case "while":
-        rule[key] = action(rule[key]);
-        break;
+      case "while": {
+        const ruleElement = rule[key];
 
+        if (!ruleElement) {
+          continue;
+        }
+
+        rule[key] = action(ruleElement);
+        break;
+      }
       default:
         break;
     }
@@ -148,14 +169,17 @@ function visitAllMatchesInRule(rule: any, action: (match: any) => any) {
  * @param rule Rule to be transformed.
  * @param key Base key of the property to be transformed.
  */
-function expandPatternMatchProperties(rule: any, key: "begin" | "end") {
-  const patternKey = `${key}Pattern`;
-  const capturesKey = `${key}Captures`;
+function expandPatternMatchProperties<T>(
+  rule: Pattern<T>,
+  key: "begin" | "end",
+) {
+  const patternKey = `${key}Pattern` as const;
+  const capturesKey = `${key}Captures` as const;
   const pattern = rule[patternKey];
   if (pattern !== undefined) {
     const patterns: string[] = Array.isArray(pattern) ? pattern : [pattern];
-    rule[key] = patterns.map((p) => `((?${p}))`).join("|");
-    const captures: { [index: string]: any } = {};
+    rule[key] = patterns.map((p) => `((?${p}))`).join("|") as T;
+    const captures: Pattern["captures"] = {};
     for (const patternIndex in patterns) {
       captures[(Number(patternIndex) + 1).toString()] = {
         patterns: [
@@ -175,7 +199,7 @@ function expandPatternMatchProperties(rule: any, key: "begin" | "end") {
  *
  * @param yaml The root of the YAML document.
  */
-function transformFile(yaml: any) {
+function transformFile(yaml: ExtendedTextmateGrammar<ExtendedMatchType>) {
   const macros = gatherMacros(yaml);
   visitAllRulesInFile(yaml, (rule) => {
     expandPatternMatchProperties(rule, "begin");
@@ -198,24 +222,29 @@ function transformFile(yaml: any) {
 
   yaml.macros = undefined;
 
-  const replacements = gatherMatchTextForRules(yaml);
+  // We have removed all object match properties, so we don't have an extended match type anymore.
+  const macrolessYaml = yaml as ExtendedTextmateGrammar;
+
+  const replacements = gatherMatchTextForRules(macrolessYaml);
   // Expand references in matches.
-  visitAllRulesInFile(yaml, (rule) => {
+  visitAllRulesInFile(macrolessYaml, (rule) => {
     visitAllMatchesInRule(rule, (match) => {
       return replaceReferencesWithStrings(match, replacements);
     });
   });
 
-  if (yaml.regexOptions !== undefined) {
-    const regexOptions = `(?${yaml.regexOptions})`;
-    visitAllRulesInFile(yaml, (rule) => {
+  if (macrolessYaml.regexOptions !== undefined) {
+    const regexOptions = `(?${macrolessYaml.regexOptions})`;
+    visitAllRulesInFile(macrolessYaml, (rule) => {
       visitAllMatchesInRule(rule, (match) => {
         return regexOptions + match;
       });
     });
 
-    yaml.regexOptions = undefined;
+    macrolessYaml.regexOptions = undefined;
   }
+
+  return macrolessYaml;
 }
 
 export function transpileTextMateGrammar() {
@@ -230,8 +259,8 @@ export function transpileTextMateGrammar() {
       } else if (file.isBuffer()) {
         const buf: Buffer = file.contents;
         const yamlText: string = buf.toString("utf8");
-        const jsonData: any = load(yamlText);
-        transformFile(jsonData);
+        const yamlData = load(yamlText) as TextmateGrammar;
+        const jsonData = transformFile(yamlData);
 
         file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), "utf8");
         file.extname = ".json";

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.12.2",
+  "version": "1.12.3",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -178,6 +178,11 @@
             "type": "string",
             "default": "",
             "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
+          },
+          "codeQL.cli.downloadTimeout": {
+            "type": "integer",
+            "default": 10,
+            "description": "Download timeout in seconds for downloading the CLI distribution."
           }
         }
       },
@@ -376,6 +381,11 @@
         "title": "Adding databases",
         "order": 6,
         "properties": {
+          "codeQL.addingDatabases.downloadTimeout": {
+            "type": "integer",
+            "default": 10,
+            "description": "Download timeout in seconds for adding a CodeQL database."
+          },
           "codeQL.addingDatabases.allowHttp": {
             "type": "boolean",
             "default": false,
@@ -1432,7 +1442,7 @@
         },
         {
           "command": "codeQL.quickEvalCount",
-          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+          "when": "editorLangId == ql"
         },
         {
           "command": "codeQL.quickEvalContextEditor",
@@ -1937,7 +1947,7 @@
     "@vscode/webview-ui-toolkit": "^1.0.1",
     "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
-    "chokidar": "^3.5.3",
+    "chokidar": "^3.6.0",
     "d3": "^7.6.1",
     "d3-graphviz": "^5.0.2",
     "fs-extra": "^11.1.1",
@@ -1968,24 +1978,24 @@
     "@babel/preset-env": "^7.23.9",
     "@babel/preset-react": "^7.18.6",
     "@babel/preset-typescript": "^7.21.4",
-    "@faker-js/faker": "^8.0.2",
+    "@faker-js/faker": "^8.4.1",
     "@github/markdownlint-github": "^0.6.0",
     "@octokit/plugin-throttling": "^8.0.0",
     "@playwright/test": "^1.40.1",
-    "@storybook/addon-a11y": "^7.6.13",
+    "@storybook/addon-a11y": "^7.6.15",
     "@storybook/addon-actions": "^7.1.0",
     "@storybook/addon-essentials": "^7.1.0",
     "@storybook/addon-interactions": "^7.1.0",
     "@storybook/addon-links": "^7.1.0",
-    "@storybook/components": "^7.6.7",
+    "@storybook/components": "^7.6.17",
     "@storybook/csf": "^0.1.1",
     "@storybook/manager-api": "^7.6.7",
     "@storybook/react": "^7.1.0",
     "@storybook/react-webpack5": "^7.6.12",
     "@storybook/theming": "^7.6.12",
     "@testing-library/dom": "^9.3.4",
-    "@testing-library/jest-dom": "^6.2.0",
-    "@testing-library/react": "^14.0.0",
+    "@testing-library/jest-dom": "^6.4.2",
+    "@testing-library/react": "^14.2.1",
     "@testing-library/user-event": "^14.5.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/d3": "^7.4.0",
@@ -2012,7 +2022,7 @@
     "@types/vscode": "^1.82.0",
     "@types/yauzl": "^2.10.3",
     "@typescript-eslint/eslint-plugin": "^6.19.0",
-    "@typescript-eslint/parser": "^6.16.0",
+    "@typescript-eslint/parser": "^6.21.0",
     "@vscode/test-electron": "^2.2.0",
     "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
@@ -2046,11 +2056,11 @@
     "lint-staged": "^15.0.2",
     "markdownlint-cli2": "^0.12.1",
     "markdownlint-cli2-formatter-pretty": "^0.0.5",
-    "mini-css-extract-plugin": "^2.7.7",
+    "mini-css-extract-plugin": "^2.8.0",
     "npm-run-all": "^4.1.5",
     "patch-package": "^8.0.0",
     "prettier": "^3.2.5",
-    "storybook": "^7.6.10",
+    "storybook": "^7.6.15",
     "tar-stream": "^3.0.0",
     "through2": "^4.0.2",
     "ts-jest": "^29.0.1",

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -156,9 +156,17 @@ type ResolvedQueries = string[];
 type ResolvedTests = string[];
 
 /**
- * A compilation message for a test message (either an error or a warning)
+ * The severity of a compilation message for a test message.
  */
-interface CompilationMessage {
+export enum CompilationMessageSeverity {
+  Error = "ERROR",
+  Warning = "WARNING",
+}
+
+/**
+ * A compilation message for a test message (either an error or a warning).
+ */
+export interface CompilationMessage {
   /**
    * The text of the message
    */
@@ -170,7 +178,7 @@ interface CompilationMessage {
   /**
    * The severity of the message
    */
-  severity: number;
+  severity: CompilationMessageSeverity;
 }
 
 /**
@@ -362,6 +370,8 @@ export class CodeQLCliServer implements Disposable {
     silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
+    // The current buffer of stderr of a single line. To be used for logging.
+    let currentLineStderrBuffer: Buffer = Buffer.alloc(0);
     if (this.commandInProcess) {
       throw new Error("runCodeQlCliInternal called while cli was running");
     }
@@ -419,6 +429,38 @@ export class CodeQLCliServer implements Disposable {
           // Listen to stderr
           process.stderr.addListener("data", (newData: Buffer) => {
             stderrBuffers.push(newData);
+
+            if (!silent) {
+              currentLineStderrBuffer = Buffer.concat([
+                currentLineStderrBuffer,
+                newData,
+              ]);
+
+              // Print the stderr to the logger as it comes in. We need to ensure that
+              // we don't split messages on the same line, so we buffer the stderr and
+              // split it on EOLs.
+              const eolBuffer = Buffer.from(EOL);
+
+              let hasCreatedSubarray = false;
+
+              let eolIndex;
+              while (
+                (eolIndex = currentLineStderrBuffer.indexOf(eolBuffer)) !== -1
+              ) {
+                const line = currentLineStderrBuffer.subarray(0, eolIndex);
+                void this.logger.log(line.toString("utf-8"));
+                currentLineStderrBuffer = currentLineStderrBuffer.subarray(
+                  eolIndex + eolBuffer.length,
+                );
+                hasCreatedSubarray = true;
+              }
+
+              // We have created a subarray, which means that the complete original buffer is now referenced
+              // by the subarray. We need to create a new buffer to avoid memory leaks.
+              if (hasCreatedSubarray) {
+                currentLineStderrBuffer = Buffer.from(currentLineStderrBuffer);
+              }
+            }
           });
           // Listen for process exit.
           process.addListener("close", (code) =>
@@ -433,6 +475,8 @@ export class CodeQLCliServer implements Disposable {
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
         if (!silent) {
+          void this.logger.log(currentLineStderrBuffer.toString("utf8"));
+          currentLineStderrBuffer = Buffer.alloc(0);
           void this.logger.log("CLI command succeeded.");
         }
         return data;
@@ -452,8 +496,8 @@ export class CodeQLCliServer implements Disposable {
         cliError.stack += getErrorStack(err);
         throw cliError;
       } finally {
-        if (!silent) {
-          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent && currentLineStderrBuffer.length > 0) {
+          void this.logger.log(currentLineStderrBuffer.toString("utf8"));
         }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
@@ -1271,12 +1315,6 @@ export class CodeQLCliServer implements Disposable {
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
     if (extensionPacksOnly) {
-      if (!(await this.cliConstraints.supportsQlpacksKind())) {
-        void this.logger.log(
-          "Warning: Running with extension packs is only supported by CodeQL CLI v2.12.3 or later.",
-        );
-        return {};
-      }
       args.push("--kind", "extension", "--no-recursive");
     } else if (kind) {
       args.push("--kind", kind);
@@ -1412,15 +1450,13 @@ export class CodeQLCliServer implements Disposable {
       args.push("--mode", "update");
     }
     if (workspaceFolders?.length > 0) {
-      if (await this.cliConstraints.supportsAdditionalPacksInstall()) {
-        args.push(
-          // Allow prerelease packs from the ql submodule.
-          "--allow-prerelease",
-          // Allow the use of --additional-packs argument without issueing a warning
-          "--no-strict-mode",
-          ...this.getAdditionalPacksArg(workspaceFolders),
-        );
-      }
+      args.push(
+        // Allow prerelease packs from the ql submodule.
+        "--allow-prerelease",
+        // Allow the use of --additional-packs argument without issueing a warning
+        "--no-strict-mode",
+        ...this.getAdditionalPacksArg(workspaceFolders),
+      );
     }
     return this.runJsonCodeQlCliCommandWithAuthentication(
       ["pack", "install"],
@@ -1521,15 +1557,7 @@ export class CodeQLCliServer implements Disposable {
         this._versionChangedListeners.forEach((listener) =>
           listener(newVersionAndFeatures),
         );
-
         // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
-        await this.app.commands.execute(
-          "setContext",
-          "codeql.supportsQuickEvalCount",
-          newVersionAndFeatures.version.compare(
-            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
-          ) >= 0,
-        );
         await this.app.commands.execute(
           "setContext",
           "codeql.supportsTrimCache",
@@ -1573,11 +1601,8 @@ export class CodeQLCliServer implements Disposable {
     return paths.length ? ["--additional-packs", paths.join(delimiter)] : [];
   }
 
-  public async useExtensionPacks(): Promise<boolean> {
-    return (
-      this.cliConfig.useExtensionPacks &&
-      (await this.cliConstraints.supportsQlpacksKind())
-    );
+  public useExtensionPacks(): boolean {
+    return this.cliConfig.useExtensionPacks;
   }
 
   public async setUseExtensionPacks(useExtensionPacks: boolean) {
@@ -1694,26 +1719,7 @@ function shouldDebugCliServer() {
 export class CliVersionConstraint {
   // The oldest version of the CLI that we support. This is used to determine
   // whether to show a warning about the CLI being too old on startup.
-  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.11.6");
-
-  /**
-   * CLI version that supports the `--kind` option for the `resolve qlpacks` command.
-   */
-  public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
-
-  /**
-   * CLI version that supports the `--additional-packs` option for the `pack install` command.
-   */
-  public static CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL = new SemVer(
-    "2.12.4",
-  );
-
-  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
-
-  /**
-   * CLI version where the query server supports quick-eval count mode.
-   */
-  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.13.5");
 
   /**
    * CLI version where the `generate extensible-predicate-metadata`
@@ -1753,34 +1759,12 @@ export class CliVersionConstraint {
     return (await this.cli.getVersion()).compare(v) >= 0;
   }
 
-  async supportsQlpacksKind() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
-    );
-  }
-
-  async supportsAdditionalPacksInstall() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
-    );
-  }
-
-  async usesGlobalCompilationCache() {
-    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
-  }
-
   async supportsVisibilityNotifications() {
     return this.isVersionAtLeast(
       CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
     );
   }
 
-  async supportsQuickEvalCount() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
-    );
-  }
-
   async supportsGenerateExtensiblePredicateMetadata() {
     return this.isVersionAtLeast(
       CliVersionConstraint.CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA,

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -1,3 +1,4 @@
+import type { WriteStream } from "fs";
 import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
@@ -26,6 +27,8 @@ import { unzipToDirectoryConcurrently } from "../common/unzip-concurrently";
 import { reportUnzipProgress } from "../common/vscode/unzip-progress";
 import type { Release } from "./distribution/release";
 import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";
+import { createTimeoutSignal } from "../common/fetch-stream";
+import { AbortError } from "node-fetch";
 
 /**
  * distribution.ts
@@ -384,15 +387,25 @@ class ExtensionSpecificDistributionManager {
       );
     }
 
-    const assetStream =
-      await this.createReleasesApiConsumer().streamBinaryContentOfAsset(
-        assets[0],
-      );
+    const {
+      signal,
+      onData,
+      dispose: disposeTimeout,
+    } = createTimeoutSignal(this.config.downloadTimeout);
+
     const tmpDirectory = await mkdtemp(join(tmpdir(), "vscode-codeql"));
 
+    let archiveFile: WriteStream | undefined = undefined;
+
     try {
+      const assetStream =
+        await this.createReleasesApiConsumer().streamBinaryContentOfAsset(
+          assets[0],
+          signal,
+        );
+
       const archivePath = join(tmpDirectory, "distributionDownload.zip");
-      const archiveFile = createWriteStream(archivePath);
+      archiveFile = createWriteStream(archivePath);
 
       const contentLength = assetStream.headers.get("content-length");
       const totalNumBytes = contentLength
@@ -405,12 +418,23 @@ class ExtensionSpecificDistributionManager {
         progressCallback,
       );
 
-      await new Promise((resolve, reject) =>
+      assetStream.body.on("data", onData);
+
+      await new Promise((resolve, reject) => {
+        if (!archiveFile) {
+          throw new Error("Invariant violation: archiveFile not set");
+        }
+
         assetStream.body
           .pipe(archiveFile)
           .on("finish", resolve)
-          .on("error", reject),
-      );
+          .on("error", reject);
+
+        // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
+        assetStream.body.on("error", reject);
+      });
+
+      disposeTimeout();
 
       await this.bumpDistributionFolderIndex();
 
@@ -427,7 +451,19 @@ class ExtensionSpecificDistributionManager {
             )
           : undefined,
       );
+    } catch (e) {
+      if (e instanceof AbortError) {
+        const thrownError = new AbortError("The download timed out.");
+        thrownError.stack = e.stack;
+        throw thrownError;
+      }
+
+      throw e;
     } finally {
+      disposeTimeout();
+
+      archiveFile?.close();
+
       await remove(tmpDirectory);
     }
   }

extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts

@@ -90,21 +90,28 @@ export class ReleasesApiConsumer {
 
   public async streamBinaryContentOfAsset(
     asset: ReleaseAsset,
+    signal?: AbortSignal,
   ): Promise<Response> {
     const apiPath = `/repos/${this.repositoryNwo}/releases/assets/${asset.id}`;
 
-    return await this.makeApiCall(apiPath, {
-      accept: "application/octet-stream",
-    });
+    return await this.makeApiCall(
+      apiPath,
+      {
+        accept: "application/octet-stream",
+      },
+      signal,
+    );
   }
 
   protected async makeApiCall(
     apiPath: string,
     additionalHeaders: { [key: string]: string } = {},
+    signal?: AbortSignal,
   ): Promise<Response> {
     const response = await this.makeRawRequest(
       ReleasesApiConsumer.apiBase + apiPath,
       Object.assign({}, this.defaultHeaders, additionalHeaders),
+      signal,
     );
 
     if (!response.ok) {
@@ -129,11 +136,13 @@ export class ReleasesApiConsumer {
   private async makeRawRequest(
     requestUrl: string,
     headers: { [key: string]: string },
+    signal?: AbortSignal,
     redirectCount = 0,
   ): Promise<Response> {
     const response = await fetch(requestUrl, {
       headers,
       redirect: "manual",
+      signal,
     });
 
     const redirectUrl = response.headers.get("location");
@@ -153,7 +162,12 @@ export class ReleasesApiConsumer {
         // mechanism is provided.
         delete headers["authorization"];
       }
-      return await this.makeRawRequest(redirectUrl, headers, redirectCount + 1);
+      return await this.makeRawRequest(
+        redirectUrl,
+        headers,
+        signal,
+        redirectCount + 1,
+      );
     }
 
     return response;

extensions/ql-vscode/src/common/fetch-stream.ts

@@ -0,0 +1,36 @@
+import { clearTimeout } from "node:timers";
+
+export function createTimeoutSignal(timeoutSeconds: number): {
+  signal: AbortSignal;
+  onData: () => void;
+  dispose: () => void;
+} {
+  const timeout = timeoutSeconds * 1000;
+
+  const abortController = new AbortController();
+
+  let timeoutId: NodeJS.Timeout;
+
+  // If we don't get any data within the timeout, abort the download
+  timeoutId = setTimeout(() => {
+    abortController.abort();
+  }, timeout);
+
+  // If we receive any data within the timeout, reset the timeout
+  const onData = () => {
+    clearTimeout(timeoutId);
+    timeoutId = setTimeout(() => {
+      abortController.abort();
+    }, timeout);
+  };
+
+  const dispose = () => {
+    clearTimeout(timeoutId);
+  };
+
+  return {
+    signal: abortController.signal,
+    onData,
+    dispose,
+  };
+}

extensions/ql-vscode/src/common/interface-types.ts

@@ -25,6 +25,7 @@ import type {
   UrlValueResolvable,
 } from "./raw-result-types";
 import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
+import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
 
 /**
  * This module contains types and code that are shared between
@@ -528,9 +529,10 @@ interface SetMethodsMessage {
   methods: Method[];
 }
 
-interface SetModeledMethodsMessage {
-  t: "setModeledMethods";
+interface SetModeledAndModifiedMethodsMessage {
+  t: "setModeledAndModifiedMethods";
   methods: Record<string, ModeledMethod[]>;
+  modifiedMethodSignatures: string[];
 }
 
 interface SetModifiedMethodsMessage {
@@ -543,6 +545,11 @@ interface SetInProgressMethodsMessage {
   methods: string[];
 }
 
+interface SetProcessedByAutoModelMethodsMessage {
+  t: "setProcessedByAutoModelMethods";
+  methods: string[];
+}
+
 interface SwitchModeMessage {
   t: "switchMode";
   mode: Mode;
@@ -585,6 +592,14 @@ interface StopGeneratingMethodsFromLlmMessage {
   packageName: string;
 }
 
+interface StartModelEvaluationMessage {
+  t: "startModelEvaluation";
+}
+
+interface StopModelEvaluationMessage {
+  t: "stopModelEvaluation";
+}
+
 interface ModelDependencyMessage {
   t: "modelDependency";
 }
@@ -610,6 +625,11 @@ interface SetInProgressMessage {
   inProgress: boolean;
 }
 
+interface SetProcessedByAutoModelMessage {
+  t: "setProcessedByAutoModel";
+  processedByAutoModel: boolean;
+}
+
 interface RevealMethodMessage {
   t: "revealMethod";
   methodSignature: string;
@@ -620,14 +640,21 @@ interface SetAccessPathSuggestionsMessage {
   accessPathSuggestions: AccessPathSuggestionOptions;
 }
 
+interface SetModelEvaluationRunMessage {
+  t: "setModelEvaluationRun";
+  run: ModelEvaluationRunState | undefined;
+}
+
 export type ToModelEditorMessage =
   | SetExtensionPackStateMessage
   | SetMethodsMessage
-  | SetModeledMethodsMessage
+  | SetModeledAndModifiedMethodsMessage
   | SetModifiedMethodsMessage
   | SetInProgressMethodsMessage
+  | SetProcessedByAutoModelMethodsMessage
   | RevealMethodMessage
-  | SetAccessPathSuggestionsMessage;
+  | SetAccessPathSuggestionsMessage
+  | SetModelEvaluationRunMessage;
 
 export type FromModelEditorMessage =
   | CommonFromViewMessages
@@ -642,7 +669,9 @@ export type FromModelEditorMessage =
   | StopGeneratingMethodsFromLlmMessage
   | ModelDependencyMessage
   | HideModeledMethodsMessage
-  | SetMultipleModeledMethodsMessage;
+  | SetMultipleModeledMethodsMessage
+  | StartModelEvaluationMessage
+  | StopModelEvaluationMessage;
 
 interface RevealInEditorMessage {
   t: "revealInModelEditor";
@@ -680,6 +709,7 @@ interface SetSelectedMethodMessage {
   modeledMethods: ModeledMethod[];
   isModified: boolean;
   isInProgress: boolean;
+  processedByAutoModel: boolean;
 }
 
 export type ToMethodModelingMessage =
@@ -689,4 +719,5 @@ export type ToMethodModelingMessage =
   | SetMethodModifiedMessage
   | SetSelectedMethodMessage
   | SetInModelingModeMessage
-  | SetInProgressMessage;
+  | SetInProgressMessage
+  | SetProcessedByAutoModelMessage;

extensions/ql-vscode/src/compare/sarif-diff.ts

@@ -1,4 +1,29 @@
-import type { Result } from "sarif";
+import type { Location, Result } from "sarif";
+
+function toCanonicalLocation(location: Location): Location {
+  if (location.physicalLocation?.artifactLocation?.index !== undefined) {
+    const canonicalLocation = {
+      ...location,
+    };
+
+    canonicalLocation.physicalLocation = {
+      ...canonicalLocation.physicalLocation,
+    };
+
+    canonicalLocation.physicalLocation.artifactLocation = {
+      ...canonicalLocation.physicalLocation.artifactLocation,
+    };
+
+    // The index is dependent on the result of the SARIF file and usually doesn't really tell
+    // us anything useful, so we remove it from the comparison.
+    delete canonicalLocation.physicalLocation.artifactLocation.index;
+
+    return canonicalLocation;
+  }
+
+  // Don't create a new object if we don't need to
+  return location;
+}
 
 function toCanonicalResult(result: Result): Result {
   const canonicalResult = {
@@ -6,29 +31,45 @@ function toCanonicalResult(result: Result): Result {
   };
 
   if (canonicalResult.locations) {
-    canonicalResult.locations = canonicalResult.locations.map((location) => {
-      if (location.physicalLocation?.artifactLocation?.index !== undefined) {
-        const canonicalLocation = {
-          ...location,
+    canonicalResult.locations =
+      canonicalResult.locations.map(toCanonicalLocation);
+  }
+
+  if (canonicalResult.relatedLocations) {
+    canonicalResult.relatedLocations =
+      canonicalResult.relatedLocations.map(toCanonicalLocation);
+  }
+
+  if (canonicalResult.codeFlows) {
+    canonicalResult.codeFlows = canonicalResult.codeFlows.map((codeFlow) => {
+      if (codeFlow.threadFlows) {
+        return {
+          ...codeFlow,
+          threadFlows: codeFlow.threadFlows.map((threadFlow) => {
+            if (threadFlow.locations) {
+              return {
+                ...threadFlow,
+                locations: threadFlow.locations.map((threadFlowLocation) => {
+                  if (threadFlowLocation.location) {
+                    return {
+                      ...threadFlowLocation,
+                      location: toCanonicalLocation(
+                        threadFlowLocation.location,
+                      ),
+                    };
+                  }
+
+                  return threadFlowLocation;
+                }),
+              };
+            }
+
+            return threadFlow;
+          }),
         };
-
-        canonicalLocation.physicalLocation = {
-          ...canonicalLocation.physicalLocation,
-        };
-
-        canonicalLocation.physicalLocation.artifactLocation = {
-          ...canonicalLocation.physicalLocation.artifactLocation,
-        };
-
-        // The index is dependent on the result of the SARIF file and usually doesn't really tell
-        // us anything useful, so we remove it from the comparison.
-        delete canonicalLocation.physicalLocation.artifactLocation.index;
-
-        return canonicalLocation;
       }
 
-      // Don't create a new object if we don't need to
-      return location;
+      return codeFlow;
     });
   }
 

extensions/ql-vscode/src/config.ts

@@ -93,6 +93,10 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting(
   "personalAccessToken",
   DISTRIBUTION_SETTING,
 );
+const CLI_DOWNLOAD_TIMEOUT_SETTING = new Setting(
+  "downloadTimeout",
+  DISTRIBUTION_SETTING,
+);
 const CLI_CHANNEL_SETTING = new Setting("channel", DISTRIBUTION_SETTING);
 
 // Query History configuration
@@ -118,6 +122,7 @@ export interface DistributionConfig {
   updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
+  downloadTimeout: number;
   channel: CLIChannel;
   onDidChangeConfiguration?: Event<void>;
 }
@@ -272,6 +277,10 @@ export class DistributionConfigListener
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
+  public get downloadTimeout(): number {
+    return CLI_DOWNLOAD_TIMEOUT_SETTING.getValue() || 10;
+  }
+
   public async updateCustomCodeQlPath(newPath: string | undefined) {
     await CUSTOM_CODEQL_PATH_SETTING.updateValue(
       newPath,
@@ -644,8 +653,16 @@ const DEPRECATED_ALLOW_HTTP_SETTING = new Setting(
 
 const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING);
 
+const DOWNLOAD_TIMEOUT_SETTING = new Setting(
+  "downloadTimeout",
+  ADDING_DATABASES_SETTING,
+);
 const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING);
 
+export function downloadTimeout(): number {
+  return DOWNLOAD_TIMEOUT_SETTING.getValue<number>() || 10;
+}
+
 export function allowHttp(): boolean {
   return (
     ALLOW_HTTP_SETTING.getValue<boolean>() ||
@@ -707,6 +724,7 @@ export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
 const MODEL_SETTING = new Setting("model", ROOT_SETTING);
 const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING);
+const SHOW_TYPE_MODELS = new Setting("showTypeModels", MODEL_SETTING);
 const LLM_GENERATION_BATCH_SIZE = new Setting(
   "llmGenerationBatchSize",
   MODEL_SETTING,
@@ -715,6 +733,7 @@ const LLM_GENERATION_DEV_ENDPOINT = new Setting(
   "llmGenerationDevEndpoint",
   MODEL_SETTING,
 );
+const MODEL_EVALUATION = new Setting("evaluation", MODEL_SETTING);
 const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
 const ENABLE_PYTHON = new Setting("enablePython", MODEL_SETTING);
 const ENABLE_ACCESS_PATH_SUGGESTIONS = new Setting(
@@ -725,6 +744,7 @@ const ENABLE_ACCESS_PATH_SUGGESTIONS = new Setting(
 export interface ModelConfig {
   flowGeneration: boolean;
   llmGeneration: boolean;
+  showTypeModels: boolean;
   getExtensionsDirectory(languageId: string): string | undefined;
   enablePython: boolean;
   enableAccessPathSuggestions: boolean;
@@ -743,6 +763,10 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return !!LLM_GENERATION.getValue<boolean>();
   }
 
+  public get showTypeModels(): boolean {
+    return !!SHOW_TYPE_MODELS.getValue<boolean>();
+  }
+
   /**
    * Limits the number of candidates we send to the model in each request to avoid long requests.
    * Note that the model may return fewer than this number of candidates.
@@ -759,6 +783,10 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return LLM_GENERATION_DEV_ENDPOINT.getValue<string | undefined>();
   }
 
+  public get modelEvaluation(): boolean {
+    return !!MODEL_EVALUATION.getValue<boolean>();
+  }
+
   public getExtensionsDirectory(languageId: string): string | undefined {
     return EXTENSIONS_DIRECTORY.getValue<string>({
       languageId,

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -1,5 +1,5 @@
 import type { Response } from "node-fetch";
-import fetch from "node-fetch";
+import fetch, { AbortError } from "node-fetch";
 import { zip } from "zip-a-folder";
 import type { InputBoxOptions } from "vscode";
 import { Uri, window } from "vscode";
@@ -28,11 +28,16 @@ import {
 } from "../common/github-url-identifier-helper";
 import type { Credentials } from "../common/authentication";
 import type { AppCommandManager } from "../common/commands";
-import { addDatabaseSourceToWorkspace, allowHttp } from "../config";
+import {
+  addDatabaseSourceToWorkspace,
+  allowHttp,
+  downloadTimeout,
+} from "../config";
 import { showAndLogInformationMessage } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
 import { getLanguageDisplayName } from "../common/query-language";
 import type { DatabaseOrigin } from "./local-databases/database-origin";
+import { createTimeoutSignal } from "../common/fetch-stream";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -478,10 +483,33 @@ async function fetchAndUnzip(
     step: 1,
   });
 
-  const response = await checkForFailingResponse(
-    await fetch(databaseUrl, { headers: requestHeaders }),
-    "Error downloading database",
-  );
+  const {
+    signal,
+    onData,
+    dispose: disposeTimeout,
+  } = createTimeoutSignal(downloadTimeout());
+
+  let response: Response;
+  try {
+    response = await checkForFailingResponse(
+      await fetch(databaseUrl, {
+        headers: requestHeaders,
+        signal,
+      }),
+      "Error downloading database",
+    );
+  } catch (e) {
+    disposeTimeout();
+
+    if (e instanceof AbortError) {
+      const thrownError = new AbortError("The request timed out.");
+      thrownError.stack = e.stack;
+      throw thrownError;
+    }
+
+    throw e;
+  }
+
   const archiveFileStream = createWriteStream(archivePath);
 
   const contentLength = response.headers.get("content-length");
@@ -493,12 +521,34 @@ async function fetchAndUnzip(
     progress,
   );
 
-  await new Promise((resolve, reject) =>
-    response.body
-      .pipe(archiveFileStream)
-      .on("finish", resolve)
-      .on("error", reject),
-  );
+  response.body.on("data", onData);
+
+  try {
+    await new Promise((resolve, reject) => {
+      response.body
+        .pipe(archiveFileStream)
+        .on("finish", resolve)
+        .on("error", reject);
+
+      // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
+      response.body.on("error", reject);
+    });
+  } catch (e) {
+    // Close and remove the file if an error occurs
+    archiveFileStream.close(() => {
+      void remove(archivePath);
+    });
+
+    if (e instanceof AbortError) {
+      const thrownError = new AbortError("The download timed out.");
+      thrownError.stack = e.stack;
+      throw thrownError;
+    }
+
+    throw e;
+  } finally {
+    disposeTimeout();
+  }
 
   await readAndUnzip(
     Uri.file(archivePath).toString(true),

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -828,6 +828,12 @@ export class DatabaseUI extends DisposableObject {
   }
 
   private async promptForDatabase(): Promise<void> {
+    // If there aren't any existing databases,
+    // don't bother asking the user if they want to pick one.
+    if (this.databaseManager.databaseItems.length === 0) {
+      return this.importNewDatabase();
+    }
+
     const quickPickItems: DatabaseSelectionQuickPickItem[] = [
       {
         label: "$(database) Existing database",
@@ -837,7 +843,8 @@ export class DatabaseUI extends DisposableObject {
       },
       {
         label: "$(arrow-down) New database",
-        detail: "Import a new database from the cloud or your local machine",
+        detail:
+          "Import a new database from GitHub, a URL, or your local machine...",
         alwaysShow: true,
         databaseKind: "new",
       },
@@ -871,7 +878,7 @@ export class DatabaseUI extends DisposableObject {
       }));
 
     const selectedDatabase = await window.showQuickPick(dbItems, {
-      placeHolder: "Select a database",
+      placeHolder: "Select an existing database from your workspace...",
       ignoreFocusOut: true,
     });
 
@@ -913,7 +920,8 @@ export class DatabaseUI extends DisposableObject {
     ];
     const selectedImportOption =
       await window.showQuickPick<DatabaseImportQuickPickItems>(importOptions, {
-        placeHolder: "Import a database from...",
+        placeHolder:
+          "Import a new database from GitHub, a URL, or your local machine...",
         ignoreFocusOut: true,
       });
     if (!selectedImportOption) {

extensions/ql-vscode/src/extension.ts

@@ -977,6 +977,7 @@ async function activateWithInstalledDistribution(
   const modelEditorModule = await ModelEditorModule.initialize(
     app,
     dbm,
+    variantAnalysisManager,
     cliServer,
     qs,
     tmpDir.name,

extensions/ql-vscode/src/local-queries/local-queries.ts

@@ -41,7 +41,6 @@ import { LocalQueryInfo } from "../query-results";
 import type { WebviewReveal } from "./webview";
 import { asError, getErrorMessage } from "../common/helpers-pure";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
-import { CliVersionConstraint } from "../codeql-cli/cli";
 import type { LocalQueryCommands } from "../common/commands";
 import { DisposableObject } from "../common/disposable-object";
 import { SkeletonQueryWizard } from "./skeleton-query-wizard";
@@ -256,11 +255,6 @@ export class LocalQueries extends DisposableObject {
   private async quickEvalCount(uri: Uri): Promise<void> {
     await withProgress(
       async (progress, token) => {
-        if (!(await this.cliServer.cliConstraints.supportsQuickEvalCount())) {
-          throw new Error(
-            `Quick evaluation count is only supported by CodeQL CLI v${CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT} or later.`,
-          );
-        }
         await this.compileAndRunQuery(
           QuickEvalType.QuickEvalCount,
           uri,
@@ -594,7 +588,7 @@ export class LocalQueries extends DisposableObject {
   public async getDefaultExtensionPacks(
     additionalPacks: string[],
   ): Promise<string[]> {
-    return (await this.cliServer.useExtensionPacks())
+    return this.cliServer.useExtensionPacks()
       ? Object.keys(await this.cliServer.resolveQlpacks(additionalPacks, true))
       : [];
   }

extensions/ql-vscode/src/model-editor/auto-model.ts

@@ -5,53 +5,6 @@ import type { AutoModelQueriesResult } from "./auto-model-codeml-queries";
 import { assertNever } from "../common/helpers-pure";
 import type { Log } from "sarif";
 import { gzipEncode } from "../common/zlib";
-import type { Method, MethodSignature } from "./method";
-import type { ModeledMethod } from "./modeled-method";
-import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
-
-/**
- * Return the candidates that the model should be run on. This includes limiting the number of
- * candidates to the candidate limit and filtering out anything that is already modeled and respecting
- * the order in the UI.
- * @param mode Whether it is application or framework mode.
- * @param methods all methods.
- * @param modeledMethodsBySignature the currently modeled methods.
- * @returns list of modeled methods that are candidates for modeling.
- */
-export function getCandidates(
-  mode: Mode,
-  methods: readonly Method[],
-  modeledMethodsBySignature: Record<string, readonly ModeledMethod[]>,
-): MethodSignature[] {
-  // Sort the same way as the UI so we send the first ones listed in the UI first
-  const grouped = groupMethods(methods, mode);
-  const sortedGroupNames = sortGroupNames(grouped);
-  const sortedMethods = sortedGroupNames.flatMap((name) =>
-    sortMethods(grouped[name]),
-  );
-
-  const candidates: MethodSignature[] = [];
-
-  for (const method of sortedMethods) {
-    const modeledMethods: ModeledMethod[] = [
-      ...(modeledMethodsBySignature[method.signature] ?? []),
-    ];
-
-    // Anything that is modeled is not a candidate
-    if (modeledMethods.some((m) => m.type !== "none")) {
-      continue;
-    }
-
-    // A method that is supported is modeled outside of the model file, so it is not a candidate.
-    if (method.supported) {
-      continue;
-    }
-
-    // The rest are candidates
-    candidates.push(method);
-  }
-  return candidates;
-}
 
 /**
  * Encode a SARIF log to the format expected by the server: JSON, GZIP-compressed, base64-encoded

extensions/ql-vscode/src/model-editor/auto-modeler.ts

@@ -3,7 +3,8 @@ import type { ModeledMethod } from "./modeled-method";
 import { load as loadYaml } from "js-yaml";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { withProgress } from "../common/vscode/progress";
-import { createAutoModelRequest, getCandidates } from "./auto-model";
+import { createAutoModelRequest } from "./auto-model";
+import { getCandidates } from "./shared/auto-model-candidates";
 import { runAutoModelQueries } from "./auto-model-codeml-queries";
 import { loadDataExtensionYaml } from "./yaml";
 import type { ModelRequest, ModelResponse } from "./auto-model-api";
@@ -58,6 +59,7 @@ export class AutoModeler {
     packageName: string,
     methods: readonly Method[],
     modeledMethods: Record<string, readonly ModeledMethod[]>,
+    processedByAutoModelMethods: Set<string>,
     mode: Mode,
   ): Promise<void> {
     if (this.jobs.has(packageName)) {
@@ -72,6 +74,7 @@ export class AutoModeler {
         packageName,
         methods,
         modeledMethods,
+        processedByAutoModelMethods,
         mode,
         cancellationTokenSource,
       );
@@ -105,6 +108,7 @@ export class AutoModeler {
     packageName: string,
     methods: readonly Method[],
     modeledMethods: Record<string, readonly ModeledMethod[]>,
+    processedByAutoModelMethods: Set<string>,
     mode: Mode,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
@@ -114,7 +118,12 @@ export class AutoModeler {
 
     await withProgress(async (progress) => {
       // Fetch the candidates to send to the model
-      const allCandidateMethods = getCandidates(mode, methods, modeledMethods);
+      const allCandidateMethods = getCandidates(
+        mode,
+        methods,
+        modeledMethods,
+        processedByAutoModelMethods,
+      );
 
       // If there are no candidates, there is nothing to model and we just return
       if (allCandidateMethods.length === 0) {
@@ -159,6 +168,12 @@ export class AutoModeler {
             this.databaseItem,
             candidateSignatures,
           );
+
+          // Let the UI know which methods have been sent to the LLM
+          this.modelingStore.addProcessedByAutoModelMethods(
+            this.databaseItem,
+            candidateSignatures,
+          );
         }
       } finally {
         // Clear out in progress methods in case anything went wrong
@@ -223,7 +238,7 @@ export class AutoModeler {
         void showAndLogExceptionWithTelemetry(
           this.app.logger,
           this.app.telemetry,
-          redactableError(e)`Rate limit hit, please try again soon.`,
+          redactableError`Rate limit hit, please try again soon.`,
         );
         return null;
       } else {

extensions/ql-vscode/src/model-editor/generate.ts

@@ -5,19 +5,15 @@ import type { QueryRunner } from "../query-server";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import type { ModeledMethod } from "./modeled-method";
 import { runQuery } from "../local-queries/run-query";
 import type { QueryConstraints } from "../local-queries";
 import { resolveQueries } from "../local-queries";
 import type { DecodedBqrs } from "../common/bqrs-cli-types";
+
 type GenerateQueriesOptions = {
   queryConstraints: QueryConstraints;
   filterQueries?: (queryPath: string) => boolean;
-  parseResults: (
-    queryPath: string,
-    results: DecodedBqrs,
-  ) => ModeledMethod[] | Promise<ModeledMethod[]>;
-  onResults: (results: ModeledMethod[]) => void | Promise<void>;
+  onResults: (queryPath: string, results: DecodedBqrs) => void | Promise<void>;
 
   cliServer: CodeQLCliServer;
   queryRunner: QueryRunner;
@@ -28,7 +24,7 @@ type GenerateQueriesOptions = {
 };
 
 export async function runGenerateQueries(options: GenerateQueriesOptions) {
-  const { queryConstraints, filterQueries, parseResults, onResults } = options;
+  const { queryConstraints, filterQueries, onResults } = options;
 
   options.progress({
     message: "Resolving queries",
@@ -55,7 +51,7 @@ export async function runGenerateQueries(options: GenerateQueriesOptions) {
 
     const bqrs = await runSingleGenerateQuery(queryPath, i, maxStep, options);
     if (bqrs) {
-      await onResults(await parseResults(queryPath, bqrs));
+      await onResults(queryPath, bqrs);
     }
   }
 }

extensions/ql-vscode/src/model-editor/languages/languages.ts

@@ -3,12 +3,14 @@ import type {
   ModelsAsDataLanguage,
   ModelsAsDataLanguagePredicates,
 } from "./models-as-data";
+import { python } from "./python";
 import { ruby } from "./ruby";
 import { staticLanguage } from "./static";
 
 const languages: Partial<Record<QueryLanguage, ModelsAsDataLanguage>> = {
   [QueryLanguage.CSharp]: staticLanguage,
   [QueryLanguage.Java]: staticLanguage,
+  [QueryLanguage.Python]: python,
   [QueryLanguage.Ruby]: ruby,
 };
 

extensions/ql-vscode/src/model-editor/languages/models-as-data.ts

@@ -7,7 +7,7 @@ import type {
   SummaryModeledMethod,
   TypeModeledMethod,
 } from "../modeled-method";
-import type { DataTuple } from "../model-extension-file";
+import type { DataTuple, ModelExtension } from "../model-extension-file";
 import type { Mode } from "../shared/mode";
 import type { QueryConstraints } from "../../local-queries/query-constraints";
 import type {
@@ -17,9 +17,37 @@ import type {
 import type { BaseLogger } from "../../common/logging";
 import type { AccessPathSuggestionRow } from "../suggestions";
 
+// This is a subset of the model config that doesn't import the vscode module.
+// It only includes settings that are actually used.
+export type ModelConfig = {
+  showTypeModels: boolean;
+};
+
+/**
+ * This function creates a new model config object from the given model config object.
+ * The new model config object is a deep copy of the given model config object.
+ *
+ * @param modelConfig The model config object to create a new model config object from.
+ *                    In most cases, this is a `ModelConfigListener`.
+ */
+export function createModelConfig(modelConfig: ModelConfig): ModelConfig {
+  return {
+    showTypeModels: modelConfig.showTypeModels,
+  };
+}
+
+export const defaultModelConfig: ModelConfig = {
+  showTypeModels: false,
+};
+
 type GenerateMethodDefinition<T> = (method: T) => DataTuple[];
 type ReadModeledMethod = (row: DataTuple[]) => ModeledMethod;
 
+type IsHiddenContext = {
+  method: MethodDefinition;
+  config: ModelConfig;
+};
+
 export type ModelsAsDataLanguagePredicate<T> = {
   extensiblePredicate: string;
   supportedKinds?: string[];
@@ -30,22 +58,62 @@ export type ModelsAsDataLanguagePredicate<T> = {
   supportedEndpointTypes?: EndpointType[];
   generateMethodDefinition: GenerateMethodDefinition<T>;
   readModeledMethod: ReadModeledMethod;
+
+  /**
+   * Controls whether this predicate is hidden for a certain method. This only applies to the UI.
+   * If not specified, the predicate is visible for all methods.
+   *
+   * @param method The method to check if the predicate is hidden for.
+   */
+  isHidden?: (context: IsHiddenContext) => boolean;
 };
 
+export type GenerationContext = {
+  mode: Mode;
+  config: ModelConfig;
+};
+
+type ParseGenerationResults = (
+  // The path to the query that generated the results.
+  queryPath: string,
+  // The results of the query.
+  bqrs: DecodedBqrs,
+  // The language-specific predicate that was used to generate the results. This is passed to allow
+  // sharing of code between different languages.
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  // The logger to use for logging.
+  logger: BaseLogger,
+  // Context about this invocation of the generation.
+  context: GenerationContext,
+) => ModeledMethod[];
+
 type ModelsAsDataLanguageModelGeneration = {
-  queryConstraints: QueryConstraints;
+  queryConstraints: (mode: Mode) => QueryConstraints;
   filterQueries?: (queryPath: string) => boolean;
-  parseResults: (
-    // The path to the query that generated the results.
-    queryPath: string,
-    // The results of the query.
-    bqrs: DecodedBqrs,
-    // The language-specific predicate that was used to generate the results. This is passed to allow
-    // sharing of code between different languages.
-    modelsAsDataLanguage: ModelsAsDataLanguage,
-    // The logger to use for logging.
-    logger: BaseLogger,
-  ) => ModeledMethod[];
+  parseResults: ParseGenerationResults;
+};
+
+type ParseResultsToYaml = (
+  // The path to the query that generated the results.
+  queryPath: string,
+  // The results of the query.
+  bqrs: DecodedBqrs,
+  // The language-specific predicate that was used to generate the results. This is passed to allow
+  // sharing of code between different languages.
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  // The logger to use for logging.
+  logger: BaseLogger,
+) => ModelExtension[];
+
+type ModelsAsDataLanguageAutoModelGeneration = {
+  queryConstraints: (mode: Mode) => QueryConstraints;
+  filterQueries?: (queryPath: string) => boolean;
+  parseResultsToYaml: ParseResultsToYaml;
+  /**
+   * By default, auto model generation is enabled for all modes. This function can be used to
+   * override that behavior.
+   */
+  enabled?: (context: GenerationContext) => boolean;
 };
 
 type ModelsAsDataLanguageAccessPathSuggestions = {
@@ -95,6 +163,7 @@ export type ModelsAsDataLanguage = {
   ) => EndpointType | undefined;
   predicates: ModelsAsDataLanguagePredicates;
   modelGeneration?: ModelsAsDataLanguageModelGeneration;
+  autoModelGeneration?: ModelsAsDataLanguageAutoModelGeneration;
   accessPathSuggestions?: ModelsAsDataLanguageAccessPathSuggestions;
   /**
    * Returns the list of valid arguments that can be selected for the given method.

extensions/ql-vscode/src/model-editor/languages/python/access-paths.ts

@@ -0,0 +1,119 @@
+import { parseAccessPathTokens } from "../../shared/access-paths";
+import type { MethodDefinition } from "../../method";
+import { EndpointType } from "../../method";
+
+const memberTokenRegex = /^Member\[(.+)]$/;
+
+export function parsePythonAccessPath(path: string): {
+  typeName: string;
+  methodName: string;
+  endpointType: EndpointType;
+  path: string;
+} {
+  const tokens = parseAccessPathTokens(path);
+
+  if (tokens.length === 0) {
+    return {
+      typeName: "",
+      methodName: "",
+      endpointType: EndpointType.Method,
+      path: "",
+    };
+  }
+
+  const typeParts = [];
+  let endpointType = EndpointType.Function;
+
+  let remainingTokens: typeof tokens = [];
+
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    const memberMatch = token.text.match(memberTokenRegex);
+    if (memberMatch) {
+      typeParts.push(memberMatch[1]);
+    } else if (token.text === "Instance") {
+      endpointType = EndpointType.Method;
+    } else {
+      remainingTokens = tokens.slice(i);
+      break;
+    }
+  }
+
+  const methodName = typeParts.pop() ?? "";
+  const typeName = typeParts.join(".");
+  const remainingPath = remainingTokens.map((token) => token.text).join(".");
+
+  return {
+    methodName,
+    typeName,
+    endpointType,
+    path: remainingPath,
+  };
+}
+
+export function pythonMethodSignature(typeName: string, methodName: string) {
+  return `${typeName}#${methodName}`;
+}
+
+function pythonTypePath(typeName: string) {
+  if (typeName === "") {
+    return "";
+  }
+
+  return typeName
+    .split(".")
+    .map((part) => `Member[${part}]`)
+    .join(".");
+}
+
+export function pythonMethodPath(
+  typeName: string,
+  methodName: string,
+  endpointType: EndpointType,
+) {
+  if (methodName === "") {
+    return pythonTypePath(typeName);
+  }
+
+  const typePath = pythonTypePath(typeName);
+
+  let result = typePath;
+  if (typePath !== "" && endpointType === EndpointType.Method) {
+    result += ".Instance";
+  }
+
+  if (result !== "") {
+    result += ".";
+  }
+
+  result += `Member[${methodName}]`;
+
+  return result;
+}
+
+export function pythonPath(
+  typeName: string,
+  methodName: string,
+  endpointType: EndpointType,
+  path: string,
+) {
+  const methodPath = pythonMethodPath(typeName, methodName, endpointType);
+  if (methodPath === "") {
+    return path;
+  }
+
+  if (path === "") {
+    return methodPath;
+  }
+
+  return `${methodPath}.${path}`;
+}
+
+export function pythonEndpointType(
+  method: Omit<MethodDefinition, "endpointType">,
+): EndpointType {
+  if (method.methodParameters.startsWith("(self,")) {
+    return EndpointType.Method;
+  }
+  return EndpointType.Function;
+}

extensions/ql-vscode/src/model-editor/languages/python/index.ts

@@ -0,0 +1,207 @@
+import type { ModelsAsDataLanguage } from "../models-as-data";
+import { sharedExtensiblePredicates, sharedKinds } from "../shared";
+import { Mode } from "../../shared/mode";
+import type { MethodArgument } from "../../method";
+import { EndpointType, getArgumentsList } from "../../method";
+import {
+  parsePythonAccessPath,
+  pythonEndpointType,
+  pythonMethodPath,
+  pythonMethodSignature,
+  pythonPath,
+} from "./access-paths";
+
+export const python: ModelsAsDataLanguage = {
+  availableModes: [Mode.Framework],
+  createMethodSignature: ({ typeName, methodName }) =>
+    `${typeName}#${methodName}`,
+  endpointTypeForEndpoint: (method) => pythonEndpointType(method),
+  predicates: {
+    source: {
+      extensiblePredicate: sharedExtensiblePredicates.source,
+      supportedKinds: sharedKinds.source,
+      supportedEndpointTypes: [EndpointType.Method, EndpointType.Function],
+      // extensible predicate sourceModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        pythonPath(
+          method.typeName,
+          method.methodName,
+          method.endpointType,
+          method.output,
+        ),
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const {
+          typeName,
+          methodName,
+          endpointType,
+          path: output,
+        } = parsePythonAccessPath(row[1] as string);
+        return {
+          type: "source",
+          output,
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    sink: {
+      extensiblePredicate: sharedExtensiblePredicates.sink,
+      supportedKinds: sharedKinds.sink,
+      supportedEndpointTypes: [EndpointType.Method, EndpointType.Function],
+      // extensible predicate sinkModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => {
+        return [
+          method.packageName,
+          pythonPath(
+            method.typeName,
+            method.methodName,
+            method.endpointType,
+            method.input,
+          ),
+          method.kind,
+        ];
+      },
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const {
+          typeName,
+          methodName,
+          endpointType,
+          path: input,
+        } = parsePythonAccessPath(row[1] as string);
+        return {
+          type: "sink",
+          input,
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    summary: {
+      extensiblePredicate: sharedExtensiblePredicates.summary,
+      supportedKinds: sharedKinds.summary,
+      supportedEndpointTypes: [EndpointType.Method, EndpointType.Function],
+      // extensible predicate summaryModel(
+      //   string type, string path, string input, string output, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        pythonMethodPath(
+          method.typeName,
+          method.methodName,
+          method.endpointType,
+        ),
+        method.input,
+        method.output,
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const { typeName, methodName, endpointType, path } =
+          parsePythonAccessPath(row[1] as string);
+        if (path !== "") {
+          throw new Error("Summary path must be a method");
+        }
+        return {
+          type: "summary",
+          input: row[2] as string,
+          output: row[3] as string,
+          kind: row[4] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    neutral: {
+      extensiblePredicate: sharedExtensiblePredicates.neutral,
+      supportedKinds: sharedKinds.neutral,
+      // extensible predicate neutralModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        pythonMethodPath(
+          method.typeName,
+          method.methodName,
+          method.endpointType,
+        ),
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const packageName = row[0] as string;
+        const { typeName, methodName, endpointType, path } =
+          parsePythonAccessPath(row[1] as string);
+        if (path !== "") {
+          throw new Error("Neutral path must be a method");
+        }
+        return {
+          type: "neutral",
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: pythonMethodSignature(typeName, methodName),
+          endpointType,
+          packageName,
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+  },
+  getArgumentOptions: (method) => {
+    // Argument and Parameter are equivalent in Python, but we'll use Argument in the model editor
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => {
+        if (argument.endsWith(":")) {
+          return {
+            path: `Argument[${argument}]`,
+            label: `Argument[${argument}]`,
+          };
+        }
+
+        return {
+          path: `Argument[${index}]`,
+          label: `Argument[${index}]: ${argument}`,
+        };
+      },
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[self]",
+          label: "Argument[self]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[self]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[self]",
+    };
+  },
+};

extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts

@@ -1,6 +1,9 @@
 import type { BaseLogger } from "../../../common/logging";
 import type { DecodedBqrs } from "../../../common/bqrs-cli-types";
-import type { ModelsAsDataLanguage } from "../models-as-data";
+import type {
+  GenerationContext,
+  ModelsAsDataLanguage,
+} from "../models-as-data";
 import type { ModeledMethod } from "../../modeled-method";
 import type { DataTuple } from "../../model-extension-file";
 
@@ -9,10 +12,21 @@ export function parseGenerateModelResults(
   bqrs: DecodedBqrs,
   modelsAsDataLanguage: ModelsAsDataLanguage,
   logger: BaseLogger,
+  { config }: GenerationContext,
 ): ModeledMethod[] {
   const modeledMethods: ModeledMethod[] = [];
 
   for (const resultSetName in bqrs) {
+    if (
+      resultSetName ===
+        modelsAsDataLanguage.predicates.type?.extensiblePredicate &&
+      !config.showTypeModels
+    ) {
+      // Don't load generated type results when type models are hidden. These are already
+      // automatically generated on start-up.
+      continue;
+    }
+
     const definition = Object.values(modelsAsDataLanguage.predicates).find(
       (definition) => definition.extensiblePredicate === resultSetName,
     );

extensions/ql-vscode/src/model-editor/languages/ruby/index.ts

@@ -169,14 +169,50 @@ export const ruby: ModelsAsDataLanguage = {
           methodParameters: "",
         };
       },
+      isHidden: ({ config }) => !config.showTypeModels,
     },
   },
   modelGeneration: {
-    queryConstraints: {
-      "query path": "queries/modeling/GenerateModel.ql",
-    },
+    queryConstraints: (mode) => ({
+      kind: "table",
+      "tags contain all": ["modeleditor", "generate-model", modeTag(mode)],
+    }),
     parseResults: parseGenerateModelResults,
   },
+  autoModelGeneration: {
+    queryConstraints: (mode) => ({
+      kind: "table",
+      "tags contain all": ["modeleditor", "generate-model", modeTag(mode)],
+    }),
+    parseResultsToYaml: (_queryPath, bqrs, modelsAsDataLanguage) => {
+      const typePredicate = modelsAsDataLanguage.predicates.type;
+      if (!typePredicate) {
+        throw new Error("Type predicate not found");
+      }
+
+      const typeTuples = bqrs[typePredicate.extensiblePredicate];
+      if (!typeTuples) {
+        return [];
+      }
+
+      return [
+        {
+          addsTo: {
+            pack: "codeql/ruby-all",
+            extensible: typePredicate.extensiblePredicate,
+          },
+          data: typeTuples.tuples.filter((tuple): tuple is string[] => {
+            return (
+              tuple.filter((x) => typeof x === "string").length === tuple.length
+            );
+          }),
+        },
+      ];
+    },
+    // Only enabled for framework mode when type models are hidden
+    enabled: ({ mode, config }) =>
+      mode === Mode.Framework && !config.showTypeModels,
+  },
   accessPathSuggestions: {
     queryConstraints: (mode) => ({
       kind: "table",

extensions/ql-vscode/src/model-editor/languages/static/index.ts

@@ -141,9 +141,9 @@ export const staticLanguage: ModelsAsDataLanguage = {
     },
   },
   modelGeneration: {
-    queryConstraints: {
+    queryConstraints: () => ({
       "tags contain": ["modelgenerator"],
-    },
+    }),
     filterQueries: filterFlowModelQueries,
     parseResults: parseFlowModelResults,
   },

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts

@@ -15,6 +15,7 @@ import type { DatabaseItem } from "../../databases/local-databases";
 import type { ModelingEvents } from "../modeling-events";
 import type { QueryLanguage } from "../../common/query-language";
 import { tryGetQueryLanguage } from "../../common/query-language";
+import { createModelConfig } from "../languages";
 
 export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   ToMethodModelingMessage,
@@ -46,6 +47,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
       t: "setMethodModelingPanelViewState",
       viewState: {
         language: this.language,
+        modelConfig: createModelConfig(this.modelConfig),
       },
     });
   }
@@ -82,6 +84,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
           modeledMethods: selectedMethod.modeledMethods,
           isModified: selectedMethod.isModified,
           isInProgress: selectedMethod.isInProgress,
+          processedByAutoModel: selectedMethod.processedByAutoModel,
         });
       }
 
@@ -123,10 +126,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
           this.databaseItem,
           msg.methodSignature,
           msg.modeledMethods,
-        );
-        this.modelingStore.addModifiedMethod(
-          this.databaseItem,
-          msg.methodSignature,
+          true,
         );
         break;
       }
@@ -161,7 +161,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
   private registerToModelingEvents(): void {
     this.push(
-      this.modelingEvents.onModeledMethodsChanged(async (e) => {
+      this.modelingEvents.onModeledAndModifiedMethodsChanged(async (e) => {
         if (this.webviewView && e.isActiveDb && this.method) {
           const modeledMethods = e.modeledMethods[this.method.signature];
           if (modeledMethods) {
@@ -171,17 +171,10 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
               modeledMethods,
             });
           }
-        }
-      }),
-    );
 
-    this.push(
-      this.modelingEvents.onModifiedMethodsChanged(async (e) => {
-        if (this.webviewView && e.isActiveDb && this.method) {
-          const isModified = e.modifiedMethods.has(this.method.signature);
           await this.postMessage({
             t: "setMethodModified",
-            isModified,
+            isModified: e.modifiedMethodSignatures.has(this.method.signature),
           });
         }
       }),
@@ -200,6 +193,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
             modeledMethods: e.modeledMethods,
             isModified: e.isModified,
             isInProgress: e.isInProgress,
+            processedByAutoModel: e.processedByAutoModel,
           });
         }
       }),
@@ -248,6 +242,21 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         }
       }),
     );
+
+    this.push(
+      this.modelingEvents.onProcessedByAutoModelMethodsChanged(async (e) => {
+        if (this.method && this.databaseItem) {
+          const dbUri = this.databaseItem.databaseUri.toString();
+          if (e.dbUri === dbUri) {
+            const processedByAutoModel = e.methods.has(this.method.signature);
+            await this.postMessage({
+              t: "setProcessedByAutoModel",
+              processedByAutoModel,
+            });
+          }
+        }
+      }),
+    );
   }
 
   private registerToModelConfigEvents(): void {

extensions/ql-vscode/src/model-editor/method.ts

@@ -28,6 +28,7 @@ export enum EndpointType {
   Class = "class",
   Method = "method",
   Constructor = "constructor",
+  Function = "function",
 }
 
 export interface MethodDefinition {

extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-data-provider.ts

@@ -16,7 +16,7 @@ import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "../shared/hide-modeled-metho
 import { getModelingStatus } from "../shared/modeling-status";
 import { assertNever } from "../../common/helpers-pure";
 import type { ModeledMethod } from "../modeled-method";
-import { groupMethods, sortGroupNames, sortMethods } from "../shared/sorting";
+import { groupMethods, sortGroupNames } from "../shared/sorting";
 import type { Mode } from "../shared/mode";
 import { INITIAL_MODE } from "../shared/mode";
 import type { UrlValueResolvable } from "../../common/raw-result-types";
@@ -73,9 +73,7 @@ export class MethodsUsageDataProvider
       this.modifiedMethodSignatures !== modifiedMethodSignatures
     ) {
       this.methods = methods;
-      this.sortedTreeItems = createTreeItems(
-        sortMethodsInGroups(methods, mode),
-      );
+      this.sortedTreeItems = createTreeItems(createGroups(methods, mode));
       this.databaseItem = databaseItem;
       this.sourceLocationPrefix =
         await this.databaseItem.getSourceLocationPrefix(this.cliServer);
@@ -246,16 +244,9 @@ function urlValueResolvablesAreEqual(
   return false;
 }
 
-function sortMethodsInGroups(methods: readonly Method[], mode: Mode): Method[] {
+function createGroups(methods: readonly Method[], mode: Mode): Method[] {
   const grouped = groupMethods(methods, mode);
-
-  const sortedGroupNames = sortGroupNames(grouped);
-
-  return sortedGroupNames.flatMap((groupName) => {
-    const group = grouped[groupName];
-
-    return sortMethods(group);
-  });
+  return sortGroupNames(grouped).flatMap((groupName) => grouped[groupName]);
 }
 
 function createTreeItems(methods: readonly Method[]): MethodTreeViewItem[] {

extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-panel.ts

@@ -102,7 +102,7 @@ export class MethodsUsagePanel extends DisposableObject {
     );
 
     this.push(
-      this.modelingEvents.onModifiedMethodsChanged(async (event) => {
+      this.modelingEvents.onModeledAndModifiedMethodsChanged(async (event) => {
         if (event.isActiveDb) {
           await this.handleStateChangeEvent();
         }

extensions/ql-vscode/src/model-editor/model-editor-module.ts

@@ -1,7 +1,6 @@
 import { ModelEditorView } from "./model-editor-view";
 import type { ModelEditorCommands } from "../common/commands";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
-import { CliVersionConstraint } from "../codeql-cli/cli";
 import type { QueryRunner } from "../query-server";
 import type {
   DatabaseItem,
@@ -32,6 +31,7 @@ import { getModelsAsDataLanguage } from "./languages";
 import { INITIAL_MODE } from "./shared/mode";
 import { isSupportedLanguage } from "./supported-languages";
 import { DefaultNotifier, checkConsistency } from "./consistency-check";
+import type { VariantAnalysisManager } from "../variant-analysis/variant-analysis-manager";
 
 export class ModelEditorModule extends DisposableObject {
   private readonly queryStorageDir: string;
@@ -44,6 +44,7 @@ export class ModelEditorModule extends DisposableObject {
   private constructor(
     private readonly app: App,
     private readonly databaseManager: DatabaseManager,
+    private readonly variantAnalysisManager: VariantAnalysisManager,
     private readonly cliServer: CodeQLCliServer,
     private readonly queryRunner: QueryRunner,
     baseQueryStorageDir: string,
@@ -66,6 +67,7 @@ export class ModelEditorModule extends DisposableObject {
   public static async initialize(
     app: App,
     databaseManager: DatabaseManager,
+    variantAnalysisManager: VariantAnalysisManager,
     cliServer: CodeQLCliServer,
     queryRunner: QueryRunner,
     queryStorageDir: string,
@@ -73,6 +75,7 @@ export class ModelEditorModule extends DisposableObject {
     const modelEditorModule = new ModelEditorModule(
       app,
       databaseManager,
+      variantAnalysisManager,
       cliServer,
       queryRunner,
       queryStorageDir,
@@ -169,14 +172,6 @@ export class ModelEditorModule extends DisposableObject {
         async (progress, token) => {
           const maxStep = 4;
 
-          if (!(await this.cliServer.cliConstraints.supportsQlpacksKind())) {
-            void showAndLogErrorMessage(
-              this.app.logger,
-              `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND.format()} or later.`,
-            );
-            return;
-          }
-
           const modelFile = await pickExtensionPack(
             this.cliServer,
             db,
@@ -214,6 +209,7 @@ export class ModelEditorModule extends DisposableObject {
             queryDir,
             language,
             this.modelConfig,
+            initialMode,
           );
           if (!success) {
             await cleanupQueryDir();
@@ -248,6 +244,7 @@ export class ModelEditorModule extends DisposableObject {
             this.modelingEvents,
             this.modelConfig,
             this.databaseManager,
+            this.variantAnalysisManager,
             this.cliServer,
             this.queryRunner,
             this.queryStorageDir,

extensions/ql-vscode/src/model-editor/model-editor-queries-setup.ts

@@ -9,7 +9,7 @@ import {
 } from "./model-editor-queries";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import type { ModelConfig } from "../config";
-import { Mode } from "./shared/mode";
+import type { Mode } from "./shared/mode";
 import type { NotificationLogger } from "../common/logging";
 
 /**
@@ -31,6 +31,7 @@ import type { NotificationLogger } from "../common/logging";
  * @param queryDir The directory to set up.
  * @param language The language to use for the queries.
  * @param modelConfig The model config to use.
+ * @param initialMode The initial mode to use to check the existence of the queries.
  * @returns true if the setup was successful, false otherwise.
  */
 export async function setUpPack(
@@ -39,6 +40,7 @@ export async function setUpPack(
   queryDir: string,
   language: QueryLanguage,
   modelConfig: ModelConfig,
+  initialMode: Mode,
 ): Promise<boolean> {
   // Download the required query packs
   await cliServer.packDownload([`codeql/${language}-queries`]);
@@ -48,7 +50,7 @@ export async function setUpPack(
   const applicationModeQuery = await resolveEndpointsQuery(
     cliServer,
     language,
-    Mode.Application,
+    initialMode,
     [],
     [],
   );

extensions/ql-vscode/src/model-editor/model-editor-view.ts

@@ -41,7 +41,11 @@ import type { ModeledMethod } from "./modeled-method";
 import type { ExtensionPack } from "./shared/extension-pack";
 import type { ModelConfigListener } from "../config";
 import { Mode } from "./shared/mode";
-import { loadModeledMethods, saveModeledMethods } from "./modeled-method-fs";
+import {
+  GENERATED_MODELS_SUFFIX,
+  loadModeledMethods,
+  saveModeledMethods,
+} from "./modeled-method-fs";
 import { pickExtensionPack } from "./extension-pack-picker";
 import type { QueryLanguage } from "../common/query-language";
 import { getLanguageDisplayName } from "../common/query-language";
@@ -50,20 +54,31 @@ import { telemetryListener } from "../common/vscode/telemetry";
 import type { ModelingStore } from "./modeling-store";
 import type { ModelingEvents } from "./modeling-events";
 import type { ModelsAsDataLanguage } from "./languages";
-import { getModelsAsDataLanguage } from "./languages";
+import { createModelConfig, getModelsAsDataLanguage } from "./languages";
 import { runGenerateQueries } from "./generate";
 import { ResponseError } from "vscode-jsonrpc";
 import { LSPErrorCodes } from "vscode-languageclient";
 import type { AccessPathSuggestionOptions } from "./suggestions";
 import { runSuggestionsQuery } from "./suggestion-queries";
 import { parseAccessPathSuggestionRowsToOptions } from "./suggestions-bqrs";
+import { ModelEvaluator } from "./model-evaluator";
+import type { ModelEvaluationRunState } from "./shared/model-evaluation-run-state";
+import type { VariantAnalysisManager } from "../variant-analysis/variant-analysis-manager";
+import type { ModelExtensionFile } from "./model-extension-file";
+import { modelExtensionFileToYaml } from "./yaml";
+import { outputFile } from "fs-extra";
+import { join } from "path";
 
 export class ModelEditorView extends AbstractWebview<
   ToModelEditorMessage,
   FromModelEditorMessage
 > {
   private readonly autoModeler: AutoModeler;
+  private readonly modelEvaluator: ModelEvaluator;
   private readonly languageDefinition: ModelsAsDataLanguage;
+  // Cancellation token source that can be used for passing into long-running operations. Should only
+  // be cancelled when the view is closed
+  private readonly cancellationTokenSource = new CancellationTokenSource();
 
   public constructor(
     protected readonly app: App,
@@ -71,6 +86,7 @@ export class ModelEditorView extends AbstractWebview<
     private readonly modelingEvents: ModelingEvents,
     private readonly modelConfig: ModelConfigListener,
     private readonly databaseManager: DatabaseManager,
+    private readonly variantAnalysisManager: VariantAnalysisManager,
     private readonly cliServer: CodeQLCliServer,
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
@@ -83,6 +99,12 @@ export class ModelEditorView extends AbstractWebview<
   ) {
     super(app);
 
+    this.push({
+      dispose: () => {
+        this.cancellationTokenSource.cancel();
+      },
+    });
+
     this.modelingStore.initializeStateForDb(databaseItem, initialMode);
     this.registerToModelingEvents();
     this.registerToModelConfigEvents();
@@ -101,6 +123,18 @@ export class ModelEditorView extends AbstractWebview<
       },
     );
     this.languageDefinition = getModelsAsDataLanguage(language);
+
+    this.modelEvaluator = new ModelEvaluator(
+      this.app.logger,
+      this.cliServer,
+      modelingStore,
+      modelingEvents,
+      this.variantAnalysisManager,
+      databaseItem,
+      language,
+      this.updateModelEvaluationRun.bind(this),
+    );
+    this.push(this.modelEvaluator);
   }
 
   public async openView() {
@@ -238,6 +272,7 @@ export class ModelEditorView extends AbstractWebview<
                 modeledMethods,
                 mode,
                 this.cliServer,
+                this.modelConfig,
                 this.app.logger,
               );
 
@@ -262,6 +297,8 @@ export class ModelEditorView extends AbstractWebview<
             Object.keys(modeledMethods),
           );
 
+          this.modelingStore.updateMethodSorting(this.databaseItem);
+
           void telemetryListener?.sendUIInteraction(
             "model-editor-save-modeled-methods",
           );
@@ -337,6 +374,12 @@ export class ModelEditorView extends AbstractWebview<
         this.setModeledMethods(msg.methodSignature, msg.modeledMethods);
         break;
       }
+      case "startModelEvaluation":
+        await this.modelEvaluator.startEvaluation();
+        break;
+      case "stopModelEvaluation":
+        await this.modelEvaluator.stopEvaluation();
+        break;
       case "telemetry":
         telemetryListener?.sendUIInteraction(msg.action);
         break;
@@ -361,6 +404,8 @@ export class ModelEditorView extends AbstractWebview<
       this.setViewState(),
       withProgress((progress, token) => this.loadMethods(progress, token), {
         cancellable: true,
+      }).then(async () => {
+        await this.generateModeledMethodsOnStartup();
       }),
       this.loadExistingModeledMethods(),
       // Only load access path suggestions if the feature is enabled
@@ -402,6 +447,8 @@ export class ModelEditorView extends AbstractWebview<
     const showLlmButton =
       this.databaseItem.language === "java" && this.modelConfig.llmGeneration;
 
+    const showEvaluationUi = this.modelConfig.modelEvaluation;
+
     const sourceArchiveAvailable =
       this.databaseItem.hasSourceArchiveInExplorer();
 
@@ -416,9 +463,11 @@ export class ModelEditorView extends AbstractWebview<
         language: this.language,
         showGenerateButton,
         showLlmButton,
+        showEvaluationUi,
         mode: this.modelingStore.getMode(this.databaseItem),
         showModeSwitchButton,
         sourceArchiveAvailable,
+        modelConfig: createModelConfig(this.modelConfig),
       },
     });
   }
@@ -443,6 +492,7 @@ export class ModelEditorView extends AbstractWebview<
         this.extensionPack,
         this.language,
         this.cliServer,
+        this.modelConfig,
         this.app.logger,
       );
       this.modelingStore.setModeledMethods(this.databaseItem, modeledMethods);
@@ -462,7 +512,7 @@ export class ModelEditorView extends AbstractWebview<
 
     try {
       if (!token) {
-        token = new CancellationTokenSource().token;
+        token = this.cancellationTokenSource.token;
       }
       const queryResult = await runModelEditorQueries(mode, {
         cliServer: this.cliServer,
@@ -513,8 +563,6 @@ export class ModelEditorView extends AbstractWebview<
   protected async loadAccessPathSuggestions(
     progress: ProgressCallback,
   ): Promise<void> {
-    const tokenSource = new CancellationTokenSource();
-
     const mode = this.modelingStore.getMode(this.databaseItem);
 
     const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
@@ -537,7 +585,7 @@ export class ModelEditorView extends AbstractWebview<
         queryStorageDir: this.queryStorageDir,
         databaseItem: this.databaseItem,
         progress,
-        token: tokenSource.token,
+        token: this.cancellationTokenSource.token,
         logger: this.app.logger,
       });
 
@@ -568,8 +616,6 @@ export class ModelEditorView extends AbstractWebview<
   protected async generateModeledMethods(): Promise<void> {
     await withProgress(
       async (progress) => {
-        const tokenSource = new CancellationTokenSource();
-
         const mode = this.modelingStore.getMode(this.databaseItem);
 
         const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
@@ -610,16 +656,20 @@ export class ModelEditorView extends AbstractWebview<
 
         try {
           await runGenerateQueries({
-            queryConstraints: modelGeneration.queryConstraints,
+            queryConstraints: modelGeneration.queryConstraints(mode),
             filterQueries: modelGeneration.filterQueries,
-            parseResults: (queryPath, results) =>
-              modelGeneration.parseResults(
+            onResults: async (queryPath, results) => {
+              const modeledMethods = modelGeneration.parseResults(
                 queryPath,
                 results,
                 modelsAsDataLanguage,
                 this.app.logger,
-              ),
-            onResults: async (modeledMethods) => {
+                {
+                  mode,
+                  config: this.modelConfig,
+                },
+              );
+
               this.addModeledMethodsFromArray(modeledMethods);
             },
             cliServer: this.cliServer,
@@ -627,7 +677,7 @@ export class ModelEditorView extends AbstractWebview<
             queryStorageDir: this.queryStorageDir,
             databaseItem: addedDatabase ?? this.databaseItem,
             progress,
-            token: tokenSource.token,
+            token: this.cancellationTokenSource.token,
           });
         } catch (e: unknown) {
           void showAndLogExceptionWithTelemetry(
@@ -643,6 +693,91 @@ export class ModelEditorView extends AbstractWebview<
     );
   }
 
+  protected async generateModeledMethodsOnStartup(): Promise<void> {
+    const mode = this.modelingStore.getMode(this.databaseItem);
+    const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
+    const autoModelGeneration = modelsAsDataLanguage.autoModelGeneration;
+
+    if (autoModelGeneration === undefined) {
+      return;
+    }
+
+    if (
+      autoModelGeneration.enabled &&
+      !autoModelGeneration.enabled({ mode, config: this.modelConfig })
+    ) {
+      return;
+    }
+
+    await withProgress(
+      async (progress) => {
+        progress({
+          step: 0,
+          maxStep: 4000,
+          message: "Generating models",
+        });
+
+        const extensionFile: ModelExtensionFile = {
+          extensions: [],
+        };
+
+        try {
+          await runGenerateQueries({
+            queryConstraints: autoModelGeneration.queryConstraints(mode),
+            filterQueries: autoModelGeneration.filterQueries,
+            onResults: (queryPath, results) => {
+              const extensions = autoModelGeneration.parseResultsToYaml(
+                queryPath,
+                results,
+                modelsAsDataLanguage,
+                this.app.logger,
+              );
+
+              extensionFile.extensions.push(...extensions);
+            },
+            cliServer: this.cliServer,
+            queryRunner: this.queryRunner,
+            queryStorageDir: this.queryStorageDir,
+            databaseItem: this.databaseItem,
+            progress,
+            token: this.cancellationTokenSource.token,
+          });
+        } catch (e: unknown) {
+          void showAndLogExceptionWithTelemetry(
+            this.app.logger,
+            this.app.telemetry,
+            redactableError(
+              asError(e),
+            )`Failed to auto-run generating models: ${getErrorMessage(e)}`,
+          );
+          return;
+        }
+
+        progress({
+          step: 4000,
+          maxStep: 4000,
+          message: "Saving generated models",
+        });
+
+        const fileContents = `# This file was automatically generated from ${this.databaseItem.name}. Manual changes will not persist.\n\n${modelExtensionFileToYaml(extensionFile)}`;
+        const filePath = join(
+          this.extensionPack.path,
+          "models",
+          `${this.language}${GENERATED_MODELS_SUFFIX}`,
+        );
+
+        await outputFile(filePath, fileContents);
+
+        void this.app.logger.log(`Saved generated model file to ${filePath}`);
+      },
+      {
+        cancellable: false,
+        location: ProgressLocation.Window,
+        title: "Generating models",
+      },
+    );
+  }
+
   private async generateModeledMethodsFromLlm(
     packageName: string,
     methodSignatures: string[],
@@ -655,11 +790,17 @@ export class ModelEditorView extends AbstractWebview<
       this.databaseItem,
       methodSignatures,
     );
+    const processedByAutoModelMethods =
+      this.modelingStore.getProcessedByAutoModelMethods(
+        this.databaseItem,
+        methodSignatures,
+      );
     const mode = this.modelingStore.getMode(this.databaseItem);
     await this.autoModeler.startModeling(
       packageName,
       methods,
       modeledMethods,
+      processedByAutoModelMethods,
       mode,
     );
   }
@@ -704,6 +845,7 @@ export class ModelEditorView extends AbstractWebview<
         this.modelingEvents,
         this.modelConfig,
         this.databaseManager,
+        this.variantAnalysisManager,
         this.cliServer,
         this.queryRunner,
         this.queryStorageDir,
@@ -803,22 +945,12 @@ export class ModelEditorView extends AbstractWebview<
     );
 
     this.push(
-      this.modelingEvents.onModeledMethodsChanged(async (event) => {
+      this.modelingEvents.onModeledAndModifiedMethodsChanged(async (event) => {
         if (event.dbUri === this.databaseItem.databaseUri.toString()) {
           await this.postMessage({
-            t: "setModeledMethods",
+            t: "setModeledAndModifiedMethods",
             methods: event.modeledMethods,
-          });
-        }
-      }),
-    );
-
-    this.push(
-      this.modelingEvents.onModifiedMethodsChanged(async (event) => {
-        if (event.dbUri === this.databaseItem.databaseUri.toString()) {
-          await this.postMessage({
-            t: "setModifiedMethods",
-            methodSignatures: [...event.modifiedMethods],
+            modifiedMethodSignatures: [...event.modifiedMethodSignatures],
           });
         }
       }),
@@ -835,6 +967,19 @@ export class ModelEditorView extends AbstractWebview<
       }),
     );
 
+    this.push(
+      this.modelingEvents.onProcessedByAutoModelMethodsChanged(
+        async (event) => {
+          if (event.dbUri === this.databaseItem.databaseUri.toString()) {
+            await this.postMessage({
+              t: "setProcessedByAutoModelMethods",
+              methods: Array.from(event.methods),
+            });
+          }
+        },
+      ),
+    );
+
     this.push(
       this.modelingEvents.onRevealInModelEditor(async (event) => {
         if (event.dbUri === this.databaseItem.databaseUri.toString()) {
@@ -861,11 +1006,10 @@ export class ModelEditorView extends AbstractWebview<
   }
 
   private addModeledMethods(modeledMethods: Record<string, ModeledMethod[]>) {
-    this.modelingStore.addModeledMethods(this.databaseItem, modeledMethods);
-
-    this.modelingStore.addModifiedMethods(
+    this.modelingStore.addModeledMethods(
       this.databaseItem,
-      new Set(Object.keys(modeledMethods)),
+      modeledMethods,
+      true,
     );
   }
 
@@ -888,7 +1032,14 @@ export class ModelEditorView extends AbstractWebview<
       this.databaseItem,
       signature,
       methods,
+      true,
     );
-    this.modelingStore.addModifiedMethod(this.databaseItem, signature);
+  }
+
+  private async updateModelEvaluationRun(run: ModelEvaluationRunState) {
+    await this.postMessage({
+      t: "setModelEvaluationRun",
+      run,
+    });
   }
 }

extensions/ql-vscode/src/model-editor/model-evaluation-run.ts

@@ -0,0 +1,4 @@
+export interface ModelEvaluationRun {
+  isPreparing: boolean;
+  variantAnalysisId: number | undefined;
+}

extensions/ql-vscode/src/model-editor/model-evaluator.ts

@@ -0,0 +1,145 @@
+import type { ModelingStore } from "./modeling-store";
+import type { ModelingEvents } from "./modeling-events";
+import type { DatabaseItem } from "../databases/local-databases";
+import type { ModelEvaluationRun } from "./model-evaluation-run";
+import { DisposableObject } from "../common/disposable-object";
+import type { ModelEvaluationRunState } from "./shared/model-evaluation-run-state";
+import type { BaseLogger } from "../common/logging";
+import type { CodeQLCliServer } from "../codeql-cli/cli";
+import type { VariantAnalysisManager } from "../variant-analysis/variant-analysis-manager";
+import type { QueryLanguage } from "../common/query-language";
+import { resolveCodeScanningQueryPack } from "../variant-analysis/code-scanning-pack";
+import { withProgress } from "../common/vscode/progress";
+import type { VariantAnalysis } from "../variant-analysis/shared/variant-analysis";
+
+export class ModelEvaluator extends DisposableObject {
+  public constructor(
+    private readonly logger: BaseLogger,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly modelingStore: ModelingStore,
+    private readonly modelingEvents: ModelingEvents,
+    private readonly variantAnalysisManager: VariantAnalysisManager,
+    private readonly dbItem: DatabaseItem,
+    private readonly language: QueryLanguage,
+    private readonly updateView: (
+      run: ModelEvaluationRunState,
+    ) => Promise<void>,
+  ) {
+    super();
+
+    this.registerToModelingEvents();
+  }
+
+  public async startEvaluation() {
+    // Update store with evaluation run status
+    const evaluationRun: ModelEvaluationRun = {
+      isPreparing: true,
+      variantAnalysisId: undefined,
+    };
+    this.modelingStore.updateModelEvaluationRun(this.dbItem, evaluationRun);
+
+    // Build pack
+    const qlPack = await resolveCodeScanningQueryPack(
+      this.logger,
+      this.cliServer,
+      this.language,
+    );
+
+    if (!qlPack) {
+      this.modelingStore.updateModelEvaluationRun(this.dbItem, undefined);
+      throw new Error("Unable to trigger evaluation run");
+    }
+
+    // Submit variant analysis and monitor progress
+    return withProgress(
+      async (progress, token) => {
+        let variantAnalysisId: number | undefined = undefined;
+        try {
+          variantAnalysisId =
+            await this.variantAnalysisManager.runVariantAnalysis(
+              qlPack,
+              progress,
+              token,
+            );
+        } catch (e) {
+          this.modelingStore.updateModelEvaluationRun(this.dbItem, undefined);
+          throw e;
+        }
+
+        if (variantAnalysisId) {
+          this.monitorVariantAnalysis(variantAnalysisId);
+        } else {
+          this.modelingStore.updateModelEvaluationRun(this.dbItem, undefined);
+          throw new Error(
+            "Unable to trigger variant analysis for evaluation run",
+          );
+        }
+      },
+      {
+        title: "Run Variant Analysis",
+        cancellable: true,
+      },
+    );
+  }
+
+  public async stopEvaluation() {
+    // For now just update the store.
+    // This will be fleshed out in the near future.
+    const evaluationRun: ModelEvaluationRun = {
+      isPreparing: false,
+      variantAnalysisId: undefined,
+    };
+    this.modelingStore.updateModelEvaluationRun(this.dbItem, evaluationRun);
+  }
+
+  private registerToModelingEvents() {
+    this.push(
+      this.modelingEvents.onModelEvaluationRunChanged(async (event) => {
+        if (event.dbUri === this.dbItem.databaseUri.toString()) {
+          if (!event.evaluationRun) {
+            await this.updateView({
+              isPreparing: false,
+              variantAnalysis: undefined,
+            });
+          } else {
+            const variantAnalysis = await this.getVariantAnalysisForRun(
+              event.evaluationRun,
+            );
+            const run: ModelEvaluationRunState = {
+              isPreparing: event.evaluationRun.isPreparing,
+              variantAnalysis,
+            };
+            await this.updateView(run);
+          }
+        }
+      }),
+    );
+  }
+
+  private async getVariantAnalysisForRun(
+    evaluationRun: ModelEvaluationRun,
+  ): Promise<VariantAnalysis | undefined> {
+    if (evaluationRun.variantAnalysisId) {
+      return this.variantAnalysisManager.tryGetVariantAnalysis(
+        evaluationRun.variantAnalysisId,
+      );
+    }
+    return undefined;
+  }
+
+  private monitorVariantAnalysis(variantAnalysisId: number) {
+    this.push(
+      this.variantAnalysisManager.onVariantAnalysisStatusUpdated(
+        async (variantAnalysis) => {
+          // Make sure it's the variant analysis we're interested in
+          if (variantAnalysisId === variantAnalysis.id) {
+            await this.updateView({
+              isPreparing: false,
+              variantAnalysis,
+            });
+          }
+        },
+      ),
+    );
+  }
+}

extensions/ql-vscode/src/model-editor/model-extension-file.schema.json

@@ -8,36 +8,39 @@
         "extensions": {
           "type": "array",
           "items": {
-            "type": "object",
-            "properties": {
-              "addsTo": {
-                "type": "object",
-                "properties": {
-                  "pack": {
-                    "type": "string"
-                  },
-                  "extensible": {
-                    "type": "string"
-                  }
-                },
-                "required": ["pack", "extensible"]
-              },
-              "data": {
-                "type": "array",
-                "items": {
-                  "type": "array",
-                  "items": {
-                    "$ref": "#/definitions/DataTuple"
-                  }
-                }
-              }
-            },
-            "required": ["addsTo", "data"]
+            "$ref": "#/definitions/ModelExtension"
           }
         }
       },
       "required": ["extensions"]
     },
+    "ModelExtension": {
+      "type": "object",
+      "properties": {
+        "addsTo": {
+          "type": "object",
+          "properties": {
+            "pack": {
+              "type": "string"
+            },
+            "extensible": {
+              "type": "string"
+            }
+          },
+          "required": ["pack", "extensible"]
+        },
+        "data": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/DataTuple"
+            }
+          }
+        }
+      },
+      "required": ["addsTo", "data"]
+    },
     "DataTuple": {
       "type": ["boolean", "number", "string"]
     }

extensions/ql-vscode/src/model-editor/model-extension-file.ts

@@ -7,7 +7,7 @@ export type DataTuple = boolean | number | string;
 
 type DataRow = DataTuple[];
 
-type ModelExtension = {
+export type ModelExtension = {
   addsTo: ExtensibleReference;
   data: DataRow[];
 };

extensions/ql-vscode/src/model-editor/modeled-method-fs.ts

@@ -12,6 +12,9 @@ import { load as loadYaml } from "js-yaml";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import { pathsEqual } from "../common/files";
 import type { QueryLanguage } from "../common/query-language";
+import type { ModelConfig } from "./languages";
+
+export const GENERATED_MODELS_SUFFIX = ".model.generated.yml";
 
 export async function saveModeledMethods(
   extensionPack: ExtensionPack,
@@ -20,12 +23,14 @@ export async function saveModeledMethods(
   modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
   mode: Mode,
   cliServer: CodeQLCliServer,
+  modelConfig: ModelConfig,
   logger: NotificationLogger,
 ): Promise<void> {
   const existingModeledMethods = await loadModeledMethodFiles(
     extensionPack,
     language,
     cliServer,
+    modelConfig,
     logger,
   );
 
@@ -48,9 +53,14 @@ async function loadModeledMethodFiles(
   extensionPack: ExtensionPack,
   language: QueryLanguage,
   cliServer: CodeQLCliServer,
+  modelConfig: ModelConfig,
   logger: NotificationLogger,
 ): Promise<Record<string, Record<string, ModeledMethod[]>>> {
-  const modelFiles = await listModelFiles(extensionPack.path, cliServer);
+  const modelFiles = await listModelFiles(
+    extensionPack.path,
+    cliServer,
+    modelConfig,
+  );
 
   const modeledMethodsByFile: Record<
     string,
@@ -82,6 +92,7 @@ export async function loadModeledMethods(
   extensionPack: ExtensionPack,
   language: QueryLanguage,
   cliServer: CodeQLCliServer,
+  modelConfig: ModelConfig,
   logger: NotificationLogger,
 ): Promise<Record<string, ModeledMethod[]>> {
   const existingModeledMethods: Record<string, ModeledMethod[]> = {};
@@ -90,6 +101,7 @@ export async function loadModeledMethods(
     extensionPack,
     language,
     cliServer,
+    modelConfig,
     logger,
   );
   for (const modeledMethods of Object.values(modeledMethodsByFile)) {
@@ -108,6 +120,7 @@ export async function loadModeledMethods(
 export async function listModelFiles(
   extensionPackPath: string,
   cliServer: CodeQLCliServer,
+  modelConfig: ModelConfig,
 ): Promise<Set<string>> {
   const result = await cliServer.resolveExtensions(
     extensionPackPath,
@@ -118,6 +131,14 @@ export async function listModelFiles(
   for (const [path, extensions] of Object.entries(result.data)) {
     if (pathsEqual(path, extensionPackPath)) {
       for (const extension of extensions) {
+        // We only load generated models when type models are shown
+        if (
+          !modelConfig.showTypeModels &&
+          extension.file.endsWith(GENERATED_MODELS_SUFFIX)
+        ) {
+          continue;
+        }
+
         modelFiles.add(relative(extensionPackPath, extension.file));
       }
     }

extensions/ql-vscode/src/model-editor/modeled-method.ts

@@ -111,19 +111,27 @@ export function modeledMethodSupportsProvenance(
   );
 }
 
-export function isModelAccepted(
+export function isModelPending(
   modeledMethod: ModeledMethod | undefined,
   modelingStatus: ModelingStatus,
+  processedByAutoModel: boolean,
 ): boolean {
-  if (!modeledMethod) {
+  if (
+    (!modeledMethod || modeledMethod.type === "none") &&
+    processedByAutoModel
+  ) {
     return true;
   }
 
+  if (!modeledMethod) {
+    return false;
+  }
+
   return (
-    modelingStatus !== "unsaved" ||
-    modeledMethod.type === "none" ||
-    !modeledMethodSupportsProvenance(modeledMethod) ||
-    modeledMethod.provenance !== "ai-generated"
+    modelingStatus === "unsaved" &&
+    modeledMethod.type !== "none" &&
+    modeledMethodSupportsProvenance(modeledMethod) &&
+    modeledMethod.provenance === "ai-generated"
   );
 }
 

extensions/ql-vscode/src/model-editor/modeling-events.ts

@@ -3,6 +3,7 @@ import { DisposableObject } from "../common/disposable-object";
 import type { AppEvent, AppEventEmitter } from "../common/events";
 import type { DatabaseItem } from "../databases/local-databases";
 import type { Method, Usage } from "./method";
+import type { ModelEvaluationRun } from "./model-evaluation-run";
 import type { ModeledMethod } from "./modeled-method";
 import type { Mode } from "./shared/mode";
 
@@ -23,14 +24,9 @@ interface ModeChangedEvent {
   readonly isActiveDb: boolean;
 }
 
-interface ModeledMethodsChangedEvent {
+interface ModeledAndModifiedMethodsChangedEvent {
   readonly modeledMethods: Readonly<Record<string, ModeledMethod[]>>;
-  readonly dbUri: string;
-  readonly isActiveDb: boolean;
-}
-
-interface ModifiedMethodsChangedEvent {
-  readonly modifiedMethods: ReadonlySet<string>;
+  readonly modifiedMethodSignatures: ReadonlySet<string>;
   readonly dbUri: string;
   readonly isActiveDb: boolean;
 }
@@ -42,6 +38,7 @@ interface SelectedMethodChangedEvent {
   readonly modeledMethods: readonly ModeledMethod[];
   readonly isModified: boolean;
   readonly isInProgress: boolean;
+  readonly processedByAutoModel: boolean;
 }
 
 interface InProgressMethodsChangedEvent {
@@ -49,6 +46,16 @@ interface InProgressMethodsChangedEvent {
   readonly methods: ReadonlySet<string>;
 }
 
+interface ProcessedByAutoModelMethodsChangedEvent {
+  readonly dbUri: string;
+  readonly methods: ReadonlySet<string>;
+}
+
+interface ModelEvaluationRunChangedEvent {
+  readonly dbUri: string;
+  readonly evaluationRun: ModelEvaluationRun | undefined;
+}
+
 interface RevealInModelEditorEvent {
   dbUri: string;
   method: Method;
@@ -65,10 +72,11 @@ export class ModelingEvents extends DisposableObject {
   public readonly onMethodsChanged: AppEvent<MethodsChangedEvent>;
   public readonly onHideModeledMethodsChanged: AppEvent<HideModeledMethodsChangedEvent>;
   public readonly onModeChanged: AppEvent<ModeChangedEvent>;
-  public readonly onModeledMethodsChanged: AppEvent<ModeledMethodsChangedEvent>;
-  public readonly onModifiedMethodsChanged: AppEvent<ModifiedMethodsChangedEvent>;
+  public readonly onModeledAndModifiedMethodsChanged: AppEvent<ModeledAndModifiedMethodsChangedEvent>;
   public readonly onSelectedMethodChanged: AppEvent<SelectedMethodChangedEvent>;
   public readonly onInProgressMethodsChanged: AppEvent<InProgressMethodsChangedEvent>;
+  public readonly onProcessedByAutoModelMethodsChanged: AppEvent<ProcessedByAutoModelMethodsChangedEvent>;
+  public readonly onModelEvaluationRunChanged: AppEvent<ModelEvaluationRunChangedEvent>;
   public readonly onRevealInModelEditor: AppEvent<RevealInModelEditorEvent>;
   public readonly onFocusModelEditor: AppEvent<FocusModelEditorEvent>;
 
@@ -78,10 +86,11 @@ export class ModelingEvents extends DisposableObject {
   private readonly onMethodsChangedEventEmitter: AppEventEmitter<MethodsChangedEvent>;
   private readonly onHideModeledMethodsChangedEventEmitter: AppEventEmitter<HideModeledMethodsChangedEvent>;
   private readonly onModeChangedEventEmitter: AppEventEmitter<ModeChangedEvent>;
-  private readonly onModeledMethodsChangedEventEmitter: AppEventEmitter<ModeledMethodsChangedEvent>;
-  private readonly onModifiedMethodsChangedEventEmitter: AppEventEmitter<ModifiedMethodsChangedEvent>;
+  private readonly onModeledAndModifiedMethodsChangedEventEmitter: AppEventEmitter<ModeledAndModifiedMethodsChangedEvent>;
   private readonly onSelectedMethodChangedEventEmitter: AppEventEmitter<SelectedMethodChangedEvent>;
   private readonly onInProgressMethodsChangedEventEmitter: AppEventEmitter<InProgressMethodsChangedEvent>;
+  private readonly onProcessedByAutoModelMethodsChangedEventEmitter: AppEventEmitter<ProcessedByAutoModelMethodsChangedEvent>;
+  private readonly onModelEvaluationRunChangedEventEmitter: AppEventEmitter<ModelEvaluationRunChangedEvent>;
   private readonly onRevealInModelEditorEventEmitter: AppEventEmitter<RevealInModelEditorEvent>;
   private readonly onFocusModelEditorEventEmitter: AppEventEmitter<FocusModelEditorEvent>;
 
@@ -117,17 +126,11 @@ export class ModelingEvents extends DisposableObject {
     );
     this.onModeChanged = this.onModeChangedEventEmitter.event;
 
-    this.onModeledMethodsChangedEventEmitter = this.push(
-      app.createEventEmitter<ModeledMethodsChangedEvent>(),
+    this.onModeledAndModifiedMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<ModeledAndModifiedMethodsChangedEvent>(),
     );
-    this.onModeledMethodsChanged =
-      this.onModeledMethodsChangedEventEmitter.event;
-
-    this.onModifiedMethodsChangedEventEmitter = this.push(
-      app.createEventEmitter<ModifiedMethodsChangedEvent>(),
-    );
-    this.onModifiedMethodsChanged =
-      this.onModifiedMethodsChangedEventEmitter.event;
+    this.onModeledAndModifiedMethodsChanged =
+      this.onModeledAndModifiedMethodsChangedEventEmitter.event;
 
     this.onSelectedMethodChangedEventEmitter = this.push(
       app.createEventEmitter<SelectedMethodChangedEvent>(),
@@ -141,6 +144,18 @@ export class ModelingEvents extends DisposableObject {
     this.onInProgressMethodsChanged =
       this.onInProgressMethodsChangedEventEmitter.event;
 
+    this.onProcessedByAutoModelMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<ProcessedByAutoModelMethodsChangedEvent>(),
+    );
+    this.onProcessedByAutoModelMethodsChanged =
+      this.onProcessedByAutoModelMethodsChangedEventEmitter.event;
+
+    this.onModelEvaluationRunChangedEventEmitter = this.push(
+      app.createEventEmitter<ModelEvaluationRunChangedEvent>(),
+    );
+    this.onModelEvaluationRunChanged =
+      this.onModelEvaluationRunChangedEventEmitter.event;
+
     this.onRevealInModelEditorEventEmitter = this.push(
       app.createEventEmitter<RevealInModelEditorEvent>(),
     );
@@ -195,25 +210,15 @@ export class ModelingEvents extends DisposableObject {
     });
   }
 
-  public fireModeledMethodsChangedEvent(
+  public fireModeledAndModifiedMethodsChangedEvent(
     modeledMethods: Record<string, ModeledMethod[]>,
+    modifiedMethodSignatures: ReadonlySet<string>,
     dbUri: string,
     isActiveDb: boolean,
   ) {
-    this.onModeledMethodsChangedEventEmitter.fire({
+    this.onModeledAndModifiedMethodsChangedEventEmitter.fire({
       modeledMethods,
-      dbUri,
-      isActiveDb,
-    });
-  }
-
-  public fireModifiedMethodsChangedEvent(
-    modifiedMethods: ReadonlySet<string>,
-    dbUri: string,
-    isActiveDb: boolean,
-  ) {
-    this.onModifiedMethodsChangedEventEmitter.fire({
-      modifiedMethods,
+      modifiedMethodSignatures,
       dbUri,
       isActiveDb,
     });
@@ -226,6 +231,7 @@ export class ModelingEvents extends DisposableObject {
     modeledMethods: ModeledMethod[],
     isModified: boolean,
     isInProgress: boolean,
+    processedByAutoModel: boolean,
   ) {
     this.onSelectedMethodChangedEventEmitter.fire({
       databaseItem,
@@ -234,6 +240,7 @@ export class ModelingEvents extends DisposableObject {
       modeledMethods,
       isModified,
       isInProgress,
+      processedByAutoModel,
     });
   }
 
@@ -247,6 +254,26 @@ export class ModelingEvents extends DisposableObject {
     });
   }
 
+  public fireProcessedByAutoModelMethodsChangedEvent(
+    dbUri: string,
+    methods: ReadonlySet<string>,
+  ) {
+    this.onProcessedByAutoModelMethodsChangedEventEmitter.fire({
+      dbUri,
+      methods,
+    });
+  }
+
+  public fireModelEvaluationRunChangedEvent(
+    dbUri: string,
+    evaluationRun: ModelEvaluationRun | undefined,
+  ) {
+    this.onModelEvaluationRunChangedEventEmitter.fire({
+      dbUri,
+      evaluationRun,
+    });
+  }
+
   public fireRevealInModelEditorEvent(dbUri: string, method: Method) {
     this.onRevealInModelEditorEventEmitter.fire({
       dbUri,

extensions/ql-vscode/src/model-editor/modeling-store.ts

@@ -1,10 +1,12 @@
 import { DisposableObject } from "../common/disposable-object";
 import type { DatabaseItem } from "../databases/local-databases";
 import type { Method, Usage } from "./method";
+import type { ModelEvaluationRun } from "./model-evaluation-run";
 import type { ModeledMethod } from "./modeled-method";
 import type { ModelingEvents } from "./modeling-events";
 import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "./shared/hide-modeled-methods";
 import type { Mode } from "./shared/mode";
+import { sortMethods } from "./shared/sorting";
 
 interface InternalDbModelingState {
   databaseItem: DatabaseItem;
@@ -14,8 +16,10 @@ interface InternalDbModelingState {
   modeledMethods: Record<string, ModeledMethod[]>;
   modifiedMethodSignatures: Set<string>;
   inProgressMethods: Set<string>;
+  processedByAutoModelMethods: Set<string>;
   selectedMethod: Method | undefined;
   selectedUsage: Usage | undefined;
+  modelEvaluationRun: ModelEvaluationRun | undefined;
 }
 
 interface DbModelingState {
@@ -26,8 +30,10 @@ interface DbModelingState {
   readonly modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>;
   readonly modifiedMethodSignatures: ReadonlySet<string>;
   readonly inProgressMethods: ReadonlySet<string>;
+  readonly processedByAutoModelMethods: ReadonlySet<string>;
   readonly selectedMethod: Method | undefined;
   readonly selectedUsage: Usage | undefined;
+  readonly modelEvaluationRun: ModelEvaluationRun | undefined;
 }
 
 interface SelectedMethodDetails {
@@ -37,6 +43,7 @@ interface SelectedMethodDetails {
   readonly modeledMethods: readonly ModeledMethod[];
   readonly isModified: boolean;
   readonly isInProgress: boolean;
+  readonly processedByAutoModel: boolean;
 }
 
 export class ModelingStore extends DisposableObject {
@@ -59,9 +66,11 @@ export class ModelingStore extends DisposableObject {
       mode,
       modeledMethods: {},
       modifiedMethodSignatures: new Set(),
+      processedByAutoModelMethods: new Set(),
       selectedMethod: undefined,
       selectedUsage: undefined,
       inProgressMethods: new Set(),
+      modelEvaluationRun: undefined,
     });
 
     this.modelingEvents.fireDbOpenedEvent(databaseItem);
@@ -147,17 +156,25 @@ export class ModelingStore extends DisposableObject {
   }
 
   public setMethods(dbItem: DatabaseItem, methods: Method[]) {
-    const dbState = this.getState(dbItem);
-    const dbUri = dbItem.databaseUri.toString();
+    this.changeMethods(dbItem, (state) => {
+      state.methods = sortMethods(
+        methods,
+        state.modeledMethods,
+        state.modifiedMethodSignatures,
+        state.processedByAutoModelMethods,
+      );
+    });
+  }
 
-    dbState.methods = [...methods];
-
-    this.modelingEvents.fireMethodsChangedEvent(
-      methods,
-      dbUri,
-      dbItem,
-      dbUri === this.activeDb,
-    );
+  public updateMethodSorting(dbItem: DatabaseItem) {
+    this.changeMethods(dbItem, (state) => {
+      state.methods = sortMethods(
+        state.methods,
+        state.modeledMethods,
+        state.modifiedMethodSignatures,
+        state.processedByAutoModelMethods,
+      );
+    });
   }
 
   public setHideModeledMethods(
@@ -210,8 +227,9 @@ export class ModelingStore extends DisposableObject {
   public addModeledMethods(
     dbItem: DatabaseItem,
     methods: Record<string, ModeledMethod[]>,
+    setModified: boolean,
   ) {
-    this.changeModeledMethods(dbItem, (state) => {
+    this.changeModeledAndModifiedMethods(dbItem, (state) => {
       const newModeledMethods = {
         ...methods,
         // Keep all methods that are already modeled in some form in the state
@@ -222,6 +240,14 @@ export class ModelingStore extends DisposableObject {
         ),
       };
       state.modeledMethods = newModeledMethods;
+
+      if (setModified) {
+        const newModifiedMethods = new Set([
+          ...state.modifiedMethodSignatures,
+          ...new Set(Object.keys(methods)),
+        ]);
+        state.modifiedMethodSignatures = newModifiedMethods;
+      }
     });
   }
 
@@ -229,7 +255,7 @@ export class ModelingStore extends DisposableObject {
     dbItem: DatabaseItem,
     methods: Record<string, ModeledMethod[]>,
   ) {
-    this.changeModeledMethods(dbItem, (state) => {
+    this.changeModeledAndModifiedMethods(dbItem, (state) => {
       state.modeledMethods = { ...methods };
     });
   }
@@ -238,45 +264,28 @@ export class ModelingStore extends DisposableObject {
     dbItem: DatabaseItem,
     signature: string,
     modeledMethods: ModeledMethod[],
+    setModified: boolean,
   ) {
-    this.changeModeledMethods(dbItem, (state) => {
+    this.changeModeledAndModifiedMethods(dbItem, (state) => {
       const newModeledMethods = { ...state.modeledMethods };
       newModeledMethods[signature] = modeledMethods;
       state.modeledMethods = newModeledMethods;
-    });
-  }
 
-  public setModifiedMethods(
-    dbItem: DatabaseItem,
-    methodSignatures: Set<string>,
-  ) {
-    this.changeModifiedMethods(dbItem, (state) => {
-      state.modifiedMethodSignatures = new Set(methodSignatures);
+      if (setModified) {
+        const newModifiedMethods = new Set([
+          ...state.modifiedMethodSignatures,
+          signature,
+        ]);
+        state.modifiedMethodSignatures = newModifiedMethods;
+      }
     });
   }
 
-  public addModifiedMethods(
-    dbItem: DatabaseItem,
-    methodSignatures: Iterable<string>,
-  ) {
-    this.changeModifiedMethods(dbItem, (state) => {
-      const newModifiedMethods = new Set([
-        ...state.modifiedMethodSignatures,
-        ...methodSignatures,
-      ]);
-      state.modifiedMethodSignatures = newModifiedMethods;
-    });
-  }
-
-  public addModifiedMethod(dbItem: DatabaseItem, methodSignature: string) {
-    this.addModifiedMethods(dbItem, [methodSignature]);
-  }
-
   public removeModifiedMethods(
     dbItem: DatabaseItem,
     methodSignatures: string[],
   ) {
-    this.changeModifiedMethods(dbItem, (state) => {
+    this.changeModeledAndModifiedMethods(dbItem, (state) => {
       const newModifiedMethods = Array.from(
         state.modifiedMethodSignatures,
       ).filter((s) => !methodSignatures.includes(s));
@@ -301,6 +310,9 @@ export class ModelingStore extends DisposableObject {
     const modeledMethods = dbState.modeledMethods[method.signature] ?? [];
     const isModified = dbState.modifiedMethodSignatures.has(method.signature);
     const isInProgress = dbState.inProgressMethods.has(method.signature);
+    const processedByAutoModel = dbState.processedByAutoModelMethods.has(
+      method.signature,
+    );
     this.modelingEvents.fireSelectedMethodChangedEvent(
       dbItem,
       method,
@@ -308,6 +320,7 @@ export class ModelingStore extends DisposableObject {
       modeledMethods,
       isModified,
       isInProgress,
+      processedByAutoModel,
     );
   }
 
@@ -336,6 +349,44 @@ export class ModelingStore extends DisposableObject {
     });
   }
 
+  public getProcessedByAutoModelMethods(
+    dbItem: DatabaseItem,
+    methodSignatures?: string[],
+  ): Set<string> {
+    const processedByAutoModelMethods =
+      this.getState(dbItem).processedByAutoModelMethods;
+    if (!methodSignatures) {
+      return processedByAutoModelMethods;
+    }
+    return new Set(
+      Array.from(processedByAutoModelMethods).filter((x) =>
+        methodSignatures.includes(x),
+      ),
+    );
+  }
+
+  public addProcessedByAutoModelMethods(
+    dbItem: DatabaseItem,
+    processedByAutoModelMethods: string[],
+  ) {
+    this.changeProcessedByAutoModelMethods(dbItem, (state) => {
+      state.processedByAutoModelMethods = new Set([
+        ...state.processedByAutoModelMethods,
+        ...processedByAutoModelMethods,
+      ]);
+    });
+    this.updateMethodSorting(dbItem);
+  }
+
+  public updateModelEvaluationRun(
+    dbItem: DatabaseItem,
+    evaluationRun: ModelEvaluationRun | undefined,
+  ) {
+    this.changeModelEvaluationRun(dbItem, (state) => {
+      state.modelEvaluationRun = evaluationRun;
+    });
+  }
+
   public getSelectedMethodDetails(): SelectedMethodDetails | undefined {
     const dbState = this.getInternalStateForActiveDb();
     if (!dbState) {
@@ -356,6 +407,9 @@ export class ModelingStore extends DisposableObject {
         selectedMethod.signature,
       ),
       isInProgress: dbState.inProgressMethods.has(selectedMethod.signature),
+      processedByAutoModel: dbState.processedByAutoModelMethods.has(
+        selectedMethod.signature,
+      ),
     };
   }
 
@@ -369,7 +423,7 @@ export class ModelingStore extends DisposableObject {
     return this.state.get(databaseItem.databaseUri.toString())!;
   }
 
-  private changeModifiedMethods(
+  private changeMethods(
     dbItem: DatabaseItem,
     updateState: (state: InternalDbModelingState) => void,
   ) {
@@ -377,14 +431,15 @@ export class ModelingStore extends DisposableObject {
 
     updateState(state);
 
-    this.modelingEvents.fireModifiedMethodsChangedEvent(
-      state.modifiedMethodSignatures,
+    this.modelingEvents.fireMethodsChangedEvent(
+      state.methods,
       dbItem.databaseUri.toString(),
+      dbItem,
       dbItem.databaseUri.toString() === this.activeDb,
     );
   }
 
-  private changeModeledMethods(
+  private changeModeledAndModifiedMethods(
     dbItem: DatabaseItem,
     updateState: (state: InternalDbModelingState) => void,
   ) {
@@ -392,8 +447,9 @@ export class ModelingStore extends DisposableObject {
 
     updateState(state);
 
-    this.modelingEvents.fireModeledMethodsChangedEvent(
+    this.modelingEvents.fireModeledAndModifiedMethodsChangedEvent(
       state.modeledMethods,
+      state.modifiedMethodSignatures,
       dbItem.databaseUri.toString(),
       dbItem.databaseUri.toString() === this.activeDb,
     );
@@ -412,4 +468,32 @@ export class ModelingStore extends DisposableObject {
       state.inProgressMethods,
     );
   }
+
+  private changeProcessedByAutoModelMethods(
+    dbItem: DatabaseItem,
+    updateState: (state: InternalDbModelingState) => void,
+  ) {
+    const state = this.getState(dbItem);
+
+    updateState(state);
+
+    this.modelingEvents.fireProcessedByAutoModelMethodsChangedEvent(
+      dbItem.databaseUri.toString(),
+      state.processedByAutoModelMethods,
+    );
+  }
+
+  private changeModelEvaluationRun(
+    dbItem: DatabaseItem,
+    updateState: (state: InternalDbModelingState) => void,
+  ) {
+    const state = this.getState(dbItem);
+
+    updateState(state);
+
+    this.modelingEvents.fireModelEvaluationRunChangedEvent(
+      dbItem.databaseUri.toString(),
+      state.modelEvaluationRun,
+    );
+  }
 }

extensions/ql-vscode/src/model-editor/shared/auto-model-candidates.ts

@@ -0,0 +1,48 @@
+import type { Method, MethodSignature } from "../method";
+import type { ModeledMethod } from "../modeled-method";
+import type { Mode } from "./mode";
+import { groupMethods, sortGroupNames } from "./sorting";
+
+/**
+ * Return the candidates that the model should be run on. This includes limiting the number of
+ * candidates to the candidate limit and filtering out anything that is already modeled and respecting
+ * the order in the UI.
+ * @param mode Whether it is application or framework mode.
+ * @param methods all methods.
+ * @param modeledMethodsBySignature the currently modeled methods.
+ * @returns list of modeled methods that are candidates for modeling.
+ */
+
+export function getCandidates(
+  mode: Mode,
+  methods: readonly Method[],
+  modeledMethodsBySignature: Record<string, readonly ModeledMethod[]>,
+  processedByAutoModelMethods: Set<string>,
+): MethodSignature[] {
+  const candidateMethods = methods.filter((method) => {
+    // Filter out any methods already processed by auto-model
+    if (processedByAutoModelMethods.has(method.signature)) {
+      return false;
+    }
+
+    const modeledMethods: ModeledMethod[] = [
+      ...(modeledMethodsBySignature[method.signature] ?? []),
+    ];
+
+    // Anything that is modeled is not a candidate
+    if (modeledMethods.some((m) => m.type !== "none")) {
+      return false;
+    }
+
+    // A method that is supported is modeled outside of the model file, so it is not a candidate.
+    if (method.supported) {
+      return false;
+    }
+
+    return true;
+  });
+
+  // Sort the same way as the UI so we send the first ones listed in the UI first
+  const grouped = groupMethods(candidateMethods, mode);
+  return sortGroupNames(grouped).flatMap((name) => grouped[name]);
+}

extensions/ql-vscode/src/model-editor/shared/model-evaluation-run-state.ts

@@ -0,0 +1,19 @@
+import { VariantAnalysisStatus } from "../../variant-analysis/shared/variant-analysis";
+import type { VariantAnalysis } from "../../variant-analysis/shared/variant-analysis";
+
+export interface ModelEvaluationRunState {
+  isPreparing: boolean;
+  variantAnalysis: VariantAnalysis | undefined;
+}
+
+export function modelEvaluationRunIsRunning(
+  run: ModelEvaluationRunState,
+): boolean {
+  return (
+    run.isPreparing ||
+    !!(
+      run.variantAnalysis &&
+      run.variantAnalysis.status === VariantAnalysisStatus.InProgress
+    )
+  );
+}

extensions/ql-vscode/src/model-editor/shared/sorting.ts

@@ -1,7 +1,13 @@
+import { canMethodBeModeled } from "../method";
 import type { Method } from "../method";
+import type { ModeledMethod } from "../modeled-method";
 import { Mode } from "./mode";
 import { calculateModeledPercentage } from "./modeled-percentage";
 
+/**
+ * Groups methods by library or package name.
+ * Does not change the order of methods within a group.
+ */
 export function groupMethods(
   methods: readonly Method[],
   mode: Mode,
@@ -27,12 +33,84 @@ export function sortGroupNames(
   );
 }
 
-export function sortMethods(methods: readonly Method[]): Method[] {
+/**
+ * Primarily sorts methods into the following order:
+ * - Unsaved positive AutoModel predictions
+ * - Negative AutoModel predictions
+ * - Unsaved manual models + unmodeled methods
+ * - Saved models from this model pack (AutoModel and manual)
+ * - Methods not modelable in this model pack
+ *
+ * Secondary sort order is by number of usages descending, then by method signature ascending.
+ */
+export function sortMethods(
+  methods: readonly Method[],
+  modeledMethodsMap: Record<string, readonly ModeledMethod[]>,
+  modifiedSignatures: ReadonlySet<string>,
+  processedByAutoModelMethods: ReadonlySet<string>,
+): Method[] {
   const sortedMethods = [...methods];
-  sortedMethods.sort((a, b) => compareMethod(a, b));
+  sortedMethods.sort((a, b) => {
+    // First sort by the type of method
+    const methodAPrimarySortOrdinal = getMethodPrimarySortOrdinal(
+      a,
+      modeledMethodsMap[a.signature] ?? [],
+      modifiedSignatures.has(a.signature),
+      processedByAutoModelMethods.has(a.signature),
+    );
+    const methodBPrimarySortOrdinal = getMethodPrimarySortOrdinal(
+      b,
+      modeledMethodsMap[b.signature] ?? [],
+      modifiedSignatures.has(b.signature),
+      processedByAutoModelMethods.has(b.signature),
+    );
+    if (methodAPrimarySortOrdinal !== methodBPrimarySortOrdinal) {
+      return methodAPrimarySortOrdinal - methodBPrimarySortOrdinal;
+    }
+
+    // Then sort by number of usages descending
+    const usageDifference = b.usages.length - a.usages.length;
+    if (usageDifference !== 0) {
+      return usageDifference;
+    }
+
+    // Then sort by method signature ascending
+    return a.signature.localeCompare(b.signature);
+  });
   return sortedMethods;
 }
 
+/**
+ * Assigns numbers to the following classes of methods:
+ * - Unsaved positive AutoModel predictions => 0
+ * - Negative AutoModel predictions => 1
+ * - Unsaved manual models + unmodeled methods => 2
+ * - Saved models from this model pack (AutoModel and manual) => 3
+ * - Methods not modelable in this model pack => 4
+ */
+function getMethodPrimarySortOrdinal(
+  method: Method,
+  modeledMethods: readonly ModeledMethod[],
+  isUnsaved: boolean,
+  isProcessedByAutoModel: boolean,
+): number {
+  const canBeModeled = canMethodBeModeled(method, modeledMethods, isUnsaved);
+  const isModeled = modeledMethods.length > 0;
+  if (canBeModeled) {
+    if (isModeled && isUnsaved && isProcessedByAutoModel) {
+      return 0;
+    } else if (!isModeled && isProcessedByAutoModel) {
+      return 1;
+    } else if ((isModeled && isUnsaved) || !isModeled) {
+      return 2;
+    } else {
+      return 3;
+    }
+  } else {
+    return 4;
+  }
+}
+
 function compareGroups(
   a: readonly Method[],
   aName: string,
@@ -69,22 +147,3 @@ function compareGroups(
   // Then sort by number of usages descending
   return numberOfUsagesB - numberOfUsagesA;
 }
-
-function compareMethod(a: Method, b: Method): number {
-  // Sort first by supported, putting unmodeled methods first.
-  if (a.supported && !b.supported) {
-    return 1;
-  }
-  if (!a.supported && b.supported) {
-    return -1;
-  }
-
-  // Then sort by number of usages descending
-  const usageDifference = b.usages.length - a.usages.length;
-  if (usageDifference !== 0) {
-    return usageDifference;
-  }
-
-  // Then sort by method signature ascending
-  return a.signature.localeCompare(b.signature);
-}

extensions/ql-vscode/src/model-editor/shared/view-state.ts

@@ -1,17 +1,21 @@
 import type { ExtensionPack } from "./extension-pack";
 import type { Mode } from "./mode";
 import type { QueryLanguage } from "../../common/query-language";
+import type { ModelConfig } from "../languages";
 
 export interface ModelEditorViewState {
   extensionPack: ExtensionPack;
   language: QueryLanguage;
   showGenerateButton: boolean;
   showLlmButton: boolean;
+  showEvaluationUi: boolean;
   mode: Mode;
   showModeSwitchButton: boolean;
   sourceArchiveAvailable: boolean;
+  modelConfig: ModelConfig;
 }
 
 export interface MethodModelingPanelViewState {
   language: QueryLanguage | undefined;
+  modelConfig: ModelConfig;
 }

extensions/ql-vscode/src/model-editor/yaml.ts

@@ -16,7 +16,10 @@ import type {
 import { getModelsAsDataLanguage } from "./languages";
 import { Mode } from "./shared/mode";
 import { assertNever } from "../common/helpers-pure";
-import type { ModelExtensionFile } from "./model-extension-file";
+import type {
+  ModelExtension,
+  ModelExtensionFile,
+} from "./model-extension-file";
 import type { QueryLanguage } from "../common/query-language";
 
 import modelExtensionFileSchema from "./model-extension-file.schema.json";
@@ -24,38 +27,22 @@ import modelExtensionFileSchema from "./model-extension-file.schema.json";
 const ajv = new Ajv({ allErrors: true, allowUnionTypes: true });
 const modelExtensionFileSchemaValidate = ajv.compile(modelExtensionFileSchema);
 
-function createDataProperty<T>(
-  methods: readonly T[],
-  definition: ModelsAsDataLanguagePredicate<T>,
-) {
-  if (methods.length === 0) {
-    return " []";
-  }
-
-  return `\n${methods
-    .map(
-      (method) =>
-        `      - ${JSON.stringify(
-          definition.generateMethodDefinition(method),
-        )}`,
-    )
-    .join("\n")}`;
-}
-
 function createExtensions<T>(
   language: QueryLanguage,
   methods: readonly T[],
   definition: ModelsAsDataLanguagePredicate<T> | undefined,
-) {
+): ModelExtension | undefined {
   if (!definition) {
-    return "";
+    return undefined;
   }
 
-  return `  - addsTo:
-      pack: codeql/${language}-all
-      extensible: ${definition.extensiblePredicate}
-    data:${createDataProperty(methods, definition)}
-`;
+  return {
+    addsTo: {
+      pack: `codeql/${language}-all`,
+      extensible: definition.extensiblePredicate,
+    },
+    data: methods.map((method) => definition.generateMethodDefinition(method)),
+  };
 }
 
 export function createDataExtensionYaml(
@@ -99,7 +86,7 @@ export function createDataExtensionYaml(
   }
 
   const extensions = Object.keys(methodsByType)
-    .map((typeKey) => {
+    .map((typeKey): ModelExtension | undefined => {
       const type = typeKey as keyof ModelsAsDataLanguagePredicates;
 
       switch (type) {
@@ -137,10 +124,11 @@ export function createDataExtensionYaml(
           assertNever(type);
       }
     })
-    .filter((extensions) => extensions !== "");
+    .filter(
+      (extension): extension is ModelExtension => extension !== undefined,
+    );
 
-  return `extensions:
-${extensions.join("\n")}`;
+  return modelExtensionFileToYaml({ extensions });
 }
 
 export function createDataExtensionYamls(
@@ -341,6 +329,36 @@ function validateModelExtensionFile(data: unknown): data is ModelExtensionFile {
   return true;
 }
 
+/**
+ * Creates a string for the data extension YAML file from the
+ * structure of the data extension file. This should be used
+ * instead of creating a JSON string directly or dumping the
+ * YAML directly to ensure that the file is formatted correctly.
+ *
+ * @param data The data extension file
+ */
+export function modelExtensionFileToYaml(data: ModelExtensionFile) {
+  const extensions = data.extensions
+    .map((extension) => {
+      const data =
+        extension.data.length === 0
+          ? " []"
+          : `\n${extension.data
+              .map((row) => `      - ${JSON.stringify(row)}`)
+              .join("\n")}`;
+
+      return `  - addsTo:
+      pack: ${extension.addsTo.pack}
+      extensible: ${extension.addsTo.extensible}
+    data:${data}
+`;
+    })
+    .filter((extensions) => extensions !== "");
+
+  return `extensions:
+${extensions.join("\n")}`;
+}
+
 export function loadDataExtensionYaml(
   data: unknown,
   language: QueryLanguage,

extensions/ql-vscode/src/packages/commands/CommandManager.ts

@@ -10,6 +10,7 @@ import type { Disposable } from "./Disposable";
 /**
  * A command function is a completely untyped command.
  */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type CommandFunction = (...args: any[]) => Promise<unknown>;
 
 /**

extensions/ql-vscode/src/query-history/query-status.ts

@@ -17,6 +17,8 @@ export function variantAnalysisStatusToQueryStatus(
       return QueryStatus.Failed;
     case VariantAnalysisStatus.InProgress:
       return QueryStatus.InProgress;
+    case VariantAnalysisStatus.Canceling:
+      return QueryStatus.InProgress;
     case VariantAnalysisStatus.Canceled:
       return QueryStatus.Completed;
     default:

extensions/ql-vscode/src/query-history/store/query-history-variant-analysis-domain-mapper.ts

@@ -195,6 +195,11 @@ function mapVariantAnalysisStatusToDto(
       return VariantAnalysisStatusDto.Succeeded;
     case VariantAnalysisStatus.Failed:
       return VariantAnalysisStatusDto.Failed;
+    case VariantAnalysisStatus.Canceling:
+      // The canceling state shouldn't be persisted. We can just
+      // assume that the analysis is still in progress, since the
+      // canceling state is very short-lived.
+      return VariantAnalysisStatusDto.InProgress;
     case VariantAnalysisStatus.Canceled:
       return VariantAnalysisStatusDto.Canceled;
     default:

extensions/ql-vscode/src/query-testing/test-manager.ts

@@ -21,7 +21,8 @@ import {
 } from "vscode";
 import { DisposableObject } from "../common/disposable-object";
 import { QLTestDiscovery } from "./qltest-discovery";
-import type { CodeQLCliServer } from "../codeql-cli/cli";
+import type { CodeQLCliServer, CompilationMessage } from "../codeql-cli/cli";
+import { CompilationMessageSeverity } from "../codeql-cli/cli";
 import { getErrorMessage } from "../common/helpers-pure";
 import type { BaseLogger, LogOptions } from "../common/logging";
 import type { TestRunner } from "./test-runner";
@@ -66,6 +67,23 @@ function changeExtension(p: string, ext: string): string {
   return p.slice(0, -extname(p).length) + ext;
 }
 
+function compilationMessageToTestMessage(
+  compilationMessage: CompilationMessage,
+): TestMessage {
+  const location = new Location(
+    Uri.file(compilationMessage.position.fileName),
+    new Range(
+      compilationMessage.position.line - 1,
+      compilationMessage.position.column - 1,
+      compilationMessage.position.endLine - 1,
+      compilationMessage.position.endColumn - 1,
+    ),
+  );
+  const testMessage = new TestMessage(compilationMessage.message);
+  testMessage.location = location;
+  return testMessage;
+}
+
 /**
  * Returns the complete text content of the specified file. If there is an error reading the file,
  * an error message is added to `testMessages` and this function returns undefined.
@@ -398,23 +416,15 @@ export class TestManager extends DisposableObject {
               );
             }
           }
-          if (event.messages?.length > 0) {
+          const errorMessages = event.messages.filter(
+            (m) => m.severity === CompilationMessageSeverity.Error,
+          );
+          if (errorMessages.length > 0) {
             // The test didn't make it far enough to produce results. Transform any error messages
             // into `TestMessage`s and report the test as "errored".
-            const testMessages = event.messages.map((m) => {
-              const location = new Location(
-                Uri.file(m.position.fileName),
-                new Range(
-                  m.position.line - 1,
-                  m.position.column - 1,
-                  m.position.endLine - 1,
-                  m.position.endColumn - 1,
-                ),
-              );
-              const testMessage = new TestMessage(m.message);
-              testMessage.location = location;
-              return testMessage;
-            });
+            const testMessages = event.messages.map(
+              compilationMessageToTestMessage,
+            );
             testRun.errored(testItem, testMessages, duration);
           } else {
             // Results didn't match expectations. Report the test as "failed".
@@ -423,6 +433,12 @@ export class TestManager extends DisposableObject {
               // here. Any failed test needs at least one message.
               testMessages.push(new TestMessage("Test failed"));
             }
+
+            // Add any warnings produced by the test to the test messages.
+            testMessages.push(
+              ...event.messages.map(compilationMessageToTestMessage),
+            );
+
             testRun.failed(testItem, testMessages, duration);
           }
         }

extensions/ql-vscode/src/stories/method-modeling/MethodModelingInputs.stories.tsx

@@ -6,6 +6,7 @@ import { createSinkModeledMethod } from "../../../test/factories/model-editor/mo
 import { useState } from "react";
 import type { ModeledMethod } from "../../model-editor/modeled-method";
 import { QueryLanguage } from "../../common/query-language";
+import { defaultModelConfig } from "../../model-editor/languages";
 
 export default {
   title: "Method Modeling/Method Modeling Inputs",
@@ -34,6 +35,7 @@ const Template: StoryFn<typeof MethodModelingInputsComponent> = (args) => {
       language={QueryLanguage.Java}
       modeledMethod={m}
       onChange={onChange}
+      modelConfig={defaultModelConfig}
     />
   );
 };
@@ -66,5 +68,5 @@ export const ModelingNotAccepted = Template.bind({});
 ModelingNotAccepted.args = {
   method,
   modeledMethod: generatedModeledMethod,
-  modelingStatus: "unsaved",
+  modelPending: true,
 };

extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisActions.stories.tsx

@@ -70,3 +70,9 @@ Failed.args = {
   ...InProgress.args,
   variantAnalysisStatus: VariantAnalysisStatus.Failed,
 };
+
+export const Canceling = Template.bind({});
+Canceling.args = {
+  ...InProgress.args,
+  variantAnalysisStatus: VariantAnalysisStatus.Canceling,
+};

extensions/ql-vscode/src/variant-analysis/code-scanning-pack.ts

@@ -0,0 +1,77 @@
+import { join } from "path";
+import type { BaseLogger } from "../common/logging";
+import type { QueryLanguage } from "../common/query-language";
+import type { CodeQLCliServer } from "../codeql-cli/cli";
+import type { QlPackDetails } from "./ql-pack-details";
+import { getQlPackFilePath } from "../common/ql";
+
+export async function resolveCodeScanningQueryPack(
+  logger: BaseLogger,
+  cliServer: CodeQLCliServer,
+  language: QueryLanguage,
+): Promise<QlPackDetails> {
+  // Get pack
+  void logger.log(`Downloading pack for language: ${language}`);
+  const packName = `codeql/${language}-queries`;
+  const packDownloadResult = await cliServer.packDownload([packName]);
+  const downloadedPack = packDownloadResult.packs[0];
+
+  const packDir = join(
+    packDownloadResult.packDir,
+    downloadedPack.name,
+    downloadedPack.version,
+  );
+
+  // Resolve queries
+  void logger.log(`Resolving queries for pack: ${packName}`);
+  const suitePath = join(
+    packDir,
+    "codeql-suites",
+    `${language}-code-scanning.qls`,
+  );
+  const resolvedQueries = await cliServer.resolveQueries(suitePath);
+
+  const problemQueries = await filterToOnlyProblemQueries(
+    logger,
+    cliServer,
+    resolvedQueries,
+  );
+
+  if (problemQueries.length === 0) {
+    throw Error(
+      `No problem queries found in published query pack: ${packName}.`,
+    );
+  }
+
+  // Return pack details
+  const qlPackFilePath = await getQlPackFilePath(packDir);
+
+  const qlPackDetails: QlPackDetails = {
+    queryFiles: problemQueries,
+    qlPackRootPath: packDir,
+    qlPackFilePath,
+    language,
+  };
+
+  return qlPackDetails;
+}
+
+async function filterToOnlyProblemQueries(
+  logger: BaseLogger,
+  cliServer: CodeQLCliServer,
+  queries: string[],
+): Promise<string[]> {
+  const problemQueries: string[] = [];
+  for (const query of queries) {
+    const queryMetadata = await cliServer.resolveMetadata(query);
+    if (
+      queryMetadata.kind === "problem" ||
+      queryMetadata.kind === "path-problem"
+    ) {
+      problemQueries.push(query);
+    } else {
+      void logger.log(`Skipping non-problem query ${query}`);
+    }
+  }
+  return problemQueries;
+}

extensions/ql-vscode/src/variant-analysis/export-results.ts

@@ -42,7 +42,7 @@ export async function exportVariantAnalysisResults(
   await withProgress(
     async (progress: ProgressCallback, token: CancellationToken) => {
       const variantAnalysis =
-        await variantAnalysisManager.getVariantAnalysis(variantAnalysisId);
+        variantAnalysisManager.tryGetVariantAnalysis(variantAnalysisId);
       if (!variantAnalysis) {
         void extLogger.log(
           `Could not find variant analysis with id ${variantAnalysisId}`,
@@ -57,7 +57,7 @@ export async function exportVariantAnalysisResults(
       }
 
       const repoStates =
-        await variantAnalysisManager.getRepoStates(variantAnalysisId);
+        variantAnalysisManager.getRepoStates(variantAnalysisId);
 
       void extLogger.log(
         `Exporting variant analysis results for variant analysis with id ${variantAnalysis.id}`,

extensions/ql-vscode/src/variant-analysis/run-remote-query.ts

@@ -131,16 +131,7 @@ async function generateQueryPack(
       ...extensionPacks.map((p) => `--extension-pack=${p}@*`),
     ];
   } else {
-    if (await cliServer.cliConstraints.usesGlobalCompilationCache()) {
-      precompilationOpts = ["--qlx"];
-    } else {
-      const cache = join(qlPackDetails.qlPackRootPath, ".cache");
-      precompilationOpts = [
-        "--qlx",
-        "--no-default-compilation-cache",
-        `--compilation-cache=${cache}`,
-      ];
-    }
+    precompilationOpts = ["--qlx"];
 
     if (extensionPacks.length > 0) {
       await addExtensionPacksAsDependencies(targetPackPath, extensionPacks);
@@ -408,7 +399,7 @@ async function getExtensionPacksToInject(
   workspaceFolders: string[],
 ): Promise<string[]> {
   const result: string[] = [];
-  if (await cliServer.useExtensionPacks()) {
+  if (cliServer.useExtensionPacks()) {
     const extensionPacks = await cliServer.resolveQlpacks(
       workspaceFolders,
       true,

extensions/ql-vscode/src/variant-analysis/shared/variant-analysis.ts

@@ -39,6 +39,7 @@ export enum VariantAnalysisStatus {
   InProgress = "inProgress",
   Succeeded = "succeeded",
   Failed = "failed",
+  Canceling = "canceling",
   Canceled = "canceled",
 }
 

extensions/ql-vscode/src/variant-analysis/variant-analysis-content-provider.ts

@@ -22,7 +22,7 @@ export const createVariantAnalysisContentProvider = (
     const variantAnalysisId = parseInt(variantAnalysisIdString);
 
     const variantAnalysis =
-      await variantAnalysisManager.getVariantAnalysis(variantAnalysisId);
+      variantAnalysisManager.tryGetVariantAnalysis(variantAnalysisId);
     if (!variantAnalysis) {
       void showAndLogWarningMessage(
         extLogger,

extensions/ql-vscode/src/variant-analysis/variant-analysis-manager.ts

@@ -34,6 +34,7 @@ import {
   isVariantAnalysisComplete,
   parseVariantAnalysisQueryLanguage,
   VariantAnalysisScannedRepositoryDownloadStatus,
+  VariantAnalysisStatus,
 } from "./shared/variant-analysis";
 import { getErrorMessage } from "../common/helpers-pure";
 import { VariantAnalysisView } from "./variant-analysis-view";
@@ -44,7 +45,7 @@ import type {
 } from "./variant-analysis-results-manager";
 import { getQueryName, prepareRemoteQueryRun } from "./run-remote-query";
 import {
-  mapVariantAnalysis,
+  mapVariantAnalysisFromSubmission,
   mapVariantAnalysisRepositoryTask,
 } from "./variant-analysis-mapper";
 import PQueue from "p-queue";
@@ -94,6 +95,7 @@ import { getQlPackFilePath } from "../common/ql";
 import { tryGetQueryMetadata } from "../codeql-cli/query-metadata";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { findVariantAnalysisQlPackRoot } from "./ql";
+import { resolveCodeScanningQueryPack } from "./code-scanning-pack";
 
 const maxRetryCount = 3;
 
@@ -144,6 +146,7 @@ export class VariantAnalysisManager
       new VariantAnalysisMonitor(
         app,
         this.shouldCancelMonitorVariantAnalysis.bind(this),
+        this.getVariantAnalysis.bind(this),
       ),
     );
     this.variantAnalysisMonitor.onVariantAnalysisChange(
@@ -219,7 +222,7 @@ export class VariantAnalysisManager
   public async runVariantAnalysisFromPublishedPack(): Promise<void> {
     return withProgress(async (progress, token) => {
       progress({
-        maxStep: 8,
+        maxStep: 7,
         step: 0,
         message: "Determining query language",
       });
@@ -230,53 +233,17 @@ export class VariantAnalysisManager
       }
 
       progress({
-        maxStep: 8,
-        step: 1,
-        message: "Downloading query pack",
-      });
-
-      const packName = `codeql/${language}-queries`;
-      const packDownloadResult = await this.cliServer.packDownload([packName]);
-      const downloadedPack = packDownloadResult.packs[0];
-
-      const packDir = join(
-        packDownloadResult.packDir,
-        downloadedPack.name,
-        downloadedPack.version,
-      );
-
-      progress({
-        maxStep: 8,
+        maxStep: 7,
         step: 2,
-        message: "Resolving queries in pack",
+        message: "Downloading query pack and resolving queries",
       });
 
-      const suitePath = join(
-        packDir,
-        "codeql-suites",
-        `${language}-code-scanning.qls`,
-      );
-      const resolvedQueries = await this.cliServer.resolveQueries(suitePath);
-
-      const problemQueries =
-        await this.filterToOnlyProblemQueries(resolvedQueries);
-
-      if (problemQueries.length === 0) {
-        void this.app.logger.showErrorMessage(
-          `Unable to trigger variant analysis. No problem queries found in published query pack: ${packName}.`,
-        );
-        return;
-      }
-
-      const qlPackFilePath = await getQlPackFilePath(packDir);
-
       // Build up details to pass to the functions that run the variant analysis.
-      const qlPackDetails: QlPackDetails = {
-        queryFiles: problemQueries,
-        qlPackRootPath: packDir,
-        qlPackFilePath,
+      const qlPackDetails = await resolveCodeScanningQueryPack(
+        this.app.logger,
+        this.cliServer,
         language,
-      };
+      );
 
       await this.runVariantAnalysis(
         qlPackDetails,
@@ -291,24 +258,6 @@ export class VariantAnalysisManager
     });
   }
 
-  private async filterToOnlyProblemQueries(
-    queries: string[],
-  ): Promise<string[]> {
-    const problemQueries: string[] = [];
-    for (const query of queries) {
-      const queryMetadata = await this.cliServer.resolveMetadata(query);
-      if (
-        queryMetadata.kind === "problem" ||
-        queryMetadata.kind === "path-problem"
-      ) {
-        problemQueries.push(query);
-      } else {
-        void this.app.logger.log(`Skipping non-problem query ${query}`);
-      }
-    }
-    return problemQueries;
-  }
-
   private async runVariantAnalysisCommand(queryFiles: Uri[]): Promise<void> {
     if (queryFiles.length === 0) {
       throw new Error("Please select a .ql file to run as a variant analysis");
@@ -351,7 +300,7 @@ export class VariantAnalysisManager
     qlPackDetails: QlPackDetails,
     progress: ProgressCallback,
     token: CancellationToken,
-  ): Promise<void> {
+  ): Promise<number | undefined> {
     await saveBeforeStart();
 
     progress({
@@ -432,32 +381,34 @@ export class VariantAnalysisManager
     } catch (e: unknown) {
       // If the error is handled by the handleRequestError function, we don't need to throw
       if (e instanceof RequestError && handleRequestError(e, this.app.logger)) {
-        return;
+        return undefined;
       }
 
       throw e;
     }
 
-    const processedVariantAnalysis = mapVariantAnalysis(
+    const mappedVariantAnalysis = mapVariantAnalysisFromSubmission(
       variantAnalysisSubmission,
       variantAnalysisResponse,
     );
 
-    await this.onVariantAnalysisSubmitted(processedVariantAnalysis);
+    await this.onVariantAnalysisSubmitted(mappedVariantAnalysis);
 
     void showAndLogInformationMessage(
       this.app.logger,
-      `Variant analysis ${processedVariantAnalysis.query.name} submitted for processing`,
+      `Variant analysis ${mappedVariantAnalysis.query.name} submitted for processing`,
     );
 
     void this.app.commands.execute(
       "codeQL.openVariantAnalysisView",
-      processedVariantAnalysis.id,
+      mappedVariantAnalysis.id,
     );
     void this.app.commands.execute(
       "codeQL.monitorNewVariantAnalysis",
-      processedVariantAnalysis,
+      mappedVariantAnalysis,
     );
+
+    return mappedVariantAnalysis.id;
   }
 
   public async rehydrateVariantAnalysis(variantAnalysis: VariantAnalysis) {
@@ -535,7 +486,7 @@ export class VariantAnalysisManager
   }
 
   public async openQueryText(variantAnalysisId: number): Promise<void> {
-    const variantAnalysis = await this.getVariantAnalysis(variantAnalysisId);
+    const variantAnalysis = this.tryGetVariantAnalysis(variantAnalysisId);
     if (!variantAnalysis) {
       void showAndLogWarningMessage(
         this.app.logger,
@@ -566,7 +517,7 @@ export class VariantAnalysisManager
   }
 
   public async openQueryFile(variantAnalysisId: number): Promise<void> {
-    const variantAnalysis = await this.getVariantAnalysis(variantAnalysisId);
+    const variantAnalysis = this.tryGetVariantAnalysis(variantAnalysisId);
 
     if (!variantAnalysis) {
       void showAndLogWarningMessage(
@@ -608,15 +559,15 @@ export class VariantAnalysisManager
     return this.views.get(variantAnalysisId);
   }
 
-  public async getVariantAnalysis(
+  public tryGetVariantAnalysis(
     variantAnalysisId: number,
-  ): Promise<VariantAnalysis | undefined> {
+  ): VariantAnalysis | undefined {
     return this.variantAnalyses.get(variantAnalysisId);
   }
 
-  public async getRepoStates(
+  public getRepoStates(
     variantAnalysisId: number,
-  ): Promise<VariantAnalysisScannedRepositoryState[]> {
+  ): VariantAnalysisScannedRepositoryState[] {
     return Object.values(this.repoStates.get(variantAnalysisId) ?? {});
   }
 
@@ -655,6 +606,16 @@ export class VariantAnalysisManager
     return !this.variantAnalyses.has(variantAnalysisId);
   }
 
+  private getVariantAnalysis(variantAnalysisId: number): VariantAnalysis {
+    const variantAnalysis = this.tryGetVariantAnalysis(variantAnalysisId);
+
+    if (!variantAnalysis) {
+      throw new Error(`No variant analysis with id: ${variantAnalysisId}`);
+    }
+
+    return variantAnalysis;
+  }
+
   public async onVariantAnalysisUpdated(
     variantAnalysis: VariantAnalysis | undefined,
   ): Promise<void> {
@@ -662,7 +623,11 @@ export class VariantAnalysisManager
       return;
     }
 
-    if (!this.variantAnalyses.has(variantAnalysis.id)) {
+    const originalVariantAnalysis = this.variantAnalyses.get(
+      variantAnalysis.id,
+    );
+
+    if (!originalVariantAnalysis) {
       return;
     }
 
@@ -895,11 +860,24 @@ export class VariantAnalysisManager
       );
     }
 
+    await this.onVariantAnalysisUpdated({
+      ...variantAnalysis,
+      status: VariantAnalysisStatus.Canceling,
+    });
+
     void showAndLogInformationMessage(
       this.app.logger,
       "Cancelling variant analysis. This may take a while.",
     );
-    await cancelVariantAnalysis(this.app.credentials, variantAnalysis);
+    try {
+      await cancelVariantAnalysis(this.app.credentials, variantAnalysis);
+    } catch (e) {
+      await this.onVariantAnalysisUpdated({
+        ...variantAnalysis,
+        status: VariantAnalysisStatus.InProgress,
+      });
+      throw e;
+    }
   }
 
   public async openVariantAnalysisLogs(variantAnalysisId: number) {

extensions/ql-vscode/src/variant-analysis/variant-analysis-mapper.ts

@@ -23,11 +23,11 @@ import {
   VariantAnalysisRepoStatus,
 } from "./shared/variant-analysis";
 
-export function mapVariantAnalysis(
+export function mapVariantAnalysisFromSubmission(
   submission: VariantAnalysisSubmission,
-  response: ApiVariantAnalysis,
+  apiVariantAnalysis: ApiVariantAnalysis,
 ): VariantAnalysis {
-  return mapUpdatedVariantAnalysis(
+  return mapVariantAnalysis(
     {
       language: submission.language,
       query: {
@@ -40,15 +40,28 @@ export function mapVariantAnalysis(
       databases: submission.databases,
       executionStartTime: submission.startTime,
     },
-    response,
+    undefined,
+    apiVariantAnalysis,
   );
 }
 
 export function mapUpdatedVariantAnalysis(
-  previousVariantAnalysis: Pick<
+  currentVariantAnalysis: VariantAnalysis,
+  apiVariantAnalysis: ApiVariantAnalysis,
+): VariantAnalysis {
+  return mapVariantAnalysis(
+    currentVariantAnalysis,
+    currentVariantAnalysis.status,
+    apiVariantAnalysis,
+  );
+}
+
+function mapVariantAnalysis(
+  currentVariantAnalysis: Pick<
     VariantAnalysis,
     "language" | "query" | "queries" | "databases" | "executionStartTime"
   >,
+  currentStatus: VariantAnalysisStatus | undefined,
   response: ApiVariantAnalysis,
 ): VariantAnalysis {
   let scannedRepos: VariantAnalysisScannedRepository[] = [];
@@ -66,6 +79,13 @@ export function mapUpdatedVariantAnalysis(
     );
   }
 
+  // Maintain the canceling status if we are still canceling.
+  const status =
+    currentStatus === VariantAnalysisStatus.Canceling &&
+    response.status === "in_progress"
+      ? VariantAnalysisStatus.Canceling
+      : mapApiStatus(response.status);
+
   const variantAnalysis: VariantAnalysis = {
     id: response.id,
     controllerRepo: {
@@ -73,14 +93,14 @@ export function mapUpdatedVariantAnalysis(
       fullName: response.controller_repo.full_name,
       private: response.controller_repo.private,
     },
-    language: previousVariantAnalysis.language,
-    query: previousVariantAnalysis.query,
-    queries: previousVariantAnalysis.queries,
-    databases: previousVariantAnalysis.databases,
-    executionStartTime: previousVariantAnalysis.executionStartTime,
+    language: currentVariantAnalysis.language,
+    query: currentVariantAnalysis.query,
+    queries: currentVariantAnalysis.queries,
+    databases: currentVariantAnalysis.databases,
+    executionStartTime: currentVariantAnalysis.executionStartTime,
     createdAt: response.created_at,
     updatedAt: response.updated_at,
-    status: mapApiStatus(response.status),
+    status,
     completedAt: response.completed_at,
     actionsWorkflowRunId: response.actions_workflow_run_id,
     scannedRepos,

extensions/ql-vscode/src/variant-analysis/variant-analysis-monitor.ts

@@ -17,6 +17,7 @@ import { sleep } from "../common/time";
 import { getErrorMessage } from "../common/helpers-pure";
 import type { App } from "../common/app";
 import { showAndLogWarningMessage } from "../common/logging";
+import type { QueryLanguage } from "../common/query-language";
 
 export class VariantAnalysisMonitor extends DisposableObject {
   // With a sleep of 5 seconds, the maximum number of attempts takes
@@ -36,6 +37,9 @@ export class VariantAnalysisMonitor extends DisposableObject {
     private readonly shouldCancelMonitor: (
       variantAnalysisId: number,
     ) => Promise<boolean>,
+    private readonly getVariantAnalysis: (
+      variantAnalysisId: number,
+    ) => VariantAnalysis,
   ) {
     super();
   }
@@ -56,20 +60,28 @@ export class VariantAnalysisMonitor extends DisposableObject {
 
     this.monitoringVariantAnalyses.add(variantAnalysis.id);
     try {
-      await this._monitorVariantAnalysis(variantAnalysis);
+      await this._monitorVariantAnalysis(
+        variantAnalysis.id,
+        variantAnalysis.controllerRepo.id,
+        variantAnalysis.executionStartTime,
+        variantAnalysis.query.name,
+        variantAnalysis.language,
+      );
     } finally {
       this.monitoringVariantAnalyses.delete(variantAnalysis.id);
     }
   }
 
   private async _monitorVariantAnalysis(
-    variantAnalysis: VariantAnalysis,
+    variantAnalysisId: number,
+    controllerRepoId: number,
+    executionStartTime: number,
+    queryName: string,
+    language: QueryLanguage,
   ): Promise<void> {
-    const variantAnalysisLabel = `${variantAnalysis.query.name} (${
-      variantAnalysis.language
-    }) [${new Date(variantAnalysis.executionStartTime).toLocaleString(
-      env.language,
-    )}]`;
+    const variantAnalysisLabel = `${queryName} (${language}) [${new Date(
+      executionStartTime,
+    ).toLocaleString(env.language)}]`;
 
     let attemptCount = 0;
     const scannedReposDownloaded: number[] = [];
@@ -79,7 +91,7 @@ export class VariantAnalysisMonitor extends DisposableObject {
     while (attemptCount <= VariantAnalysisMonitor.maxAttemptCount) {
       await sleep(VariantAnalysisMonitor.sleepTime);
 
-      if (await this.shouldCancelMonitor(variantAnalysis.id)) {
+      if (await this.shouldCancelMonitor(variantAnalysisId)) {
         return;
       }
 
@@ -87,8 +99,8 @@ export class VariantAnalysisMonitor extends DisposableObject {
       try {
         variantAnalysisSummary = await getVariantAnalysis(
           this.app.credentials,
-          variantAnalysis.controllerRepo.id,
-          variantAnalysis.id,
+          controllerRepoId,
+          variantAnalysisId,
         );
       } catch (e) {
         const errorMessage = getErrorMessage(e);
@@ -119,8 +131,10 @@ export class VariantAnalysisMonitor extends DisposableObject {
         continue;
       }
 
-      variantAnalysis = mapUpdatedVariantAnalysis(
-        variantAnalysis,
+      const variantAnalysis = mapUpdatedVariantAnalysis(
+        // Get the variant analysis as known by the rest of the app, because it may
+        // have been changed by the user and the monitors may not be aware of it yet.
+        this.getVariantAnalysis(variantAnalysisId),
         variantAnalysisSummary,
       );
 

extensions/ql-vscode/src/variant-analysis/variant-analysis-view-manager.ts

@@ -19,12 +19,10 @@ export interface VariantAnalysisViewManager<
   unregisterView(view: T): void;
   getView(variantAnalysisId: number): T | undefined;
 
-  getVariantAnalysis(
-    variantAnalysisId: number,
-  ): Promise<VariantAnalysis | undefined>;
+  tryGetVariantAnalysis(variantAnalysisId: number): VariantAnalysis | undefined;
   getRepoStates(
     variantAnalysisId: number,
-  ): Promise<VariantAnalysisScannedRepositoryState[]>;
+  ): VariantAnalysisScannedRepositoryState[];
   openQueryFile(variantAnalysisId: number): Promise<void>;
   openQueryText(variantAnalysisId: number): Promise<void>;
   cancelVariantAnalysis(variantAnalysisId: number): Promise<void>;

extensions/ql-vscode/src/variant-analysis/variant-analysis-view.ts

@@ -96,7 +96,7 @@ export class VariantAnalysisView
   }
 
   protected async getPanelConfig(): Promise<WebviewPanelConfig> {
-    const variantAnalysis = await this.manager.getVariantAnalysis(
+    const variantAnalysis = this.manager.tryGetVariantAnalysis(
       this.variantAnalysisId,
     );
 
@@ -178,7 +178,7 @@ export class VariantAnalysisView
 
     void this.app.logger.log("Variant analysis view loaded");
 
-    const variantAnalysis = await this.manager.getVariantAnalysis(
+    const variantAnalysis = this.manager.tryGetVariantAnalysis(
       this.variantAnalysisId,
     );
 
@@ -206,7 +206,7 @@ export class VariantAnalysisView
       filterSortState,
     });
 
-    const repoStates = await this.manager.getRepoStates(this.variantAnalysisId);
+    const repoStates = this.manager.getRepoStates(this.variantAnalysisId);
     if (repoStates.length === 0) {
       return;
     }

extensions/ql-vscode/src/view/common/SuggestBox/useEffectEvent.ts

@@ -9,6 +9,7 @@ import { useCallback, useInsertionEffect, useRef } from "react";
  *
  * @param callback The callback to call when the event is triggered.
  */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function useEffectEvent<T extends (...args: any[]) => any>(callback: T) {
   const ref = useRef<T>(callback);
 

extensions/ql-vscode/src/view/method-modeling/MethodModeling.tsx

@@ -8,6 +8,7 @@ import { VSCodeTag } from "@vscode/webview-ui-toolkit/react";
 import { ReviewInEditorButton } from "./ReviewInEditorButton";
 import { MultipleModeledMethodsPanel } from "./MultipleModeledMethodsPanel";
 import type { QueryLanguage } from "../../common/query-language";
+import type { ModelConfig } from "../../model-editor/languages";
 
 const Container = styled.div`
   padding-top: 0.5rem;
@@ -50,19 +51,23 @@ const UnsavedTag = ({ modelingStatus }: { modelingStatus: ModelingStatus }) => (
 
 export type MethodModelingProps = {
   language: QueryLanguage;
+  modelConfig: ModelConfig;
   modelingStatus: ModelingStatus;
   method: Method;
   modeledMethods: ModeledMethod[];
   isModelingInProgress: boolean;
+  isProcessedByAutoModel: boolean;
   onChange: (methodSignature: string, modeledMethods: ModeledMethod[]) => void;
 };
 
 export const MethodModeling = ({
   language,
+  modelConfig,
   modelingStatus,
   modeledMethods,
   method,
   isModelingInProgress,
+  isProcessedByAutoModel,
   onChange,
 }: MethodModelingProps): React.JSX.Element => {
   return (
@@ -78,9 +83,11 @@ export const MethodModeling = ({
       </DependencyContainer>
       <MultipleModeledMethodsPanel
         language={language}
+        modelConfig={modelConfig}
         method={method}
         modeledMethods={modeledMethods}
         isModelingInProgress={isModelingInProgress}
+        isProcessedByAutoModel={isProcessedByAutoModel}
         modelingStatus={modelingStatus}
         onChange={onChange}
       />

extensions/ql-vscode/src/view/method-modeling/MethodModelingInputs.tsx

@@ -7,7 +7,7 @@ import { ModelOutputDropdown } from "../model-editor/ModelOutputDropdown";
 import { ModelKindDropdown } from "../model-editor/ModelKindDropdown";
 import { InProgressDropdown } from "../model-editor/InProgressDropdown";
 import type { QueryLanguage } from "../../common/query-language";
-import type { ModelingStatus } from "../../model-editor/shared/modeling-status";
+import type { ModelConfig } from "../../model-editor/languages";
 
 const Container = styled.div`
   padding-top: 0.5rem;
@@ -25,18 +25,20 @@ const Name = styled.span`
 
 export type MethodModelingInputsProps = {
   language: QueryLanguage;
+  modelConfig: ModelConfig;
   method: Method;
   modeledMethod: ModeledMethod | undefined;
-  modelingStatus: ModelingStatus;
+  modelPending: boolean;
   isModelingInProgress: boolean;
   onChange: (modeledMethod: ModeledMethod) => void;
 };
 
 export const MethodModelingInputs = ({
   language,
+  modelConfig,
   method,
   modeledMethod,
-  modelingStatus,
+  modelPending,
   isModelingInProgress,
   onChange,
 }: MethodModelingInputsProps): React.JSX.Element => {
@@ -44,7 +46,7 @@ export const MethodModelingInputs = ({
     language,
     method,
     modeledMethod,
-    modelingStatus,
+    modelPending,
     onChange,
   };
 
@@ -56,7 +58,7 @@ export const MethodModelingInputs = ({
           {isModelingInProgress ? (
             <InProgressDropdown />
           ) : (
-            <ModelTypeDropdown {...inputProps} />
+            <ModelTypeDropdown modelConfig={modelConfig} {...inputProps} />
           )}
         </Input>
       </Container>

extensions/ql-vscode/src/view/method-modeling/MethodModelingView.tsx

@@ -11,6 +11,7 @@ import { NotInModelingMode } from "./NotInModelingMode";
 import { NoMethodSelected } from "./NoMethodSelected";
 import type { MethodModelingPanelViewState } from "../../model-editor/shared/view-state";
 import { MethodAlreadyModeled } from "./MethodAlreadyModeled";
+import { defaultModelConfig } from "../../model-editor/languages";
 
 type Props = {
   initialViewState?: MethodModelingPanelViewState;
@@ -33,6 +34,9 @@ export function MethodModelingView({
   const [isModelingInProgress, setIsModelingInProgress] =
     useState<boolean>(false);
 
+  const [isProcessedByAutoModel, setIsProcessedByAutoModel] =
+    useState<boolean>(false);
+
   const modelingStatus = useMemo(
     () => getModelingStatus(modeledMethods, isMethodModified),
     [modeledMethods, isMethodModified],
@@ -62,10 +66,15 @@ export function MethodModelingView({
             setMethod(msg.method);
             setModeledMethods(msg.modeledMethods);
             setIsMethodModified(msg.isModified);
+            setIsModelingInProgress(msg.isInProgress);
+            setIsProcessedByAutoModel(msg.processedByAutoModel);
             break;
           case "setInProgress":
             setIsModelingInProgress(msg.inProgress);
             break;
+          case "setProcessedByAutoModel":
+            setIsProcessedByAutoModel(msg.processedByAutoModel);
+            break;
           default:
             assertNever(msg);
         }
@@ -108,10 +117,12 @@ export function MethodModelingView({
   return (
     <MethodModeling
       language={viewState?.language}
+      modelConfig={viewState?.modelConfig ?? defaultModelConfig}
       modelingStatus={modelingStatus}
       method={method}
       modeledMethods={modeledMethods}
       isModelingInProgress={isModelingInProgress}
+      isProcessedByAutoModel={isProcessedByAutoModel}
       onChange={onChange}
     />
   );

extensions/ql-vscode/src/view/method-modeling/MultipleModeledMethodsPanel.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import type { Method } from "../../model-editor/method";
 import type { ModeledMethod } from "../../model-editor/modeled-method";
+import { isModelPending } from "../../model-editor/modeled-method";
 import {
   canAddNewModeledMethod,
   canRemoveModeledMethod,
@@ -15,13 +16,16 @@ import type { QueryLanguage } from "../../common/query-language";
 import { createEmptyModeledMethod } from "../../model-editor/modeled-method-empty";
 import { sendTelemetry } from "../common/telemetry";
 import type { ModelingStatus } from "../../model-editor/shared/modeling-status";
+import type { ModelConfig } from "../../model-editor/languages";
 
 export type MultipleModeledMethodsPanelProps = {
   language: QueryLanguage;
+  modelConfig: ModelConfig;
   method: Method;
   modeledMethods: ModeledMethod[];
   modelingStatus: ModelingStatus;
   isModelingInProgress: boolean;
+  isProcessedByAutoModel: boolean;
   onChange: (methodSignature: string, modeledMethods: ModeledMethod[]) => void;
 };
 
@@ -59,10 +63,12 @@ const ModificationActions = styled.div`
 
 export const MultipleModeledMethodsPanel = ({
   language,
+  modelConfig,
   method,
   modeledMethods,
   modelingStatus,
   isModelingInProgress,
+  isProcessedByAutoModel,
   onChange,
 }: MultipleModeledMethodsPanelProps) => {
   const [selectedIndex, setSelectedIndex] = useState<number>(0);
@@ -154,18 +160,28 @@ export const MultipleModeledMethodsPanel = ({
       {modeledMethods.length > 0 ? (
         <MethodModelingInputs
           language={language}
+          modelConfig={modelConfig}
           method={method}
           modeledMethod={modeledMethods[selectedIndex]}
-          modelingStatus={modelingStatus}
+          modelPending={isModelPending(
+            modeledMethods[selectedIndex],
+            modelingStatus,
+            isProcessedByAutoModel,
+          )}
           isModelingInProgress={isModelingInProgress}
           onChange={handleChange}
         />
       ) : (
         <MethodModelingInputs
           language={language}
+          modelConfig={modelConfig}
           method={method}
           modeledMethod={undefined}
-          modelingStatus={modelingStatus}
+          modelPending={isModelPending(
+            modeledMethods[selectedIndex],
+            modelingStatus,
+            isProcessedByAutoModel,
+          )}
           isModelingInProgress={isModelingInProgress}
           onChange={handleChange}
         />

extensions/ql-vscode/src/view/method-modeling/__tests__/MethodModeling.spec.tsx

@@ -4,6 +4,7 @@ import { MethodModeling } from "../MethodModeling";
 import { createMethod } from "../../../../test/factories/model-editor/method-factories";
 import { createSinkModeledMethod } from "../../../../test/factories/model-editor/modeled-method-factories";
 import { QueryLanguage } from "../../../common/query-language";
+import { defaultModelConfig } from "../../../model-editor/languages";
 
 describe(MethodModeling.name, () => {
   const render = (props: MethodModelingProps) =>
@@ -13,14 +14,17 @@ describe(MethodModeling.name, () => {
     const method = createMethod();
     const modeledMethod = createSinkModeledMethod();
     const isModelingInProgress = false;
+    const isProcessedByAutoModel = false;
     const onChange = jest.fn();
 
     render({
       language: QueryLanguage.Java,
+      modelConfig: defaultModelConfig,
       modelingStatus: "saved",
       method,
       modeledMethods: [modeledMethod],
       isModelingInProgress,
+      isProcessedByAutoModel,
       onChange,
     });
 

extensions/ql-vscode/src/view/method-modeling/__tests__/MethodModelingInputs.spec.tsx

@@ -9,6 +9,7 @@ import {
 } from "../../../../test/factories/model-editor/modeled-method-factories";
 import { QueryLanguage } from "../../../common/query-language";
 import { createEmptyModeledMethod } from "../../../model-editor/modeled-method-empty";
+import { defaultModelConfig } from "../../../model-editor/languages";
 
 describe(MethodModelingInputs.name, () => {
   const render = (props: MethodModelingInputsProps) =>
@@ -17,8 +18,9 @@ describe(MethodModelingInputs.name, () => {
   const language = QueryLanguage.Java;
   const method = createMethod();
   const modeledMethod = createSinkModeledMethod();
-  const modelingStatus = "unmodeled";
+  const modelPending = false;
   const isModelingInProgress = false;
+  const modelConfig = defaultModelConfig;
   const onChange = jest.fn();
 
   it("renders the method modeling inputs", () => {
@@ -26,8 +28,9 @@ describe(MethodModelingInputs.name, () => {
       language,
       method,
       modeledMethod,
-      modelingStatus,
+      modelPending,
       isModelingInProgress,
+      modelConfig,
       onChange,
     });
 
@@ -53,8 +56,9 @@ describe(MethodModelingInputs.name, () => {
       language,
       method,
       modeledMethod,
-      modelingStatus,
+      modelPending,
       isModelingInProgress,
+      modelConfig,
       onChange,
     });
 
@@ -76,8 +80,9 @@ describe(MethodModelingInputs.name, () => {
       language,
       method,
       modeledMethod,
-      modelingStatus,
+      modelPending,
       isModelingInProgress,
+      modelConfig,
       onChange,
     });
 
@@ -91,8 +96,9 @@ describe(MethodModelingInputs.name, () => {
         language={language}
         method={method}
         modeledMethod={updatedModeledMethod}
-        modelingStatus={modelingStatus}
+        modelPending={modelPending}
         isModelingInProgress={isModelingInProgress}
+        modelConfig={modelConfig}
         onChange={onChange}
       />,
     );
@@ -121,8 +127,9 @@ describe(MethodModelingInputs.name, () => {
       language,
       method,
       modeledMethod,
-      modelingStatus,
+      modelPending,
       isModelingInProgress: true,
+      modelConfig,
       onChange,
     });
 

extensions/ql-vscode/src/view/method-modeling/__tests__/MultipleModeledMethodsPanel.spec.tsx

@@ -10,29 +10,46 @@ import { MultipleModeledMethodsPanel } from "../MultipleModeledMethodsPanel";
 import { userEvent } from "@testing-library/user-event";
 import type { ModeledMethod } from "../../../model-editor/modeled-method";
 import { QueryLanguage } from "../../../common/query-language";
+import type { ModelingStatus } from "../../../model-editor/shared/modeling-status";
+import { defaultModelConfig } from "../../../model-editor/languages";
 
 describe(MultipleModeledMethodsPanel.name, () => {
-  const render = (props: MultipleModeledMethodsPanelProps) =>
-    reactRender(<MultipleModeledMethodsPanel {...props} />);
-
   const language = QueryLanguage.Java;
   const method = createMethod();
   const isModelingInProgress = false;
-  const modelingStatus = "unmodeled";
+  const isProcessedByAutoModel = false;
+  const modelingStatus: ModelingStatus = "unmodeled";
   const onChange = jest.fn<void, [string, ModeledMethod[]]>();
+  const modelConfig = defaultModelConfig;
+
+  const baseProps = {
+    language,
+    method,
+    modelingStatus,
+    isModelingInProgress,
+    modelConfig,
+    isProcessedByAutoModel,
+    onChange,
+  };
+
+  const createRender =
+    (modeledMethods: ModeledMethod[]) =>
+    (props: Partial<MultipleModeledMethodsPanelProps> = {}) =>
+      reactRender(
+        <MultipleModeledMethodsPanel
+          {...baseProps}
+          modeledMethods={modeledMethods}
+          {...props}
+        />,
+      );
 
   describe("with no modeled methods", () => {
     const modeledMethods: ModeledMethod[] = [];
 
+    const render = createRender(modeledMethods);
+
     it("renders the method modeling inputs once", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       expect(screen.getAllByRole("combobox")).toHaveLength(4);
       expect(
@@ -43,14 +60,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("disables all pagination", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       expect(
         screen
@@ -65,14 +75,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("cannot add or delete modeling", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       expect(
         screen
@@ -95,15 +98,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
       }),
     ];
 
+    const render = createRender(modeledMethods);
+
     it("renders the method modeling inputs once", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       expect(screen.getAllByRole("combobox")).toHaveLength(4);
       expect(
@@ -114,14 +112,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("disables all pagination", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       expect(
         screen
@@ -135,14 +126,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("cannot delete modeling", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       expect(
         screen
@@ -152,14 +136,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can add modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Add modeling"));
 
@@ -178,27 +155,16 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("changes selection to the newly added modeling", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        modelingStatus,
-        isModelingInProgress,
-        onChange,
-      });
+      const { rerender } = render();
 
       await userEvent.click(screen.getByLabelText("Add modeling"));
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={
             onChange.mock.calls[onChange.mock.calls.length - 1][1]
           }
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -216,15 +182,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
       }),
     ];
 
+    const render = createRender(modeledMethods);
+
     it("renders the method modeling inputs once", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(screen.getAllByRole("combobox")).toHaveLength(4);
       expect(
@@ -235,14 +196,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("renders the pagination", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(screen.getByLabelText("Previous modeling")).toBeInTheDocument();
       expect(screen.getByLabelText("Next modeling")).toBeInTheDocument();
@@ -250,14 +204,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("disables the correct pagination", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(
         screen
@@ -270,14 +217,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can use the pagination", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Next modeling"));
 
@@ -307,25 +247,14 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("correctly updates selected pagination index when the number of models decreases", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      const { rerender } = render();
 
       await userEvent.click(screen.getByLabelText("Next modeling"));
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={[modeledMethods[1]]}
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -338,27 +267,13 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("does not show errors", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(screen.queryByRole("alert")).not.toBeInTheDocument();
     });
 
     it("can update the first modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       const modelTypeDropdown = screen.getByRole("combobox", {
         name: "Model type",
@@ -384,14 +299,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can update the second modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Next modeling"));
 
@@ -419,14 +327,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can delete modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Delete modeling"));
 
@@ -437,14 +338,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can add modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Add modeling"));
 
@@ -463,27 +357,16 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("shows an error when adding a neutral modeling", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      const { rerender } = render();
 
       await userEvent.click(screen.getByLabelText("Add modeling"));
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={
             onChange.mock.calls[onChange.mock.calls.length - 1][1]
           }
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -497,14 +380,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={
             onChange.mock.calls[onChange.mock.calls.length - 1][1]
           }
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -516,14 +395,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={
             onChange.mock.calls[onChange.mock.calls.length - 1][1]
           }
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -534,14 +409,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("changes selection to the newly added modeling", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      const { rerender } = render();
 
       expect(screen.getByText("1/2")).toBeInTheDocument();
 
@@ -549,14 +417,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={
             onChange.mock.calls[onChange.mock.calls.length - 1][1]
           }
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -583,15 +447,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
       }),
     ];
 
+    const render = createRender(modeledMethods);
+
     it("can use the pagination", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(
         screen
@@ -675,25 +534,14 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("preserves selection when a modeling other than the selected modeling is removed", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      const { rerender } = render();
 
       expect(screen.getByText("1/3")).toBeInTheDocument();
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={modeledMethods.slice(0, 2)}
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -701,14 +549,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("reduces selection when the selected modeling is removed", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      const { rerender } = render();
 
       await userEvent.click(screen.getByLabelText("Next modeling"));
       await userEvent.click(screen.getByLabelText("Next modeling"));
@@ -716,12 +557,8 @@ describe(MultipleModeledMethodsPanel.name, () => {
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={modeledMethods.slice(0, 2)}
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -742,15 +579,10 @@ describe(MultipleModeledMethodsPanel.name, () => {
       }),
     ];
 
+    const render = createRender(modeledMethods);
+
     it("can add modeling", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(
         screen.getByLabelText("Add modeling").getElementsByTagName("input")[0],
@@ -758,14 +590,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can delete first modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Delete modeling"));
 
@@ -776,14 +601,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can delete second modeling", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       await userEvent.click(screen.getByLabelText("Next modeling"));
       await userEvent.click(screen.getByLabelText("Delete modeling"));
@@ -795,14 +613,7 @@ describe(MultipleModeledMethodsPanel.name, () => {
     });
 
     it("can add modeling after deleting second modeling", async () => {
-      const { rerender } = render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      const { rerender } = render();
 
       await userEvent.click(screen.getByLabelText("Next modeling"));
       await userEvent.click(screen.getByLabelText("Delete modeling"));
@@ -814,12 +625,8 @@ describe(MultipleModeledMethodsPanel.name, () => {
 
       rerender(
         <MultipleModeledMethodsPanel
-          language={language}
-          method={method}
+          {...baseProps}
           modeledMethods={modeledMethods.slice(0, 1)}
-          isModelingInProgress={isModelingInProgress}
-          modelingStatus={modelingStatus}
-          onChange={onChange}
         />,
       );
 
@@ -851,28 +658,16 @@ describe(MultipleModeledMethodsPanel.name, () => {
       }),
     ];
 
+    const render = createRender(modeledMethods);
+
     it("shows errors", () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(screen.getByRole("alert")).toBeInTheDocument();
     });
 
     it("shows the correct error message", async () => {
-      render({
-        language,
-        method,
-        modeledMethods,
-        isModelingInProgress,
-        modelingStatus,
-        onChange,
-      });
+      render();
 
       expect(
         screen.getByText("Error: Duplicated classification"),

extensions/ql-vscode/src/view/model-editor/InputDropdown.tsx

@@ -1,6 +1,6 @@
 import { styled } from "styled-components";
 import { Dropdown } from "../common/Dropdown";
 
-export const InputDropdown = styled(Dropdown)<{ $accepted: boolean }>`
-  font-style: ${(props) => (props.$accepted ? "normal" : "italic")};
+export const InputDropdown = styled(Dropdown)<{ $pending: boolean }>`
+  font-style: ${(props) => (props.$pending ? "italic" : "normal")};
 `;

extensions/ql-vscode/src/view/model-editor/LibraryRow.tsx

@@ -14,6 +14,7 @@ import {
 } from "@vscode/webview-ui-toolkit/react";
 import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
 import type { AccessPathSuggestionOptions } from "../../model-editor/suggestions";
+import { getCandidates } from "../../model-editor/shared/auto-model-candidates";
 
 const LibraryContainer = styled.div`
   background-color: var(--vscode-peekViewResult-background);
@@ -74,6 +75,7 @@ export type LibraryRowProps = {
   modifiedSignatures: Set<string>;
   selectedSignatures: Set<string>;
   inProgressMethods: Set<string>;
+  processedByAutoModelMethods: Set<string>;
   viewState: ModelEditorViewState;
   hideModeledMethods: boolean;
   revealedMethodSignature: string | null;
@@ -98,6 +100,7 @@ export const LibraryRow = ({
   modifiedSignatures,
   selectedSignatures,
   inProgressMethods,
+  processedByAutoModelMethods,
   viewState,
   hideModeledMethods,
   revealedMethodSignature,
@@ -184,6 +187,17 @@ export const LibraryRow = ({
     return methods.some((method) => inProgressMethods.has(method.signature));
   }, [methods, inProgressMethods]);
 
+  const modelWithAIDisabled = useMemo(() => {
+    return (
+      getCandidates(
+        viewState.mode,
+        methods,
+        modeledMethodsMap,
+        processedByAutoModelMethods,
+      ).length === 0
+    );
+  }, [methods, modeledMethodsMap, processedByAutoModelMethods, viewState.mode]);
+
   return (
     <LibraryContainer>
       <TitleContainer onClick={toggleExpanded} aria-expanded={isExpanded}>
@@ -203,7 +217,11 @@ export const LibraryRow = ({
           {hasUnsavedChanges ? <VSCodeTag>UNSAVED</VSCodeTag> : null}
         </NameContainer>
         {viewState.showLlmButton && !canStopAutoModeling && (
-          <VSCodeButton appearance="icon" onClick={handleModelWithAI}>
+          <VSCodeButton
+            appearance="icon"
+            disabled={modelWithAIDisabled}
+            onClick={handleModelWithAI}
+          >
             <Codicon name="lightbulb-autofix" label="Model with AI" />
             &nbsp;Model with AI
           </VSCodeButton>
@@ -237,6 +255,7 @@ export const LibraryRow = ({
             modifiedSignatures={modifiedSignatures}
             selectedSignatures={selectedSignatures}
             inProgressMethods={inProgressMethods}
+            processedByAutoModelMethods={processedByAutoModelMethods}
             viewState={viewState}
             hideModeledMethods={hideModeledMethods}
             revealedMethodSignature={revealedMethodSignature}

extensions/ql-vscode/src/view/model-editor/MethodRow.tsx

@@ -16,6 +16,7 @@ import { vscode } from "../vscode-api";
 
 import type { Method } from "../../model-editor/method";
 import type { ModeledMethod } from "../../model-editor/modeled-method";
+import { isModelPending } from "../../model-editor/modeled-method";
 import { ModelKindDropdown } from "./ModelKindDropdown";
 import { Mode } from "../../model-editor/shared/mode";
 import { MethodClassifications } from "./MethodClassifications";
@@ -36,6 +37,7 @@ import { createEmptyModeledMethod } from "../../model-editor/modeled-method-empt
 import type { AccessPathOption } from "../../model-editor/suggestions";
 import { ModelInputSuggestBox } from "./ModelInputSuggestBox";
 import { ModelOutputSuggestBox } from "./ModelOutputSuggestBox";
+import { getModelsAsDataLanguage } from "../../model-editor/languages";
 
 const ApiOrMethodRow = styled.div`
   min-height: calc(var(--input-height) * 1px);
@@ -75,6 +77,7 @@ export type MethodRowProps = {
   methodIsUnsaved: boolean;
   methodIsSelected: boolean;
   modelingInProgress: boolean;
+  processedByAutoModel: boolean;
   viewState: ModelEditorViewState;
   revealedMethodSignature: string | null;
   inputAccessPathSuggestions?: AccessPathOption[];
@@ -111,6 +114,7 @@ const ModelableMethodRow = forwardRef<HTMLElement | undefined, MethodRowProps>(
       modeledMethods: modeledMethodsProp,
       methodIsUnsaved,
       methodIsSelected,
+      processedByAutoModel,
       viewState,
       revealedMethodSignature,
       inputAccessPathSuggestions,
@@ -189,9 +193,37 @@ const ModelableMethodRow = forwardRef<HTMLElement | undefined, MethodRowProps>(
       [method],
     );
 
-    const modelingStatus = getModelingStatus(modeledMethods, methodIsUnsaved);
+    // Only show modeled methods that are non-hidden. These are also the ones that are
+    // used for determining the modeling status.
+    const shownModeledMethods = useMemo(() => {
+      const modelsAsDataLanguage = getModelsAsDataLanguage(viewState.language);
 
-    const addModelButtonDisabled = !canAddNewModeledMethod(modeledMethods);
+      return modeledMethodsToDisplay(
+        modeledMethods.filter((modeledMethod) => {
+          if (modeledMethod.type === "none") {
+            return true;
+          }
+
+          const predicate = modelsAsDataLanguage.predicates[modeledMethod.type];
+          if (!predicate) {
+            return true;
+          }
+
+          return !predicate.isHidden?.({
+            method,
+            config: viewState.modelConfig,
+          });
+        }),
+        method,
+      );
+    }, [method, modeledMethods, viewState]);
+
+    const modelingStatus = getModelingStatus(
+      shownModeledMethods,
+      methodIsUnsaved,
+    );
+
+    const addModelButtonDisabled = !canAddNewModeledMethod(shownModeledMethods);
 
     return (
       <DataGridRow
@@ -203,7 +235,7 @@ const ModelableMethodRow = forwardRef<HTMLElement | undefined, MethodRowProps>(
         }}
       >
         <DataGridCell
-          gridRow={`span ${modeledMethods.length + validationErrors.length}`}
+          gridRow={`span ${shownModeledMethods.length + validationErrors.length}`}
           ref={ref}
         >
           <ApiOrMethodRow>
@@ -254,88 +286,97 @@ const ModelableMethodRow = forwardRef<HTMLElement | undefined, MethodRowProps>(
         )}
         {!props.modelingInProgress && (
           <>
-            {modeledMethods.map((modeledMethod, index) => (
-              <DataGridRow key={index} focused={focusedIndex === index}>
-                <DataGridCell>
-                  <ModelTypeDropdown
-                    language={viewState.language}
-                    method={method}
-                    modeledMethod={modeledMethod}
-                    modelingStatus={modelingStatus}
-                    onChange={modeledMethodChangedHandlers[index]}
-                  />
-                </DataGridCell>
-                <DataGridCell>
-                  {inputAccessPathSuggestions === undefined ? (
-                    <ModelInputDropdown
+            {shownModeledMethods.map((modeledMethod, index) => {
+              const modelPending = isModelPending(
+                modeledMethod,
+                modelingStatus,
+                processedByAutoModel,
+              );
+
+              return (
+                <DataGridRow key={index} focused={focusedIndex === index}>
+                  <DataGridCell>
+                    <ModelTypeDropdown
                       language={viewState.language}
+                      modelConfig={viewState.modelConfig}
                       method={method}
                       modeledMethod={modeledMethod}
-                      modelingStatus={modelingStatus}
+                      modelPending={modelPending}
                       onChange={modeledMethodChangedHandlers[index]}
                     />
-                  ) : (
-                    <ModelInputSuggestBox
-                      modeledMethod={modeledMethod}
-                      suggestions={inputAccessPathSuggestions}
-                      typePathSuggestions={outputAccessPathSuggestions ?? []}
-                      onChange={modeledMethodChangedHandlers[index]}
-                    />
-                  )}
-                </DataGridCell>
-                <DataGridCell>
-                  {outputAccessPathSuggestions === undefined ? (
-                    <ModelOutputDropdown
+                  </DataGridCell>
+                  <DataGridCell>
+                    {inputAccessPathSuggestions === undefined ? (
+                      <ModelInputDropdown
+                        language={viewState.language}
+                        method={method}
+                        modeledMethod={modeledMethod}
+                        modelPending={modelPending}
+                        onChange={modeledMethodChangedHandlers[index]}
+                      />
+                    ) : (
+                      <ModelInputSuggestBox
+                        modeledMethod={modeledMethod}
+                        suggestions={inputAccessPathSuggestions}
+                        typePathSuggestions={outputAccessPathSuggestions ?? []}
+                        onChange={modeledMethodChangedHandlers[index]}
+                      />
+                    )}
+                  </DataGridCell>
+                  <DataGridCell>
+                    {outputAccessPathSuggestions === undefined ? (
+                      <ModelOutputDropdown
+                        language={viewState.language}
+                        method={method}
+                        modeledMethod={modeledMethod}
+                        modelPending={modelPending}
+                        onChange={modeledMethodChangedHandlers[index]}
+                      />
+                    ) : (
+                      <ModelOutputSuggestBox
+                        modeledMethod={modeledMethod}
+                        suggestions={outputAccessPathSuggestions}
+                        onChange={modeledMethodChangedHandlers[index]}
+                      />
+                    )}
+                  </DataGridCell>
+                  <DataGridCell>
+                    <ModelKindDropdown
                       language={viewState.language}
-                      method={method}
                       modeledMethod={modeledMethod}
-                      modelingStatus={modelingStatus}
+                      modelPending={modelPending}
                       onChange={modeledMethodChangedHandlers[index]}
                     />
-                  ) : (
-                    <ModelOutputSuggestBox
-                      modeledMethod={modeledMethod}
-                      suggestions={outputAccessPathSuggestions}
-                      onChange={modeledMethodChangedHandlers[index]}
-                    />
-                  )}
-                </DataGridCell>
-                <DataGridCell>
-                  <ModelKindDropdown
-                    language={viewState.language}
-                    modeledMethod={modeledMethod}
-                    modelingStatus={modelingStatus}
-                    onChange={modeledMethodChangedHandlers[index]}
-                  />
-                </DataGridCell>
-                <DataGridCell>
-                  {index === 0 ? (
-                    <CodiconRow
-                      appearance="icon"
-                      aria-label="Add new model"
-                      onClick={(event: React.MouseEvent) => {
-                        event.stopPropagation();
-                        handleAddModelClick();
-                      }}
-                      disabled={addModelButtonDisabled}
-                    >
-                      <Codicon name="add" />
-                    </CodiconRow>
-                  ) : (
-                    <CodiconRow
-                      appearance="icon"
-                      aria-label="Remove model"
-                      onClick={(event: React.MouseEvent) => {
-                        event.stopPropagation();
-                        removeModelClickedHandlers[index]();
-                      }}
-                    >
-                      <Codicon name="trash" />
-                    </CodiconRow>
-                  )}
-                </DataGridCell>
-              </DataGridRow>
-            ))}
+                  </DataGridCell>
+                  <DataGridCell>
+                    {index === 0 ? (
+                      <CodiconRow
+                        appearance="icon"
+                        aria-label="Add new model"
+                        onClick={(event: React.MouseEvent) => {
+                          event.stopPropagation();
+                          handleAddModelClick();
+                        }}
+                        disabled={addModelButtonDisabled}
+                      >
+                        <Codicon name="add" />
+                      </CodiconRow>
+                    ) : (
+                      <CodiconRow
+                        appearance="icon"
+                        aria-label="Remove model"
+                        onClick={(event: React.MouseEvent) => {
+                          event.stopPropagation();
+                          removeModelClickedHandlers[index]();
+                        }}
+                      >
+                        <Codicon name="trash" />
+                      </CodiconRow>
+                    )}
+                  </DataGridCell>
+                </DataGridRow>
+              );
+            })}
             {validationErrors.map((error, index) => (
               <DataGridCell gridColumn="span 5" key={index}>
                 <ModeledMethodAlert

extensions/ql-vscode/src/view/model-editor/ModelEditor.tsx

@@ -3,6 +3,7 @@ import type { ToModelEditorMessage } from "../../common/interface-types";
 import {
   VSCodeButton,
   VSCodeCheckbox,
+  VSCodeProgressRing,
   VSCodeTag,
 } from "@vscode/webview-ui-toolkit/react";
 import { styled } from "styled-components";
@@ -19,6 +20,8 @@ import { Mode } from "../../model-editor/shared/mode";
 import { getLanguageDisplayName } from "../../common/query-language";
 import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "../../model-editor/shared/hide-modeled-methods";
 import type { AccessPathSuggestionOptions } from "../../model-editor/suggestions";
+import type { ModelEvaluationRunState } from "../../model-editor/shared/model-evaluation-run-state";
+import { modelEvaluationRunIsRunning } from "../../model-editor/shared/model-evaluation-run-state";
 
 const LoadingContainer = styled.div`
   text-align: center;
@@ -74,6 +77,57 @@ const ButtonsContainer = styled.div`
   margin-top: 1rem;
 `;
 
+const ProgressRing = styled(VSCodeProgressRing)`
+  width: 16px;
+  height: 16px;
+  margin-right: 5px;
+`;
+
+const ModelEvaluation = ({
+  viewState,
+  modeledMethods,
+  modifiedSignatures,
+  onStartEvaluation,
+  onStopEvaluation,
+  evaluationRun,
+}: {
+  viewState: ModelEditorViewState;
+  modeledMethods: Record<string, ModeledMethod[]>;
+  modifiedSignatures: Set<string>;
+  onStartEvaluation: () => void;
+  onStopEvaluation: () => void;
+  evaluationRun: ModelEvaluationRunState | undefined;
+}) => {
+  if (!viewState.showEvaluationUi) {
+    return null;
+  }
+
+  if (!evaluationRun || !modelEvaluationRunIsRunning(evaluationRun)) {
+    const customModelsExist = Object.values(modeledMethods).some(
+      (methods) => methods.filter((m) => m.type !== "none").length > 0,
+    );
+
+    const unsavedChanges = modifiedSignatures.size > 0;
+
+    return (
+      <VSCodeButton
+        onClick={onStartEvaluation}
+        appearance="secondary"
+        disabled={!customModelsExist || unsavedChanges}
+      >
+        Evaluate
+      </VSCodeButton>
+    );
+  } else {
+    return (
+      <VSCodeButton onClick={onStopEvaluation} appearance="secondary">
+        <ProgressRing />
+        Stop evaluation
+      </VSCodeButton>
+    );
+  }
+};
+
 type Props = {
   initialViewState?: ModelEditorViewState;
   initialMethods?: Method[];
@@ -103,6 +157,8 @@ export function ModelEditor({
   const [inProgressMethods, setInProgressMethods] = useState<Set<string>>(
     new Set(),
   );
+  const [processedByAutoModelMethods, setProcessedByAutoModelMethods] =
+    useState<Set<string>>(new Set());
 
   const [hideModeledMethods, setHideModeledMethods] = useState(
     initialHideModeledMethods,
@@ -112,6 +168,10 @@ export function ModelEditor({
     string | null
   >(null);
 
+  const [evaluationRun, setEvaluationRun] = useState<
+    ModelEvaluationRunState | undefined
+  >(undefined);
+
   useEffect(() => {
     vscode.postMessage({
       t: "hideModeledMethods",
@@ -138,8 +198,9 @@ export function ModelEditor({
           case "setMethods":
             setMethods(msg.methods);
             break;
-          case "setModeledMethods":
+          case "setModeledAndModifiedMethods":
             setModeledMethods(msg.methods);
+            setModifiedSignatures(new Set(msg.modifiedMethodSignatures));
             break;
           case "setModifiedMethods":
             setModifiedSignatures(new Set(msg.methodSignatures));
@@ -148,12 +209,19 @@ export function ModelEditor({
             setInProgressMethods(new Set(msg.methods));
             break;
           }
+          case "setProcessedByAutoModelMethods": {
+            setProcessedByAutoModelMethods(new Set(msg.methods));
+            break;
+          }
           case "revealMethod":
             setRevealedMethodSignature(msg.methodSignature);
             break;
           case "setAccessPathSuggestions":
             setAccessPathSuggestions(msg.accessPathSuggestions);
             break;
+          case "setModelEvaluationRun":
+            setEvaluationRun(msg.run);
+            break;
           default:
             assertNever(msg);
         }
@@ -248,6 +316,18 @@ export function ModelEditor({
     [selectedSignatures],
   );
 
+  const onStartEvaluation = useCallback(() => {
+    vscode.postMessage({
+      t: "startModelEvaluation",
+    });
+  }, []);
+
+  const onStopEvaluation = useCallback(() => {
+    vscode.postMessage({
+      t: "stopModelEvaluation",
+    });
+  }, []);
+
   const onGenerateFromSourceClick = useCallback(() => {
     vscode.postMessage({
       t: "generateMethod",
@@ -367,6 +447,14 @@ export function ModelEditor({
                     Generate
                   </VSCodeButton>
                 )}
+              <ModelEvaluation
+                viewState={viewState}
+                modeledMethods={modeledMethods}
+                modifiedSignatures={modifiedSignatures}
+                onStartEvaluation={onStartEvaluation}
+                onStopEvaluation={onStopEvaluation}
+                evaluationRun={evaluationRun}
+              />
             </ButtonsContainer>
           </HeaderRow>
         </HeaderColumn>
@@ -388,6 +476,7 @@ export function ModelEditor({
           modifiedSignatures={modifiedSignatures}
           selectedSignatures={selectedSignatures}
           inProgressMethods={inProgressMethods}
+          processedByAutoModelMethods={processedByAutoModelMethods}
           viewState={viewState}
           hideModeledMethods={hideModeledMethods}
           revealedMethodSignature={revealedMethodSignature}

extensions/ql-vscode/src/view/model-editor/ModelInputDropdown.tsx

@@ -3,13 +3,11 @@ import { useCallback, useMemo } from "react";
 import type { ModeledMethod } from "../../model-editor/modeled-method";
 import {
   calculateNewProvenance,
-  isModelAccepted,
   modeledMethodSupportsInput,
 } from "../../model-editor/modeled-method";
 import type { Method } from "../../model-editor/method";
 import type { QueryLanguage } from "../../common/query-language";
 import { getModelsAsDataLanguage } from "../../model-editor/languages";
-import type { ModelingStatus } from "../../model-editor/shared/modeling-status";
 import { InputDropdown } from "./InputDropdown";
 import { ModelTypeTextbox } from "./ModelTypeTextbox";
 
@@ -17,7 +15,7 @@ type Props = {
   language: QueryLanguage;
   method: Method;
   modeledMethod: ModeledMethod | undefined;
-  modelingStatus: ModelingStatus;
+  modelPending: boolean;
   onChange: (modeledMethod: ModeledMethod) => void;
 };
 
@@ -25,7 +23,7 @@ export const ModelInputDropdown = ({
   language,
   method,
   modeledMethod,
-  modelingStatus,
+  modelPending,
   onChange,
 }: Props): React.JSX.Element => {
   const options = useMemo(() => {
@@ -77,14 +75,12 @@ export const ModelInputDropdown = ({
     );
   }
 
-  const modelAccepted = isModelAccepted(modeledMethod, modelingStatus);
-
   return (
     <InputDropdown
       value={value}
       options={options}
       disabled={!enabled}
-      $accepted={modelAccepted}
+      $pending={modelPending}
       onChange={handleChange}
       aria-label="Input"
     />

extensions/ql-vscode/src/view/model-editor/ModelKindDropdown.tsx

@@ -6,25 +6,23 @@ import type {
 } from "../../model-editor/modeled-method";
 import {
   modeledMethodSupportsKind,
-  isModelAccepted,
   calculateNewProvenance,
 } from "../../model-editor/modeled-method";
 import { getModelsAsDataLanguage } from "../../model-editor/languages";
 import type { QueryLanguage } from "../../common/query-language";
-import type { ModelingStatus } from "../../model-editor/shared/modeling-status";
 import { InputDropdown } from "./InputDropdown";
 
 type Props = {
   language: QueryLanguage;
   modeledMethod: ModeledMethod | undefined;
-  modelingStatus: ModelingStatus;
+  modelPending: boolean;
   onChange: (modeledMethod: ModeledMethod) => void;
 };
 
 export const ModelKindDropdown = ({
   language,
   modeledMethod,
-  modelingStatus,
+  modelPending,
   onChange,
 }: Props) => {
   const predicate = useMemo(() => {
@@ -89,14 +87,12 @@ export const ModelKindDropdown = ({
     }
   }, [modeledMethod, value, kinds, onChangeKind]);
 
-  const modelAccepted = isModelAccepted(modeledMethod, modelingStatus);
-
   return (
     <InputDropdown
       value={value}
       options={options}
       disabled={disabled}
-      $accepted={modelAccepted}
+      $pending={modelPending}
       onChange={handleChange}
       aria-label="Kind"
     />

extensions/ql-vscode/src/view/model-editor/ModelOutputDropdown.tsx

@@ -3,13 +3,11 @@ import { useCallback, useMemo } from "react";
 import type { ModeledMethod } from "../../model-editor/modeled-method";
 import {
   calculateNewProvenance,
-  isModelAccepted,
   modeledMethodSupportsOutput,
 } from "../../model-editor/modeled-method";
 import type { Method } from "../../model-editor/method";
 import { getModelsAsDataLanguage } from "../../model-editor/languages";
 import type { QueryLanguage } from "../../common/query-language";
-import type { ModelingStatus } from "../../model-editor/shared/modeling-status";
 import { InputDropdown } from "./InputDropdown";
 import { ModelTypeTextbox } from "./ModelTypeTextbox";
 
@@ -17,7 +15,7 @@ type Props = {
   language: QueryLanguage;
   method: Method;
   modeledMethod: ModeledMethod | undefined;
-  modelingStatus: ModelingStatus;
+  modelPending: boolean;
   onChange: (modeledMethod: ModeledMethod) => void;
 };
 
@@ -25,7 +23,7 @@ export const ModelOutputDropdown = ({
   language,
   method,
   modeledMethod,
-  modelingStatus,
+  modelPending,
   onChange,
 }: Props): React.JSX.Element => {
   const options = useMemo(() => {
@@ -78,14 +76,12 @@ export const ModelOutputDropdown = ({
     );
   }
 
-  const modelAccepted = isModelAccepted(modeledMethod, modelingStatus);
-
   return (
     <InputDropdown
       value={value}
       options={options}
       disabled={!enabled}
-      $accepted={modelAccepted}
+      $pending={modelPending}
       onChange={handleChange}
       aria-label="Output"
     />

extensions/ql-vscode/src/view/model-editor/ModelTypeDropdown.tsx

@@ -4,25 +4,25 @@ import type {
   ModeledMethod,
   ModeledMethodType,
 } from "../../model-editor/modeled-method";
-import {
-  calculateNewProvenance,
-  isModelAccepted,
-} from "../../model-editor/modeled-method";
+import { calculateNewProvenance } from "../../model-editor/modeled-method";
 import type { Method } from "../../model-editor/method";
 import { createEmptyModeledMethod } from "../../model-editor/modeled-method-empty";
 import type { Mutable } from "../../common/mutable";
 import { ReadonlyDropdown } from "../common/ReadonlyDropdown";
 import type { QueryLanguage } from "../../common/query-language";
-import type { ModelsAsDataLanguagePredicates } from "../../model-editor/languages";
+import type {
+  ModelConfig,
+  ModelsAsDataLanguagePredicates,
+} from "../../model-editor/languages";
 import { getModelsAsDataLanguage } from "../../model-editor/languages";
-import type { ModelingStatus } from "../../model-editor/shared/modeling-status";
 import { InputDropdown } from "./InputDropdown";
 
 type Props = {
   language: QueryLanguage;
+  modelConfig: ModelConfig;
   method: Method;
   modeledMethod: ModeledMethod | undefined;
-  modelingStatus: ModelingStatus;
+  modelPending: boolean;
   onChange: (modeledMethod: ModeledMethod) => void;
 };
 
@@ -38,9 +38,10 @@ type Option = { value: ModeledMethodType; label: string };
 
 export const ModelTypeDropdown = ({
   language,
+  modelConfig,
   method,
   modeledMethod,
-  modelingStatus,
+  modelPending,
   onChange,
 }: Props): React.JSX.Element => {
   const options = useMemo(() => {
@@ -59,6 +60,13 @@ export const ModelTypeDropdown = ({
             return null;
           }
 
+          if (
+            predicate.isHidden &&
+            predicate.isHidden({ method, config: modelConfig })
+          ) {
+            return null;
+          }
+
           return {
             value: type,
             label: typeLabels[type],
@@ -68,7 +76,7 @@ export const ModelTypeDropdown = ({
     ];
 
     return baseOptions;
-  }, [language, method.endpointType]);
+  }, [language, modelConfig, method]);
 
   const handleChange = useCallback(
     (e: ChangeEvent<HTMLSelectElement>) => {
@@ -114,13 +122,11 @@ export const ModelTypeDropdown = ({
     );
   }
 
-  const modelAccepted = isModelAccepted(modeledMethod, modelingStatus);
-
   return (
     <InputDropdown
       value={modeledMethod?.type ?? "none"}
       options={options}
-      $accepted={modelAccepted}
+      $pending={modelPending}
       onChange={handleChange}
       aria-label="Model type"
     />

extensions/ql-vscode/src/view/model-editor/ModeledMethodDataGrid.tsx

@@ -3,7 +3,6 @@ import type { Method } from "../../model-editor/method";
 import { canMethodBeModeled } from "../../model-editor/method";
 import type { ModeledMethod } from "../../model-editor/modeled-method";
 import { useMemo } from "react";
-import { sortMethods } from "../../model-editor/shared/sorting";
 import { HiddenMethodsRow } from "./HiddenMethodsRow";
 import type { ModelEditorViewState } from "../../model-editor/shared/view-state";
 import { ScreenReaderOnly } from "../common/ScreenReaderOnly";
@@ -19,6 +18,7 @@ export type ModeledMethodDataGridProps = {
   modifiedSignatures: Set<string>;
   selectedSignatures: Set<string>;
   inProgressMethods: Set<string>;
+  processedByAutoModelMethods: Set<string>;
   viewState: ModelEditorViewState;
   hideModeledMethods: boolean;
   revealedMethodSignature: string | null;
@@ -33,6 +33,7 @@ export const ModeledMethodDataGrid = ({
   modifiedSignatures,
   selectedSignatures,
   inProgressMethods,
+  processedByAutoModelMethods,
   viewState,
   hideModeledMethods,
   revealedMethodSignature,
@@ -46,7 +47,7 @@ export const ModeledMethodDataGrid = ({
   ] = useMemo(() => {
     const methodsWithModelability = [];
     let numHiddenMethods = 0;
-    for (const method of sortMethods(methods)) {
+    for (const method of methods) {
       const modeledMethods = modeledMethodsMap[method.signature] ?? [];
       const methodIsUnsaved = modifiedSignatures.has(method.signature);
       const methodCanBeModeled = canMethodBeModeled(
@@ -93,6 +94,9 @@ export const ModeledMethodDataGrid = ({
                 methodIsUnsaved={modifiedSignatures.has(method.signature)}
                 methodIsSelected={selectedSignatures.has(method.signature)}
                 modelingInProgress={inProgressMethods.has(method.signature)}
+                processedByAutoModel={processedByAutoModelMethods.has(
+                  method.signature,
+                )}
                 viewState={viewState}
                 revealedMethodSignature={revealedMethodSignature}
                 inputAccessPathSuggestions={inputAccessPathSuggestions}

extensions/ql-vscode/src/view/model-editor/ModeledMethodsList.tsx

@@ -16,6 +16,7 @@ export type ModeledMethodsListProps = {
   modifiedSignatures: Set<string>;
   selectedSignatures: Set<string>;
   inProgressMethods: Set<string>;
+  processedByAutoModelMethods: Set<string>;
   revealedMethodSignature: string | null;
   accessPathSuggestions?: AccessPathSuggestionOptions;
   viewState: ModelEditorViewState;
@@ -42,6 +43,7 @@ export const ModeledMethodsList = ({
   modifiedSignatures,
   selectedSignatures,
   inProgressMethods,
+  processedByAutoModelMethods,
   viewState,
   hideModeledMethods,
   revealedMethodSignature,
@@ -91,6 +93,7 @@ export const ModeledMethodsList = ({
           modifiedSignatures={modifiedSignatures}
           selectedSignatures={selectedSignatures}
           inProgressMethods={inProgressMethods}
+          processedByAutoModelMethods={processedByAutoModelMethods}
           viewState={viewState}
           hideModeledMethods={hideModeledMethods}
           revealedMethodSignature={revealedMethodSignature}

extensions/ql-vscode/src/view/model-editor/__tests__/LibraryRow.spec.tsx

@@ -36,6 +36,7 @@ describe(LibraryRow.name, () => {
         modifiedSignatures={new Set([method.signature])}
         selectedSignatures={new Set()}
         inProgressMethods={new Set()}
+        processedByAutoModelMethods={new Set()}
         viewState={viewState}
         hideModeledMethods={false}
         revealedMethodSignature={null}

extensions/ql-vscode/src/view/model-editor/__tests__/MethodRow.spec.tsx

@@ -43,6 +43,7 @@ describe(MethodRow.name, () => {
         methodIsUnsaved={false}
         methodIsSelected={false}
         modelingInProgress={false}
+        processedByAutoModel={false}
         revealedMethodSignature={null}
         viewState={viewState}
         onChange={onChange}

extensions/ql-vscode/src/view/model-editor/__tests__/ModelKindDropdown.spec.tsx

@@ -24,7 +24,7 @@ describe(ModelKindDropdown.name, () => {
       <ModelKindDropdown
         language={QueryLanguage.Java}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
       />,
     );
@@ -47,7 +47,7 @@ describe(ModelKindDropdown.name, () => {
       <ModelKindDropdown
         language={QueryLanguage.Java}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
       />,
     );
@@ -64,7 +64,7 @@ describe(ModelKindDropdown.name, () => {
       <ModelKindDropdown
         language={QueryLanguage.Java}
         modeledMethod={updatedModeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
       />,
     );
@@ -82,7 +82,7 @@ describe(ModelKindDropdown.name, () => {
       <ModelKindDropdown
         language={QueryLanguage.Java}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
       />,
     );
@@ -102,7 +102,7 @@ describe(ModelKindDropdown.name, () => {
       <ModelKindDropdown
         language={QueryLanguage.Java}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
       />,
     );

extensions/ql-vscode/src/view/model-editor/__tests__/ModelTypeDropdown.spec.tsx

@@ -4,6 +4,7 @@ import { createNoneModeledMethod } from "../../../../test/factories/model-editor
 import { QueryLanguage } from "../../../common/query-language";
 import { ModelTypeDropdown } from "../ModelTypeDropdown";
 import { createMethod } from "../../../../test/factories/model-editor/method-factories";
+import { defaultModelConfig } from "../../../model-editor/languages";
 
 describe(ModelTypeDropdown.name, () => {
   const onChange = jest.fn();
@@ -20,9 +21,10 @@ describe(ModelTypeDropdown.name, () => {
       <ModelTypeDropdown
         language={QueryLanguage.Java}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
         method={method}
+        modelConfig={defaultModelConfig}
       />,
     );
 
@@ -34,7 +36,7 @@ describe(ModelTypeDropdown.name, () => {
     );
   });
 
-  it("allows changing the type to 'Type' for Ruby", async () => {
+  it("allows changing the type to 'Type' for Ruby when type models are shown", async () => {
     const method = createMethod();
     const modeledMethod = createNoneModeledMethod();
 
@@ -42,9 +44,13 @@ describe(ModelTypeDropdown.name, () => {
       <ModelTypeDropdown
         language={QueryLanguage.Ruby}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
         method={method}
+        modelConfig={{
+          ...defaultModelConfig,
+          showTypeModels: true,
+        }}
       />,
     );
 
@@ -56,6 +62,26 @@ describe(ModelTypeDropdown.name, () => {
     );
   });
 
+  it("does not allow changing the type to 'Type' for Ruby when type models are not shown", async () => {
+    const method = createMethod();
+    const modeledMethod = createNoneModeledMethod();
+
+    render(
+      <ModelTypeDropdown
+        language={QueryLanguage.Ruby}
+        modeledMethod={modeledMethod}
+        modelPending={false}
+        onChange={onChange}
+        method={method}
+        modelConfig={defaultModelConfig}
+      />,
+    );
+
+    expect(
+      screen.queryByRole("option", { name: "Type" }),
+    ).not.toBeInTheDocument();
+  });
+
   it("does not allow changing the type to 'Type' for Java", async () => {
     const method = createMethod();
     const modeledMethod = createNoneModeledMethod();
@@ -64,9 +90,10 @@ describe(ModelTypeDropdown.name, () => {
       <ModelTypeDropdown
         language={QueryLanguage.Java}
         modeledMethod={modeledMethod}
-        modelingStatus="unsaved"
+        modelPending={false}
         onChange={onChange}
         method={method}
+        modelConfig={defaultModelConfig}
       />,
     );
 

extensions/ql-vscode/src/view/model-editor/__tests__/ModeledMethodDataGrid.spec.tsx

@@ -58,6 +58,7 @@ describe(ModeledMethodDataGrid.name, () => {
         modifiedSignatures={new Set([method1.signature])}
         selectedSignatures={new Set()}
         inProgressMethods={new Set()}
+        processedByAutoModelMethods={new Set()}
         viewState={viewState}
         hideModeledMethods={false}
         revealedMethodSignature={null}

extensions/ql-vscode/src/view/model-editor/__tests__/ModeledMethodsList.spec.tsx

@@ -59,6 +59,7 @@ describe(ModeledMethodsList.name, () => {
         modifiedSignatures={new Set([method1.signature])}
         selectedSignatures={new Set()}
         inProgressMethods={new Set()}
+        processedByAutoModelMethods={new Set()}
         viewState={viewState}
         hideModeledMethods={false}
         revealedMethodSignature={null}

extensions/ql-vscode/src/view/results/ResultsApp.tsx

@@ -79,55 +79,29 @@ export function ResultsApp() {
 
   const updateStateWithNewResultsInfo = useCallback(
     (resultsInfo: ResultsInfo): void => {
-      setState((prevState) => {
-        if (resultsInfo === null && prevState.isExpectingResultsUpdate) {
-          // Display loading message
-          return {
-            ...prevState,
-            displayedResults: {
-              resultsInfo: null,
-              results: null,
-              errorMessage: "Loading results…",
-            },
-            nextResultsInfo: resultsInfo,
-          };
-        } else if (resultsInfo === null) {
-          // No results to display
-          return {
-            ...prevState,
-            displayedResults: {
-              resultsInfo: null,
-              results: null,
-              errorMessage: "No results to display",
-            },
-            nextResultsInfo: resultsInfo,
-          };
-        }
-
-        let results: Results | null = null;
-        let statusText = "";
-        try {
-          const resultSets = getResultSets(resultsInfo);
-          results = {
-            resultSets,
-            database: resultsInfo.database,
-            sortStates: getSortStates(resultsInfo),
-          };
-        } catch (e) {
-          const errorMessage = getErrorMessage(e);
-
-          statusText = `Error loading results: ${errorMessage}`;
-        }
-
-        return {
-          displayedResults: {
-            resultsInfo,
-            results,
-            errorMessage: statusText,
-          },
-          nextResultsInfo: null,
-          isExpectingResultsUpdate: false,
+      let results: Results | null = null;
+      let statusText = "";
+      try {
+        const resultSets = getResultSets(resultsInfo);
+        results = {
+          resultSets,
+          database: resultsInfo.database,
+          sortStates: getSortStates(resultsInfo),
         };
+      } catch (e) {
+        const errorMessage = getErrorMessage(e);
+
+        statusText = `Error loading results: ${errorMessage}`;
+      }
+
+      setState({
+        displayedResults: {
+          resultsInfo,
+          results,
+          errorMessage: statusText,
+        },
+        nextResultsInfo: null,
+        isExpectingResultsUpdate: false,
       });
     },
     [],
@@ -173,7 +147,7 @@ export function ResultsApp() {
               },
               selectedTable: tableName,
             },
-            origResultsPaths: undefined as any, // FIXME: Not used for interpreted, refactor so this is not needed
+            origResultsPaths: undefined as unknown as ResultsPaths, // FIXME: Not used for interpreted, refactor so this is not needed
             sortedResultsMap: new Map(), // FIXME: Not used for interpreted, refactor so this is not needed
             database: msg.database,
             interpretation: msg.interpretation,

extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisActions.tsx

@@ -103,6 +103,11 @@ export const VariantAnalysisActions = ({
           Stop query
         </Button>
       )}
+      {variantAnalysisStatus === VariantAnalysisStatus.Canceling && (
+        <Button appearance="secondary" disabled={true}>
+          Stopping query
+        </Button>
+      )}
     </Container>
   );
 };

extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisStats.tsx

@@ -48,6 +48,10 @@ export const VariantAnalysisStats = ({
       return "Failed";
     }
 
+    if (variantAnalysisStatus === VariantAnalysisStatus.Canceling) {
+      return "Canceling";
+    }
+
     if (variantAnalysisStatus === VariantAnalysisStatus.Canceled) {
       return "Stopped";
     }

extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisStatusStats.tsx

@@ -28,7 +28,8 @@ export const VariantAnalysisStatusStats = ({
 }: VariantAnalysisStatusStatsProps) => {
   return (
     <Container>
-      {variantAnalysisStatus === VariantAnalysisStatus.InProgress ? (
+      {variantAnalysisStatus === VariantAnalysisStatus.InProgress ||
+      variantAnalysisStatus === VariantAnalysisStatus.Canceling ? (
         <div>
           <Icon className="codicon codicon-loading codicon-modifier-spin" />
         </div>

extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisActions.spec.tsx

@@ -45,6 +45,24 @@ describe(VariantAnalysisActions.name, () => {
     expect(onStopQueryClick).toHaveBeenCalledTimes(1);
   });
 
+  it("renders the stopping query disabled button when canceling", async () => {
+    render({
+      variantAnalysisStatus: VariantAnalysisStatus.Canceling,
+    });
+
+    const button = screen.getByText("Stopping query");
+    expect(button).toBeInTheDocument();
+    expect(button.getElementsByTagName("input")[0]).toBeDisabled();
+  });
+
+  it("does not render a stop query button when canceling", async () => {
+    render({
+      variantAnalysisStatus: VariantAnalysisStatus.Canceling,
+    });
+
+    expect(screen.queryByText("Stop query")).not.toBeInTheDocument();
+  });
+
   it("renders 3 buttons when in progress with results", async () => {
     const { container } = render({
       variantAnalysisStatus: VariantAnalysisStatus.InProgress,

extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisStats.spec.tsx

@@ -160,6 +160,12 @@ describe(VariantAnalysisStats.name, () => {
     expect(screen.getByText("Failed")).toBeInTheDocument();
   });
 
+  it("renders 'Stopping' text when the variant analysis status is canceling", () => {
+    render({ variantAnalysisStatus: VariantAnalysisStatus.Canceling });
+
+    expect(screen.getByText("Canceling")).toBeInTheDocument();
+  });
+
   it("renders 'Stopped' text when the variant analysis status is canceled", () => {
     render({ variantAnalysisStatus: VariantAnalysisStatus.Canceled });
 

extensions/ql-vscode/supported_cli_versions.json

@@ -1,9 +1,8 @@
 [
+  "v2.16.3",
   "v2.16.2",
   "v2.15.5",
   "v2.14.6",
   "v2.13.5",
-  "v2.12.7",
-  "v2.11.6",
   "nightly"
 ]

extensions/ql-vscode/test/__mocks__/model-editor/modelingEventsMock.ts

@@ -7,22 +7,24 @@ export function createMockModelingEvents({
   onMethodsChanged = jest.fn(),
   onHideModeledMethodsChanged = jest.fn(),
   onModeChanged = jest.fn(),
-  onModeledMethodsChanged = jest.fn(),
-  onModifiedMethodsChanged = jest.fn(),
+  onModeledAndModifiedMethodsChanged = jest.fn(),
   onInProgressMethodsChanged = jest.fn(),
+  onProcessedByAutoModelMethodsChanged = jest.fn(),
   onRevealInModelEditor = jest.fn(),
   onFocusModelEditor = jest.fn(),
+  onModelEvaluationRunChanged = jest.fn(),
 }: {
   onActiveDbChanged?: ModelingEvents["onActiveDbChanged"];
   onDbClosed?: ModelingEvents["onDbClosed"];
   onMethodsChanged?: ModelingEvents["onMethodsChanged"];
   onHideModeledMethodsChanged?: ModelingEvents["onHideModeledMethodsChanged"];
   onModeChanged?: ModelingEvents["onModeChanged"];
-  onModeledMethodsChanged?: ModelingEvents["onModeledMethodsChanged"];
-  onModifiedMethodsChanged?: ModelingEvents["onModifiedMethodsChanged"];
+  onModeledAndModifiedMethodsChanged?: ModelingEvents["onModeledAndModifiedMethodsChanged"];
   onInProgressMethodsChanged?: ModelingEvents["onInProgressMethodsChanged"];
+  onProcessedByAutoModelMethodsChanged?: ModelingEvents["onProcessedByAutoModelMethodsChanged"];
   onRevealInModelEditor?: ModelingEvents["onRevealInModelEditor"];
   onFocusModelEditor?: ModelingEvents["onFocusModelEditor"];
+  onModelEvaluationRunChanged?: ModelingEvents["onModelEvaluationRunChanged"];
 } = {}): ModelingEvents {
   return mockedObject<ModelingEvents>({
     onActiveDbChanged,
@@ -30,10 +32,11 @@ export function createMockModelingEvents({
     onMethodsChanged,
     onHideModeledMethodsChanged,
     onModeChanged,
-    onModeledMethodsChanged,
-    onModifiedMethodsChanged,
+    onModeledAndModifiedMethodsChanged,
     onInProgressMethodsChanged,
+    onProcessedByAutoModelMethodsChanged,
     onRevealInModelEditor,
     onFocusModelEditor,
+    onModelEvaluationRunChanged,
   });
 }

extensions/ql-vscode/test/factories/model-editor/view-state.ts

@@ -2,6 +2,7 @@ import type { ModelEditorViewState } from "../../../src/model-editor/shared/view
 import { Mode } from "../../../src/model-editor/shared/mode";
 import { createMockExtensionPack } from "./extension-pack";
 import { QueryLanguage } from "../../../src/common/query-language";
+import { defaultModelConfig } from "../../../src/model-editor/languages";
 
 export function createMockModelEditorViewState(
   data: Partial<ModelEditorViewState> = {},
@@ -11,9 +12,11 @@ export function createMockModelEditorViewState(
     mode: Mode.Application,
     showGenerateButton: false,
     showLlmButton: false,
+    showEvaluationUi: false,
     showModeSwitchButton: true,
     extensionPack: createMockExtensionPack(),
     sourceArchiveAvailable: true,
+    modelConfig: defaultModelConfig,
     ...data,
   };
 }

extensions/ql-vscode/test/unit-tests/common/short-paths.test.ts

@@ -0,0 +1,96 @@
+import { platform } from "os";
+import type { BaseLogger } from "../../../src/common/logging";
+import { expandShortPaths } from "../../../src/common/short-paths";
+import { join } from "path";
+
+describe("expandShortPaths", () => {
+  let logger: BaseLogger;
+
+  beforeEach(() => {
+    logger = {
+      log: jest.fn(),
+    };
+  });
+
+  describe("on POSIX", () => {
+    if (platform() === "win32") {
+      console.log(`Skipping test on Windows`);
+      return;
+    }
+
+    it("should return the same path for non-Windows platforms", async () => {
+      const path = "/home/user/some~path";
+      const result = await expandShortPaths(path, logger);
+
+      expect(logger.log).not.toHaveBeenCalled();
+      expect(result).toBe(path);
+    });
+  });
+
+  describe("on Windows", () => {
+    if (platform() !== "win32") {
+      console.log(`Skipping test on non-Windows`);
+      return;
+    }
+
+    it("should return the same path if no short components", async () => {
+      const path = "C:\\Program Files\\Some Folder";
+      const result = await expandShortPaths(path, logger);
+
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short paths in: ${path}`,
+      );
+      expect(logger.log).toHaveBeenCalledWith(
+        "Skipping due to no short components",
+      );
+      expect(result).toBe(path);
+    });
+
+    it("should not attempt to expand long paths with '~' in the name", async () => {
+      const testDir = join(__dirname, "../data/short-paths");
+      const path = join(testDir, "textfile-with~tilde.txt");
+      const result = await expandShortPaths(path, logger);
+
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short paths in: ${path}`,
+      );
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short path component: textfile-with~tilde.txt`,
+      );
+      expect(logger.log).toHaveBeenCalledWith(`Component is not a short name`);
+      expect(result).toBe(join(testDir, "textfile-with~tilde.txt"));
+    });
+
+    it("should expand a short path", async () => {
+      const path = "C:\\PROGRA~1\\Some Folder";
+      const result = await expandShortPaths(path, logger);
+
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short paths in: ${path}`,
+      );
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short path component: PROGRA~1`,
+      );
+      expect(result).toBe("C:\\Program Files\\Some Folder");
+    });
+
+    it("should expand multiple short paths", async () => {
+      const testDir = join(__dirname, "../data/short-paths");
+      const path = join(testDir, "FOLDER~1", "TEXTFI~1.TXT");
+      const result = await expandShortPaths(path, logger);
+
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short paths in: ${path}`,
+      );
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short path component: FOLDER~1`,
+      );
+      expect(logger.log).toHaveBeenCalledWith(
+        `Expanding short path component: TEXTFI~1.TXT`,
+      );
+      expect(result).toBe(
+        join(testDir, "folder with space", ".textfile+extra.characters.txt"),
+      );
+    });
+  });
+});

extensions/ql-vscode/test/unit-tests/compare/sarif-diff.test.ts

@@ -138,6 +138,364 @@ describe("sarifDiff", () => {
     });
   });
 
+  it("does not take into account the location index when in thread flows or related locations", () => {
+    const result1: Result = {
+      ruleId: "java/static-initialization-vector",
+      ruleIndex: 0,
+      rule: {
+        id: "java/static-initialization-vector",
+        index: 0,
+      },
+      message: {
+        text: "A [static initialization vector](1) should not be used for encryption.",
+      },
+      locations: [
+        {
+          physicalLocation: {
+            artifactLocation: {
+              uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+              uriBaseId: "%SRCROOT%",
+              index: 126,
+            },
+            region: {
+              startLine: 1272,
+              startColumn: 55,
+              endColumn: 61,
+            },
+          },
+        },
+      ],
+      partialFingerprints: {
+        primaryLocationLineHash: "9a2a0c085da38206:3",
+        primaryLocationStartColumnFingerprint: "38",
+      },
+      codeFlows: [
+        {
+          threadFlows: [
+            {
+              locations: [
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 126,
+                      },
+                      region: {
+                        startLine: 1270,
+                        startColumn: 50,
+                        endColumn: 76,
+                      },
+                    },
+                    message: {
+                      text: "new byte[] : byte[]",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/javax/crypto/spec/IvParameterSpec.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 12,
+                      },
+                      region: {
+                        startLine: 52,
+                        startColumn: 28,
+                        endColumn: 37,
+                      },
+                    },
+                    message: {
+                      text: "iv : byte[]",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/javax/crypto/spec/IvParameterSpec.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 12,
+                      },
+                      region: {
+                        startLine: 53,
+                        startColumn: 14,
+                        endColumn: 16,
+                      },
+                    },
+                    message: {
+                      text: "iv : byte[]",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/javax/crypto/spec/IvParameterSpec.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 12,
+                      },
+                      region: {
+                        startLine: 53,
+                        startColumn: 9,
+                        endColumn: 32,
+                      },
+                    },
+                    message: {
+                      text: "this <constr(this)> [post update] : IvParameterSpec",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 126,
+                      },
+                      region: {
+                        startLine: 1270,
+                        startColumn: 30,
+                        endColumn: 77,
+                      },
+                    },
+                    message: {
+                      text: "new IvParameterSpec(...) : IvParameterSpec",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 126,
+                      },
+                      region: {
+                        startLine: 1272,
+                        startColumn: 55,
+                        endColumn: 61,
+                      },
+                    },
+                    message: {
+                      text: "params",
+                    },
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      ],
+      relatedLocations: [
+        {
+          id: 1,
+          physicalLocation: {
+            artifactLocation: {
+              uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+              uriBaseId: "%SRCROOT%",
+              index: 126,
+            },
+            region: {
+              startLine: 1270,
+              startColumn: 50,
+              endColumn: 76,
+            },
+          },
+          message: {
+            text: "static initialization vector",
+          },
+        },
+      ],
+    };
+    const result2: Result = {
+      ruleId: "java/static-initialization-vector",
+      ruleIndex: 0,
+      rule: {
+        id: "java/static-initialization-vector",
+        index: 0,
+      },
+      message: {
+        text: "A [static initialization vector](1) should not be used for encryption.",
+      },
+      locations: [
+        {
+          physicalLocation: {
+            artifactLocation: {
+              uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+              uriBaseId: "%SRCROOT%",
+              index: 141,
+            },
+            region: {
+              startLine: 1272,
+              startColumn: 55,
+              endColumn: 61,
+            },
+          },
+        },
+      ],
+      partialFingerprints: {
+        primaryLocationLineHash: "9a2a0c085da38206:3",
+        primaryLocationStartColumnFingerprint: "38",
+      },
+      codeFlows: [
+        {
+          threadFlows: [
+            {
+              locations: [
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 141,
+                      },
+                      region: {
+                        startLine: 1270,
+                        startColumn: 50,
+                        endColumn: 76,
+                      },
+                    },
+                    message: {
+                      text: "new byte[] : byte[]",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/javax/crypto/spec/IvParameterSpec.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 12,
+                      },
+                      region: {
+                        startLine: 52,
+                        startColumn: 28,
+                        endColumn: 37,
+                      },
+                    },
+                    message: {
+                      text: "iv : byte[]",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/javax/crypto/spec/IvParameterSpec.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 12,
+                      },
+                      region: {
+                        startLine: 53,
+                        startColumn: 14,
+                        endColumn: 16,
+                      },
+                    },
+                    message: {
+                      text: "iv : byte[]",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/javax/crypto/spec/IvParameterSpec.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 12,
+                      },
+                      region: {
+                        startLine: 53,
+                        startColumn: 9,
+                        endColumn: 32,
+                      },
+                    },
+                    message: {
+                      text: "this <constr(this)> [post update] : IvParameterSpec",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 141,
+                      },
+                      region: {
+                        startLine: 1270,
+                        startColumn: 30,
+                        endColumn: 77,
+                      },
+                    },
+                    message: {
+                      text: "new IvParameterSpec(...) : IvParameterSpec",
+                    },
+                  },
+                },
+                {
+                  location: {
+                    physicalLocation: {
+                      artifactLocation: {
+                        uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+                        uriBaseId: "%SRCROOT%",
+                        index: 141,
+                      },
+                      region: {
+                        startLine: 1272,
+                        startColumn: 55,
+                        endColumn: 61,
+                      },
+                    },
+                    message: {
+                      text: "params",
+                    },
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      ],
+      relatedLocations: [
+        {
+          id: 1,
+          physicalLocation: {
+            artifactLocation: {
+              uri: "src/java.base/share/classes/sun/security/ssl/SSLCipher.java",
+              uriBaseId: "%SRCROOT%",
+              index: 141,
+            },
+            region: {
+              startLine: 1270,
+              startColumn: 50,
+              endColumn: 76,
+            },
+          },
+          message: {
+            text: "static initialization vector",
+          },
+        },
+      ],
+    };
+
+    expect(sarifDiff([result1], [result2])).toEqual({
+      from: [],
+      to: [],
+    });
+  });
+
   it("does not modify the input", () => {
     const result1: Result = {
       message: {