v1.8.10 (#2710)

Convert ResultTables to a function component

.github/codeql/queries/assert-no-vscode-dependency.ql

@@ -2,9 +2,8 @@
  * @name Unwanted dependency on vscode API
  * @kind path-problem
  * @problem.severity error
- * @id vscode-codeql/assert-pure
- * @description The modules stored under `pure` and tested in the `pure-tests`
- * are intended to be "pure".
+ * @id vscode-codeql/assert-no-vscode-dependency
+ * @description The modules stored under `common` should not have dependencies on the VS Code API
  */
 
 import javascript
@@ -13,12 +12,9 @@ class VSCodeImport extends ImportDeclaration {
   VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
 }
 
-class PureFile extends File {
-  PureFile() {
-    (
-      this.getRelativePath().regexpMatch(".*/src/pure/.*") or
-      this.getRelativePath().regexpMatch(".*/src/common/.*")
-    ) and
+class CommonFile extends File {
+  CommonFile() {
+    this.getRelativePath().regexpMatch(".*/src/common/.*") and
     not this.getRelativePath().regexpMatch(".*/vscode/.*")
   }
 }
@@ -34,7 +30,8 @@ query predicate edges(AstNode a, AstNode b) {
 
 from Module m, VSCodeImport v
 where
-  m.getFile() instanceof PureFile and
+  m.getFile() instanceof CommonFile and
   edges+(m, v)
 select m, m, v,
-  "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"
+  "This module is in the 'common' directory but has a transitive dependency on the vscode API imported $@",
+  v, "here"

.github/codeql/queries/qlpack.yml

@@ -1,3 +1,4 @@
 name: vscode-codeql-custom-queries-javascript
 version: 0.0.0
-libraryPathDependencies: codeql-javascript
+dependencies:
+  codeql/javascript-queries: "*"

.github/codeql/queries/unique-command-use.ql

@@ -15,134 +15,145 @@
  * that should be changed to fix the alert.
  */
 
- import javascript
+import javascript
 
- /**
-  * The name of a VS Code command.
-  */
- class CommandName extends string {
-   CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
- 
-   /**
-    * In how many ways is this command used. Will always be at least 1.
-    */
-   int getNumberOfUsages() { result = count(this.getAUse()) }
+/**
+ * The name of a VS Code command.
+ */
+class CommandName extends string {
+  CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
 
-   /**
-    * Get a usage of this command.
-    */
-   CommandUsage getAUse() { result.getCommandName() = this }
- 
-   /**
-    * Get the canonical first usage of this command, to use for the location
-    * of the alert. The implementation of this ordering of usages is arbitrary
-    * and the usage given may not be the one that should be changed when fixing
-    * the alert.
-    */
-   CommandUsage getFirstUsage() {
-     result =
-       max(CommandUsage use |
-         use = this.getAUse()
-       |
-         use
-         order by
-           use.getFile().getRelativePath(), use.getLocation().getStartLine(),
-           use.getLocation().getStartColumn()
-       )
-   }
- }
- 
- /**
-  * Represents a single usage of a command, either from within code or
-  * from the command's definition in package.json
-  */
- abstract class CommandUsage extends Locatable {
-   abstract string getCommandName();
- }
- 
- /**
-  * A usage of a command from the typescript code, by calling `executeCommand`.
-  */
- class CommandUsageCallExpr extends CommandUsage, CallExpr {
-   CommandUsageCallExpr() {
-     this.getCalleeName() = "executeCommand" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
- 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
+  /**
+   * In how many ways is this command used. Will always be at least 1.
+   */
+  int getNumberOfUsages() { result = count(this.getAUse()) }
 
- /**
-  * A usage of a command from the typescript code, by calling `CommandManager.execute`.
-  */
- class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
-   CommandUsageCommandManagerMethodCallExpr() {
-     this.getCalleeName() = "execute" and
-     this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
+  /**
+   * Get a usage of this command.
+   */
+  CommandUsage getAUse() { result.getCommandName() = this }
 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
- 
- /**
-  * A usage of a command from any menu that isn't the command palette.
-  * This means a user could invoke the command by clicking on a button in
-  * something like a menu or a dropdown.
-  */
- class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonMenuItem() {
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson, string menuName |
-       packageJson
-           .getPropValue("contributes")
-           .getPropValue("menus")
-           .getPropValue(menuName)
-           .getElementValue(_) = this and
-       menuName != "commandPalette"
-     )
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- /**
-  * Is the given command disabled for use in the command palette by
-  * a block with a `"when": "false"` field.
-  */
- predicate isDisabledInCommandPalette(string commandName) {
-   exists(PackageJson packageJson, JsonObject commandPaletteObject |
-     packageJson
-         .getPropValue("contributes")
-         .getPropValue("menus")
-         .getPropValue("commandPalette")
-         .getElementValue(_) = commandPaletteObject and
-     commandPaletteObject.getPropValue("command").getStringValue() = commandName and
-     commandPaletteObject.getPropValue("when").getStringValue() = "false"
-   )
- }
- 
- /**
-  * Represents a command being usable from the command palette.
-  * This means that a user could choose to manually invoke the command.
-  */
- class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonCommandPalette() {
-     this.getFile().getBaseName() = "package.json" and
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson |
-       packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
-     ) and
-     not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- from CommandName c
- where c.getNumberOfUsages() > 1
- select c.getFirstUsage(),
-   "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"
- 
+  /**
+   * Get the canonical first usage of this command, to use for the location
+   * of the alert. The implementation of this ordering of usages is arbitrary
+   * and the usage given may not be the one that should be changed when fixing
+   * the alert.
+   */
+  CommandUsage getFirstUsage() {
+    result =
+      max(CommandUsage use |
+        use = this.getAUse()
+      |
+        use
+        order by
+          use.getFile().getRelativePath(), use.getLocation().getStartLine(),
+          use.getLocation().getStartColumn()
+      )
+  }
+}
+
+/**
+ * Matches one of the members of `BuiltInVsCodeCommands` from `extensions/ql-vscode/src/common/commands.ts`.
+ */
+class BuiltInVSCodeCommand extends string {
+  BuiltInVSCodeCommand() {
+    exists(TypeAliasDeclaration tad |
+      tad.getIdentifier().getName() = "BuiltInVsCodeCommands" and
+      tad.getDefinition().(InterfaceTypeExpr).getAMember().getName() = this
+    )
+  }
+}
+
+/**
+ * Represents a single usage of a command, either from within code or
+ * from the command's definition in package.json
+ */
+abstract class CommandUsage extends Locatable {
+  abstract string getCommandName();
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `executeCommand`.
+ */
+class CommandUsageCallExpr extends CommandUsage, CallExpr {
+  CommandUsageCallExpr() {
+    this.getCalleeName() = "executeCommand" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `CommandManager.execute`.
+ */
+class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
+  CommandUsageCommandManagerMethodCallExpr() {
+    this.getCalleeName() = "execute" and
+    this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from any menu that isn't the command palette.
+ * This means a user could invoke the command by clicking on a button in
+ * something like a menu or a dropdown.
+ */
+class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonMenuItem() {
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson, string menuName |
+      packageJson
+          .getPropValue("contributes")
+          .getPropValue("menus")
+          .getPropValue(menuName)
+          .getElementValue(_) = this and
+      menuName != "commandPalette"
+    )
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+/**
+ * Is the given command disabled for use in the command palette by
+ * a block with a `"when": "false"` field.
+ */
+predicate isDisabledInCommandPalette(string commandName) {
+  exists(PackageJson packageJson, JsonObject commandPaletteObject |
+    packageJson
+        .getPropValue("contributes")
+        .getPropValue("menus")
+        .getPropValue("commandPalette")
+        .getElementValue(_) = commandPaletteObject and
+    commandPaletteObject.getPropValue("command").getStringValue() = commandName and
+    commandPaletteObject.getPropValue("when").getStringValue() = "false"
+  )
+}
+
+/**
+ * Represents a command being usable from the command palette.
+ * This means that a user could choose to manually invoke the command.
+ */
+class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonCommandPalette() {
+    this.getFile().getBaseName() = "package.json" and
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson |
+      packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
+    ) and
+    not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+from CommandName c
+where c.getNumberOfUsages() > 1 and not c instanceof BuiltInVSCodeCommand
+select c.getFirstUsage(),
+  "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"

.github/workflows/main.yml

@@ -110,6 +110,11 @@ jobs:
         run: |
           npm run lint:scenarios
 
+      - name: Find deadcode
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run find-deadcode
+
   unit-test:
     name: Unit Test
     runs-on: ${{ matrix.os }}
@@ -209,6 +214,7 @@ jobs:
     name: CLI Test
     runs-on: ${{ matrix.os }}
     needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]

CODEOWNERS

@@ -2,3 +2,4 @@
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
 **/data-extensions-editor/ @github/code-scanning-secexp-reviewers
+**/queries-panel/ @github/code-scanning-secexp-reviewers

docs/test-plan.md

@@ -44,21 +44,21 @@ choose to go through some of the Optional Test Cases.
 
 #### Test case 2: Running a problem query and viewing results
 
-1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
 2. Select the `babel/babel` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
    - Check that the result messages are rendered
    - Check that alert locations can be clicked on
 
-#### Test case 3: Running a non-probem query and viewing results
+#### Test case 3: Running a non-problem query and viewing results
 
 1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
 2. Select the `google/brotli` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
    - Check that the results table is rendered
-   - Check that alert locations can be clicked on
+   - Check that result locations can be clicked on
 
 #### Test case 3: Can use AST viewer
 
@@ -318,7 +318,6 @@ This requires running a MRVA query and seeing the results view.
    1. Alphabetically
    2. By number of results
    3. By popularity
-   4. By most recent commit
 9. Can filter repos
 10. Shows correct statistics
     1. Total number of results
@@ -378,6 +377,7 @@ This requires running a MRVA query and seeing the results view.
 1. Make changes via config file (ensure JSON schema is helping out)
 1. Close and re-open VS Code (ensure lists are there)
 1. Collapse/expand tree nodes
+1. Create a new list, right click and select "Add repositories with GitHub Code Search". Enter the language 'python' and the query "UserMixin". This should show a rate limiting notification after a while but eventually populate the list with roughly 770 items.
 
 Error cases that trigger an error notification:
 

extensions/ql-vscode/.eslintrc.js

@@ -65,10 +65,6 @@ const baseConfig = {
     "import/no-namespace": "off",
     "import/no-unresolved": "off",
     "import/no-webpack-loader-syntax": "off",
-    "jsx-a11y/anchor-is-valid": "off",
-    "jsx-a11y/no-noninteractive-element-interactions": "off",
-    "jsx-a11y/no-static-element-interactions": "off",
-    "jsx-a11y/click-events-have-key-events": "off",
     "no-invalid-this": "off",
     "no-fallthrough": "off",
     "no-console": "off",

extensions/ql-vscode/.storybook/main.ts

@@ -1,4 +1,4 @@
-import type { StorybookConfig } from "@storybook/core-common";
+import type { StorybookConfig } from "@storybook/react-webpack5";
 
 const config: StorybookConfig = {
   stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
@@ -8,13 +8,13 @@ const config: StorybookConfig = {
     "@storybook/addon-interactions",
     "./vscode-theme-addon/preset.ts",
   ],
-  framework: "@storybook/react",
-  core: {
-    builder: "@storybook/builder-webpack5",
+  framework: {
+    name: "@storybook/react-webpack5",
+    options: {},
   },
-  features: {
-    babelModeV7: true,
+  docs: {
+    autodocs: "tag",
   },
 };
 
-module.exports = config;
+export default config;

extensions/ql-vscode/.storybook/manager.ts

@@ -1,4 +1,4 @@
-import { addons } from "@storybook/addons";
+import { addons } from "@storybook/manager-api";
 import { themes } from "@storybook/theming";
 
 addons.setConfig({

extensions/ql-vscode/.storybook/preview.ts

@@ -1,31 +1,36 @@
+import { Preview } from "@storybook/react";
 import { themes } from "@storybook/theming";
 import { action } from "@storybook/addon-actions";
 
 // Allow all stories/components to use Codicons
 import "@vscode/codicons/dist/codicon.css";
 
-// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
-export const parameters = {
-  // All props starting with `on` will automatically receive an action as a prop
-  actions: { argTypesRegex: "^on[A-Z].*" },
-  // All props matching these names will automatically get the correct control
-  controls: {
-    matchers: {
-      color: /(background|color)$/i,
-      date: /Date$/,
-    },
-  },
-  // Use a dark theme to be aligned with VSCode
-  docs: {
-    theme: themes.dark,
-  },
-  backgrounds: {
-    // The background is injected by our theme CSS files
-    disable: true,
-  },
-};
-
 (window as any).acquireVsCodeApi = () => ({
   postMessage: action("post-vscode-message"),
   setState: action("set-vscode-state"),
 });
+
+// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
+const preview: Preview = {
+  parameters: {
+    // All props starting with `on` will automatically receive an action as a prop
+    actions: { argTypesRegex: "^on[A-Z].*" },
+    // All props matching these names will automatically get the correct control
+    controls: {
+      matchers: {
+        color: /(background|color)$/i,
+        date: /Date$/,
+      },
+    },
+    // Use a dark theme to be aligned with VSCode
+    docs: {
+      theme: themes.dark,
+    },
+    backgrounds: {
+      // The background is injected by our theme CSS files
+      disable: true,
+    },
+  },
+};
+
+export default preview;

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -1,14 +1,12 @@
 import * as React from "react";
 import { FunctionComponent, useCallback } from "react";
 
-import { useGlobals } from "@storybook/api";
+import { useGlobals } from "@storybook/manager-api";
 import {
   IconButton,
   Icons,
-  WithTooltip,
   TooltipLinkList,
-  Link,
-  WithHideFn,
+  WithTooltip,
 } from "@storybook/components";
 
 import { themeNames, VSCodeTheme } from "./theme";
@@ -26,7 +24,7 @@ export const ThemeSelector: FunctionComponent = () => {
   );
 
   const createLinks = useCallback(
-    (onHide: () => void): Link[] =>
+    (onHide: () => void) =>
       Object.values(VSCodeTheme).map((theme) => ({
         id: theme,
         onClick() {
@@ -44,8 +42,8 @@ export const ThemeSelector: FunctionComponent = () => {
     <WithTooltip
       placement="top"
       trigger="click"
-      closeOnClick
-      tooltip={({ onHide }: WithHideFn) => (
+      closeOnOutsideClick
+      tooltip={({ onHide }: { onHide: () => void }) => (
         <TooltipLinkList links={createLinks(onHide)} />
       )}
     >

extensions/ql-vscode/.storybook/vscode-theme-addon/manager.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { addons, types } from "@storybook/addons";
+import { addons, types } from "@storybook/manager-api";
 import { ThemeSelector } from "./ThemeSelector";
 
 const ADDON_ID = "vscode-theme-addon";

extensions/ql-vscode/.storybook/vscode-theme-addon/preset.ts

@@ -1,4 +1,4 @@
-export function config(entry = []) {
+export function previewAnnotations(entry = []) {
   return [...entry, require.resolve("./preview.ts")];
 }
 

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -1,6 +1,5 @@
-import { useEffect, useGlobals } from "@storybook/addons";
+import { useEffect } from "react";
 import type {
-  AnyFramework,
   PartialStoryFn as StoryFunction,
   StoryContext,
 } from "@storybook/csf";
@@ -34,11 +33,8 @@ const themeFiles: { [key in VSCodeTheme]: string } = {
       .default,
 };
 
-export const withTheme = (
-  StoryFn: StoryFunction<AnyFramework>,
-  context: StoryContext<AnyFramework>,
-) => {
-  const [{ vscodeTheme }] = useGlobals();
+export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {
+  const { vscodeTheme } = context.globals;
 
   useEffect(() => {
     const styleSelectorId =

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,31 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.8.10 - 15 August 2023
+
+- Add a code lens to make the `CodeQL: Open Referenced File` command more discoverable. Click the "Open referenced file" prompt in a `.qlref` file to jump to the referenced `.ql` file. [#2704](https://github.com/github/vscode-codeql/pull/2704)
+
+## 1.8.9 - 3 August 2023
+
+- Remove "last updated" information and sorting from variant analysis results view. [#2637](https://github.com/github/vscode-codeql/pull/2637)
+- Links to code on GitHub now include column numbers as well as line numbers. [#2406](https://github.com/github/vscode-codeql/pull/2406)
+- No longer highlight trailing commas for jump to definition.  [#2615](https://github.com/github/vscode-codeql/pull/2615)
+- Fix a bug where the QHelp preview page was not being refreshed after changes to the underlying `.qhelp` file. [#2660](https://github.com/github/vscode-codeql/pull/2660)
+
+## 1.8.8 - 17 July 2023
+
+- Remove support for CodeQL CLI versions older than 2.9.4. [#2610](https://github.com/github/vscode-codeql/pull/2610)
+- Implement syntax highlighting for the `additional` and `default` keywords. [#2609](https://github.com/github/vscode-codeql/pull/2609)
+
+## 1.8.7 - 29 June 2023
+
+- Show a run button on the file tab for query files, that will start a local query. This button will only show when a local database is selected in the extension. [#2544](https://github.com/github/vscode-codeql/pull/2544)
+- Add a `CodeQL: Quick Evaluation Count` command to generate the count summary statistics of the results set
+  without spending the time to compute locations and strings. [#2475](https://github.com/github/vscode-codeql/pull/2475)
+
+## 1.8.6 - 14 June 2023
+
+- Add repositories to a variant analysis list with GitHub Code Search. [#2439](https://github.com/github/vscode-codeql/pull/2439) and [#2476](https://github.com/github/vscode-codeql/pull/2476)
+
 ## 1.8.5 - 6 June 2023
 
 - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
@@ -7,7 +33,7 @@
 - Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
 - Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
 - Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
-- Retry MRVA results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
+- Retry variant analysis results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
 
 ## 1.8.4 - 3 May 2023
 

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -15,9 +15,6 @@ export const config: webpack.Configuration = {
   devtool: isDevBuild ? "inline-source-map" : "source-map",
   resolve: {
     extensions: [".js", ".ts", ".tsx", ".json"],
-    fallback: {
-      path: require.resolve("path-browserify"),
-    },
   },
   module: {
     rules: [
@@ -62,18 +59,10 @@ export const config: webpack.Configuration = {
       },
       {
         test: /\.(woff(2)?|ttf|eot)$/,
-        use: [
-          {
-            loader: "file-loader",
-            options: {
-              name: "[name].[ext]",
-              outputPath: "fonts/",
-              // We need this to make Webpack use the correct path for the fonts.
-              // Without this, the CSS file will use `url([object Module])`
-              esModule: false,
-            },
-          },
-        ],
+        type: "asset/resource",
+        generator: {
+          filename: "fonts/[hash][ext][query]",
+        },
       },
     ],
   },

extensions/ql-vscode/language-configuration.json

@@ -30,5 +30,5 @@
       "end": "^\\s*//\\s*#?endregion\\b"
     }
   },
-  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s\\,]+)"
 }

extensions/ql-vscode/media/dark/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#C5C5C5"/></svg>

extensions/ql-vscode/media/light/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#424242"/></svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.8.5",
+  "version": "1.8.10",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -14,7 +14,7 @@
   },
   "engines": {
     "vscode": "^1.67.0",
-    "node": "^16.13.0",
+    "node": "^16.17.1",
     "npm": ">=7.20.6"
   },
   "categories": [
@@ -71,6 +71,7 @@
   "contributes": {
     "configurationDefaults": {
       "[ql]": {
+        "debug.saveBeforeStart": "nonUntitledEditorsInActiveGroup",
         "editor.wordBasedSuggestions": false
       },
       "[dbscheme]": {
@@ -189,7 +190,7 @@
           "scope": "machine-overridable",
           "type": "string",
           "default": "",
-          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
+          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -246,8 +247,8 @@
         },
         "codeQL.runningQueries.autoSave": {
           "type": "boolean",
-          "default": false,
-          "description": "Enable automatically saving a modified query file when running a query."
+          "description": "Enable automatically saving a modified query file when running a query.",
+          "markdownDeprecationMessage": "This property is deprecated and no longer has any effect. To control automatic saving of documents before running queries, use the `debug.saveBeforeStart` setting."
         },
         "codeQL.runningQueries.maxQueries": {
           "type": "integer",
@@ -349,13 +350,11 @@
           "enum": [
             "alphabetically",
             "popularity",
-            "mostRecentCommit",
             "numberOfResults"
           ],
           "enumDescriptions": [
             "Sort repositories alphabetically in the results view.",
             "Sort repositories by popularity in the results view.",
-            "Sort repositories by most recent commit in the results view.",
             "Sort repositories by number of results in the results view."
           ],
           "description": "The default sorting order for repositories in the variant analysis results view."
@@ -457,6 +456,10 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.quickEvalCount",
+        "title": "CodeQL: Quick Evaluation Count"
+      },
       {
         "command": "codeQL.quickEvalContextEditor",
         "title": "CodeQL: Quick Evaluation"
@@ -501,6 +504,33 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+        "title": "Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesFromPanel",
+        "title": "Run local queries",
+        "icon": "$(run-all)"
+      },
+      {
+        "command": "codeQL.runLocalQueryFromFileTab",
+        "title": "CodeQL: Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueryContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runVariantAnalysisContextMenu",
+        "title": "Run against variant analysis repositories"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.openConfigFile",
         "title": "Open database configuration file",
@@ -872,6 +902,13 @@
       }
     ],
     "menus": {
+      "editor/title": [
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "group": "navigation",
+          "when": "resourceExtname == .ql && codeQL.currentDatabaseItem"
+        }
+      ],
       "view/title": [
         {
           "command": "codeQLDatabases.sortByName",
@@ -967,7 +1004,7 @@
         },
         {
           "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
-          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/ && config.codeQL.codeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
           "group": "2_qlContextMenu@1"
         },
         {
@@ -1038,17 +1075,17 @@
         {
           "command": "codeQLQueryHistory.showEvalLog",
           "group": "4_queryHistory@1",
-          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showEvalLogSummary",
           "group": "4_queryHistory@2",
-          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showEvalLogViewer",
           "group": "4_queryHistory@3",
-          "when": "config.codeQL.canary && codeql.supportsEvalLog && viewItem == rawResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == interpretedResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+          "when": "config.codeQL.canary && viewItem == rawResultsItem || config.codeQL.canary && viewItem == interpretedResultsItem || config.codeQL.canary && viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showQueryText",
@@ -1095,6 +1132,31 @@
           "group": "1_queryHistory@1",
           "when": "viewItem == remoteResultsItem"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -1154,6 +1216,18 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "when": "false"
+        },
         {
           "command": "codeQL.runQueryContextEditor",
           "when": "false"
@@ -1206,6 +1280,10 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.quickEvalCount",
+          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+        },
         {
           "command": "codeQL.quickEvalContextEditor",
           "when": "false"
@@ -1274,6 +1352,18 @@
           "command": "codeQL.openDataExtensionsEditor",
           "when": "config.codeQL.canary && config.codeQL.dataExtensions.editor"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "when": "false"
+        },
         {
           "command": "codeQLVariantAnalysisRepositories.openConfigFile",
           "when": "false"
@@ -1565,6 +1655,13 @@
           "title": "CodeQL",
           "icon": "media/logo.svg"
         }
+      ],
+      "panel": [
+        {
+          "id": "codeql-model-details",
+          "title": "CodeQL Model Details",
+          "icon": "media/logo.svg"
+        }
       ]
     },
     "views": {
@@ -1595,9 +1692,20 @@
           "name": "Evaluator Log Viewer",
           "when": "config.codeQL.canary"
         }
+      ],
+      "codeql-model-details": [
+        {
+          "id": "codeQLModelDetails",
+          "name": "CodeQL Model Details",
+          "when": "config.codeQL.canary && config.codeQL.dataExtensions.modelDetailsView"
+        }
       ]
     },
     "viewsWelcome": [
+      {
+        "view": "codeQLModelDetails",
+        "contents": "Loading..."
+      },
       {
         "view": "codeQLAstViewer",
         "contents": "Run the 'CodeQL: View AST' command on an open source file from a CodeQL database.\n[View AST](command:codeQL.viewAst)"
@@ -1608,7 +1716,7 @@
       },
       {
         "view": "codeQLQueries",
-        "contents": "This workspace doesn't contain any CodeQL queries at the moment."
+        "contents": "Looking for queries..."
       },
       {
         "view": "codeQLDatabases",
@@ -1635,21 +1743,23 @@
     "test:vscode-integration:activated-extension": "jest --projects test/vscode-tests/activated-extension",
     "test:vscode-integration:no-workspace": "jest --projects test/vscode-tests/no-workspace",
     "test:vscode-integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace",
-    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration",
+    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration --verbose",
+    "clean-test-dir": "find . -type d -name .vscode-test -exec rm -r {} +",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",
     "lint:markdown": "markdownlint-cli2 \"../../**/*.{md,mdx}\" \"!**/node_modules/**\" \"!**/.vscode-test/**\" \"!**/build/cli/v*/**\"",
+    "find-deadcode": "ts-node scripts/find-deadcode.ts",
     "format-staged": "lint-staged",
-    "storybook": "start-storybook -p 6006",
-    "build-storybook": "build-storybook",
+    "storybook": "storybook dev -p 6006",
+    "build-storybook": "storybook build",
     "lint:scenarios": "ts-node scripts/lint-scenarios.ts",
     "check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
     "postinstall": "patch-package",
     "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
-    "@octokit/plugin-retry": "^3.0.9",
+    "@octokit/plugin-retry": "^4.1.6",
     "@octokit/rest": "^19.0.4",
     "@vscode/codicons": "^0.0.31",
     "@vscode/debugadapter": "^1.59.0",
@@ -1658,103 +1768,100 @@
     "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
     "chokidar": "^3.5.3",
-    "classnames": "~2.2.6",
+    "classnames": "^2.2.6",
     "d3": "^7.6.1",
     "d3-graphviz": "^5.0.2",
     "fs-extra": "^11.1.1",
     "immutable": "^4.0.0",
     "js-yaml": "^4.1.0",
-    "minimatch": "^9.0.0",
-    "minimist": "~1.2.6",
     "msw": "^1.2.0",
     "nanoid": "^3.2.0",
-    "node-fetch": "~2.6.7",
+    "node-fetch": "^2.6.7",
     "p-queue": "^6.0.0",
-    "path-browserify": "^1.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "semver": "~7.3.2",
+    "semver": "^7.5.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
-    "stream": "^0.0.2",
-    "stream-chain": "~2.2.4",
-    "stream-json": "~1.7.3",
-    "styled-components": "^5.3.3",
+    "stream-json": "^1.7.3",
+    "styled-components": "^6.0.2",
     "tmp": "^0.1.0",
-    "tmp-promise": "~3.0.2",
-    "tree-kill": "~1.2.2",
-    "unzipper": "~0.10.5",
+    "tmp-promise": "^3.0.2",
+    "tree-kill": "^1.2.2",
+    "unzipper": "^0.10.5",
     "vscode-extension-telemetry": "^0.1.6",
     "vscode-jsonrpc": "^8.0.2",
     "vscode-languageclient": "^8.0.2",
-    "vscode-test-adapter-api": "~1.7.0",
-    "vscode-test-adapter-util": "~0.7.0",
-    "zip-a-folder": "~1.1.3"
+    "vscode-test-adapter-api": "^1.7.0",
+    "vscode-test-adapter-util": "^0.7.0",
+    "zip-a-folder": "^2.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
-    "@faker-js/faker": "^7.5.0",
+    "@babel/preset-env": "^7.21.4",
+    "@babel/preset-react": "^7.18.6",
+    "@babel/preset-typescript": "^7.21.4",
+    "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.3.0",
     "@octokit/plugin-throttling": "^5.0.1",
-    "@storybook/addon-actions": "^6.5.17-alpha.0",
-    "@storybook/addon-essentials": "^6.5.17-alpha.0",
-    "@storybook/addon-interactions": "^6.5.17-alpha.0",
-    "@storybook/addon-links": "^6.5.17-alpha.0",
-    "@storybook/builder-webpack5": "^6.5.17-alpha.0",
-    "@storybook/manager-webpack5": "^6.5.17-alpha.0",
-    "@storybook/react": "^6.5.17-alpha.0",
-    "@storybook/testing-library": "^0.0.13",
+    "@storybook/addon-actions": "^7.1.0",
+    "@storybook/addon-essentials": "^7.1.0",
+    "@storybook/addon-interactions": "^7.1.0",
+    "@storybook/addon-links": "^7.1.0",
+    "@storybook/components": "^7.1.0",
+    "@storybook/csf": "^0.1.1",
+    "@storybook/manager-api": "^7.1.0",
+    "@storybook/react": "^7.1.0",
+    "@storybook/react-webpack5": "^7.1.0",
+    "@storybook/theming": "^7.1.0",
+    "@testing-library/dom": "^9.3.0",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",
     "@types/child-process-promise": "^2.2.1",
-    "@types/classnames": "~2.2.9",
+    "@types/classnames": "^2.2.9",
     "@types/d3": "^7.4.0",
     "@types/d3-graphviz": "^2.6.6",
     "@types/del": "^4.0.0",
     "@types/fs-extra": "^11.0.1",
-    "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "^1.1.0",
-    "@types/gulp-sourcemaps": "0.0.32",
     "@types/jest": "^29.0.2",
     "@types/js-yaml": "^3.12.5",
-    "@types/jszip": "~3.1.6",
     "@types/nanoid": "^3.0.0",
     "@types/node": "^16.11.25",
-    "@types/node-fetch": "~2.5.2",
+    "@types/node-fetch": "^2.5.2",
     "@types/react": "^18.0.28",
     "@types/react-dom": "^18.0.11",
-    "@types/sarif": "~2.1.2",
-    "@types/semver": "~7.2.0",
-    "@types/stream-chain": "~2.0.1",
-    "@types/stream-json": "~1.7.1",
+    "@types/sarif": "^2.1.2",
+    "@types/semver": "^7.2.0",
+    "@types/stream-json": "^1.7.1",
     "@types/styled-components": "^5.1.11",
     "@types/tar-stream": "^2.2.2",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
-    "@types/unzipper": "~0.10.1",
+    "@types/unzipper": "^0.10.1",
     "@types/vscode": "^1.67.0",
     "@types/webpack": "^5.28.0",
     "@types/webpack-env": "^1.18.0",
-    "@types/xml2js": "~0.4.4",
-    "@typescript-eslint/eslint-plugin": "^5.38.0",
-    "@typescript-eslint/parser": "^5.38.0",
+    "@typescript-eslint/eslint-plugin": "^6.2.1",
+    "@typescript-eslint/parser": "^6.2.1",
     "@vscode/test-electron": "^2.2.0",
-    "@vscode/vsce": "^2.15.0",
+    "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
+    "cosmiconfig": "^8.2.0",
     "cross-env": "^7.0.3",
-    "css-loader": "~3.1.0",
+    "css-loader": "^6.8.1",
     "del": "^6.0.0",
     "esbuild": "^0.15.15",
     "eslint": "^8.23.1",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-etc": "^2.0.2",
     "eslint-plugin-github": "^4.4.1",
-    "eslint-plugin-jest-dom": "^4.0.2",
-    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-jest-dom": "^5.0.1",
+    "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-react": "^7.31.8",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-storybook": "^0.6.4",
@@ -1769,20 +1876,21 @@
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "~13.2.0",
+    "lint-staged": "^13.2.0",
     "markdownlint-cli2": "^0.6.0",
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "patch-package": "^7.0.0",
-    "prettier": "^2.7.1",
+    "patch-package": "^8.0.0",
+    "prettier": "^3.0.0",
+    "storybook": "^7.1.0",
     "tar-stream": "^3.0.0",
     "through2": "^4.0.2",
     "ts-jest": "^29.0.1",
     "ts-json-schema-generator": "^1.1.2",
     "ts-loader": "^9.4.2",
     "ts-node": "^10.7.0",
-    "ts-protoc-gen": "^0.9.0",
+    "ts-unused-exports": "^10.0.0",
     "typescript": "^5.0.2",
     "webpack": "^5.76.0",
     "webpack-cli": "^5.0.1"

extensions/ql-vscode/patches/jest-runner-vscode+3.0.1.patch

@@ -25,7 +25,7 @@ index 1ac28d5..f91f216 100644
          await super.setup();
          await (0, load_pnp_1.default)();
 diff --git a/node_modules/jest-runner-vscode/dist/child/runner.js b/node_modules/jest-runner-vscode/dist/child/runner.js
-index 0663c5c..4991663 100644
+index 0663c5c..bdf4a8b 100644
 --- a/node_modules/jest-runner-vscode/dist/child/runner.js
 +++ b/node_modules/jest-runner-vscode/dist/child/runner.js
 @@ -18,10 +18,13 @@ async function run() {
@@ -43,6 +43,16 @@ index 0663c5c..4991663 100644
          const options = JSON.parse(PARENT_JEST_OPTIONS);
          const jestOptions = [
              ...options.args,
+@@ -39,6 +42,9 @@ async function run() {
+             ...(argv.projects?.map(project => path_1.default.resolve(project)) || []),
+             options.workspacePath,
+         ]);
++        const testPaths = new Set(argv._.map(testPath => path_1.default.resolve(testPath)));
++        argv._ = [...testPaths];
++
+         await (0, core_1.runCLI)(argv, [...projects]);
+     }
+     catch (error) {
 diff --git a/node_modules/jest-runner-vscode/dist/public-types.d.ts b/node_modules/jest-runner-vscode/dist/public-types.d.ts
 index 57716e5..d8614af 100644
 --- a/node_modules/jest-runner-vscode/dist/public-types.d.ts

extensions/ql-vscode/scripts/find-deadcode.ts

@@ -0,0 +1,47 @@
+import { basename, join, relative, resolve } from "path";
+import analyzeTsConfig from "ts-unused-exports";
+import { containsPath, pathsEqual } from "../src/common/files";
+import { exit } from "process";
+
+function ignoreFile(file: string): boolean {
+  return (
+    containsPath("gulpfile.ts", file) ||
+    containsPath(join("src", "stories"), file) ||
+    pathsEqual(
+      join("test", "vscode-tests", "jest-runner-installed-extensions.ts"),
+      file,
+    ) ||
+    basename(file) === "jest.config.ts" ||
+    basename(file) === "index.tsx" ||
+    basename(file) === "index.ts"
+  );
+}
+
+function main() {
+  const repositoryRoot = resolve(join(__dirname, ".."));
+
+  const result = analyzeTsConfig("tsconfig.deadcode.json");
+  let foundUnusedExports = false;
+
+  for (const [filepath, exportNameAndLocations] of Object.entries(result)) {
+    const relativeFilepath = relative(repositoryRoot, filepath);
+
+    if (ignoreFile(relativeFilepath)) {
+      continue;
+    }
+
+    foundUnusedExports = true;
+
+    console.log(relativeFilepath);
+    for (const exportNameAndLocation of exportNameAndLocations) {
+      console.log(`    ${exportNameAndLocation.exportName}`);
+    }
+    console.log();
+  }
+
+  if (foundUnusedExports) {
+    exit(1);
+  }
+}
+
+main();

extensions/ql-vscode/src/code-tour/code-tour.ts

@@ -0,0 +1,55 @@
+import { AppCommandManager } from "../common/commands";
+import { Uri, workspace } from "vscode";
+import { join } from "path";
+import { pathExists } from "fs-extra";
+import { isCodespacesTemplate } from "../config";
+import { showBinaryChoiceDialog } from "../common/vscode/dialog";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Check if the current workspace is the CodeTour and open the workspace folder.
+ * Without this, we can't run the code tour correctly.
+ **/
+export async function prepareCodeTour(
+  commandManager: AppCommandManager,
+): Promise<void> {
+  if (workspace.workspaceFolders?.length) {
+    const currentFolder = workspace.workspaceFolders[0].uri.fsPath;
+
+    const tutorialWorkspacePath = join(
+      currentFolder,
+      "tutorial.code-workspace",
+    );
+    const toursFolderPath = join(currentFolder, ".tours");
+
+    /** We're opening the tutorial workspace, if we detect it.
+     * This will only happen if the following three conditions are met:
+     * - the .tours folder exists
+     * - the tutorial.code-workspace file exists
+     * - the CODESPACES_TEMPLATE setting doesn't exist (it's only set if the user has already opened
+     * the tutorial workspace so it's a good indicator that the user is in the folder but has ignored
+     * the prompt to open the workspace)
+     */
+    if (
+      (await pathExists(tutorialWorkspacePath)) &&
+      (await pathExists(toursFolderPath)) &&
+      !isCodespacesTemplate()
+    ) {
+      const answer = await showBinaryChoiceDialog(
+        "We've detected you're in the CodeQL Tour repo. We will need to open the workspace file to continue. Reload?",
+      );
+
+      if (!answer) {
+        return;
+      }
+
+      const tutorialWorkspaceUri = Uri.file(tutorialWorkspacePath);
+
+      void extLogger.log(
+        `In prepareCodeTour() method, going to open the tutorial workspace file: ${tutorialWorkspacePath}`,
+      );
+
+      await commandManager.execute("vscode.openFolder", tutorialWorkspaceUri);
+    }
+  }
+}

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,7 +1,7 @@
 import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "../common";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { Logger } from "../common/logging";
+import { getErrorMessage } from "../common/helpers-pure";
 
 /**
  * Get the version of a CodeQL CLI.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -11,7 +11,7 @@ import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "../pure/bqrs-cli-types";
+import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
 import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
@@ -21,12 +21,13 @@ import {
   assertNever,
   getErrorMessage,
   getErrorStack,
-} from "../pure/helpers-pure";
-import { QueryMetadata, SortDirection } from "../pure/interface-types";
-import { BaseLogger, Logger, ProgressReporter } from "../common";
-import { CompilationMessage } from "../pure/legacy-messages";
+} from "../common/helpers-pure";
+import { walkDirectory } from "../common/files";
+import { QueryMetadata, SortDirection } from "../common/interface-types";
+import { BaseLogger, Logger } from "../common/logging";
+import { ProgressReporter } from "../common/logging/vscode";
+import { CompilationMessage } from "../query-server/legacy-messages";
 import { sarifParser } from "../common/sarif-parser";
-import { walkDirectory } from "../helpers";
 import { App } from "../common/app";
 import { QueryLanguage } from "../common/query-language";
 
@@ -718,6 +719,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(
     workspaces: string[],
     queryPath: string,
+    silent = false,
   ): Promise<QuerySetup> {
     const subcommandArgs = [
       "--query",
@@ -728,6 +730,7 @@ export class CodeQLCliServer implements Disposable {
       ["resolve", "library-path"],
       subcommandArgs,
       "Resolving library paths",
+      { silent },
     );
   }
 
@@ -1425,21 +1428,13 @@ export class CodeQLCliServer implements Disposable {
 
   async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
     const args = includeQueries ? [dir] : ["--no-include-queries", dir];
-    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
-    // previous versions return a list of packs.
-    const results: { paths: string[] } | string[] =
-      await this.runJsonCodeQlCliCommand(
-        ["pack", "packlist"],
-        args,
-        "Generating the pack list",
-      );
+    const results: { paths: string[] } = await this.runJsonCodeQlCliCommand(
+      ["pack", "packlist"],
+      args,
+      "Generating the pack list",
+    );
 
-    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
-    if ("paths" in results) {
-      return results.paths;
-    } else {
-      return results;
-    }
+    return results.paths;
   }
 
   async packResolveDependencies(
@@ -1475,9 +1470,9 @@ export class CodeQLCliServer implements Disposable {
         // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
         await this.app.commands.execute(
           "setContext",
-          "codeql.supportsEvalLog",
+          "codeql.supportsQuickEvalCount",
           newVersion.compare(
-            CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
           ) >= 0,
         );
       } catch (e) {
@@ -1756,34 +1751,31 @@ async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
   }
 }
 
-export function shouldDebugIdeServer() {
+function isEnvTrue(name: string): boolean {
   return (
-    "IDE_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.IDE_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
+    name in process.env &&
+    process.env[name] !== "0" &&
+    // Use en-US since we expect the value to be either "false" or "FALSE", not a localized version.
+    process.env[name]?.toLocaleLowerCase("en-US") !== "false"
   );
 }
 
+export function shouldDebugIdeServer() {
+  return isEnvTrue("IDE_SERVER_JAVA_DEBUG");
+}
+
 export function shouldDebugQueryServer() {
-  return (
-    "QUERY_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.QUERY_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("QUERY_SERVER_JAVA_DEBUG");
 }
 
 export function shouldDebugCliServer() {
-  return (
-    "CLI_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.CLI_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("CLI_SERVER_JAVA_DEBUG");
 }
 
 export class CliVersionConstraint {
   // The oldest version of the CLI that we support. This is used to determine
   // whether to show a warning about the CLI being too old on startup.
-  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.7.6");
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.9.4");
 
   /**
    * CLI version where building QLX packs for remote queries is supported.
@@ -1800,21 +1792,9 @@ export class CliVersionConstraint {
   );
 
   /**
-   * CLI version where the `--evaluator-log` and related options to the query server were introduced,
-   * on a per-query server basis.
+   * CLI version where the `resolve extensions` subcommand exists.
    */
-  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer("2.8.2");
-
-  /**
-   * CLI version that supports rotating structured logs to produce one per query.
-   *
-   * Note that 2.8.4 supports generating the evaluation logs and summaries,
-   * but 2.9.0 includes a new option to produce the end-of-query summary logs to
-   * the query server console. For simplicity we gate all features behind 2.9.0,
-   * but if a user is tied to the 2.8 release, we can enable evaluator logs
-   * and summaries for them.
-   */
-  public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer("2.9.0");
+  public static CLI_VERSION_WITH_RESOLVE_EXTENSIONS = new SemVer("2.10.2");
 
   /**
    * CLI version that supports the `--sourcemap` option for log generation.
@@ -1845,6 +1825,18 @@ export class CliVersionConstraint {
 
   public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
 
+  /**
+   * CLI version where the query server supports quick-eval count mode.
+   */
+  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
+
+  /**
+   * CLI version where the langauge server supports visisbility change notifications.
+   */
+  public static CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS = new SemVer(
+    "2.14.0",
+  );
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1863,15 +1855,9 @@ export class CliVersionConstraint {
     );
   }
 
-  async supportsStructuredEvalLog() {
+  async supportsResolveExtensions() {
     return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG,
-    );
-  }
-
-  async supportsPerQueryEvalLog() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+      CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS,
     );
   }
 
@@ -1918,4 +1904,16 @@ export class CliVersionConstraint {
   async usesGlobalCompilationCache() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
   }
+
+  async supportsVisibilityNotifications() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
+    );
+  }
+
+  async supportsQuickEvalCount() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -6,8 +6,7 @@ import * as semver from "semver";
 import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "../config";
-import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers";
-import { extLogger } from "../common";
+import { extLogger } from "../common/logging/vscode";
 import { getCodeQlCliVersion } from "./cli-version";
 import {
   ProgressCallback,
@@ -18,11 +17,15 @@ import {
   deprecatedCodeQlLauncherName,
   extractZipArchive,
   getRequiredAssetName,
-} from "../pure/distribution";
+} from "../common/distribution";
 import {
   InvocationRateLimiter,
   InvocationRateLimiterResultKind,
 } from "../common/invocation-rate-limiter";
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+} from "../common/logging";
 
 /**
  * distribution.ts
@@ -157,6 +160,7 @@ export class DistributionManager implements DistributionProvider {
     if (this.config.customCodeQlPath) {
       if (!(await pathExists(this.config.customCodeQlPath))) {
         void showAndLogErrorMessage(
+          extLogger,
           `The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
             "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
             "that a CodeQL executable exists at the specified path or remove the setting.",
@@ -714,7 +718,7 @@ export enum DistributionKind {
   PathEnvironmentVariable,
 }
 
-export interface Distribution {
+interface Distribution {
   codeQlPath: string;
   kind: DistributionKind;
 }
@@ -772,22 +776,22 @@ type DistributionUpdateCheckResult =
   | InvalidLocationResult
   | UpdateAvailableResult;
 
-export interface AlreadyCheckedRecentlyResult {
+interface AlreadyCheckedRecentlyResult {
   kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
 }
 
-export interface AlreadyUpToDateResult {
+interface AlreadyUpToDateResult {
   kind: DistributionUpdateCheckResultKind.AlreadyUpToDate;
 }
 
 /**
  * The distribution could not be installed or updated because it is not managed by the extension.
  */
-export interface InvalidLocationResult {
+interface InvalidLocationResult {
   kind: DistributionUpdateCheckResultKind.InvalidLocation;
 }
 
-export interface UpdateAvailableResult {
+interface UpdateAvailableResult {
   kind: DistributionUpdateCheckResultKind.UpdateAvailable;
   updatedRelease: Release;
 }
@@ -849,6 +853,7 @@ export async function getExecutableFromDirectory(
 
 function warnDeprecatedLauncher() {
   void showAndLogWarningMessage(
+    extLogger,
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
       `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`,
   );
@@ -857,7 +862,7 @@ function warnDeprecatedLauncher() {
 /**
  * A release on GitHub.
  */
-export interface Release {
+interface Release {
   assets: ReleaseAsset[];
 
   /**
@@ -879,7 +884,7 @@ export interface Release {
 /**
  * An asset corresponding to a release on GitHub.
  */
-export interface ReleaseAsset {
+interface ReleaseAsset {
   /**
    * The id associated with the asset on GitHub.
    */
@@ -949,7 +954,10 @@ export interface GithubReleaseAsset {
 }
 
 export class GithubApiError extends Error {
-  constructor(public status: number, public body: string) {
+  constructor(
+    public status: number,
+    public body: string,
+  ) {
     super(`API call failed with status code ${status}, body: ${body}`);
   }
 }

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -0,0 +1,80 @@
+import { CodeQLCliServer } from "./cli";
+import { Uri, window } from "vscode";
+import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { UserCancellationException } from "../common/vscode/progress";
+import { showAndLogErrorMessage } from "../common/logging";
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined,
+): Promise<QueryLanguage | undefined> {
+  const uri = queryUri || window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(
+        getOnDiskWorkspaceFolders(),
+        uri,
+      );
+      const language = Object.keys(queryInfo.byLanguage)[0];
+      void extLogger.log(`Detected query language: ${language}`);
+
+      if (isQueryLanguage(language)) {
+        return language;
+      }
+
+      void extLogger.log(
+        "Query language is unsupported. Select language manually.",
+      );
+    } catch (e) {
+      void extLogger.log(
+        "Could not autodetect query language. Select language manually.",
+      );
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(
+  cliServer: CodeQLCliServer,
+  throwOnEmpty = true,
+): Promise<QueryLanguage | undefined> {
+  const language = await window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    {
+      placeHolder: "Select target language for your query",
+      ignoreFocusOut: true,
+    },
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException("Cancelled.");
+    } else {
+      void showAndLogErrorMessage(
+        extLogger,
+        "Language not found. Language must be specified manually.",
+      );
+    }
+    return undefined;
+  }
+
+  if (!isQueryLanguage(language)) {
+    void showAndLogErrorMessage(
+      extLogger,
+      `Language '${language}' is not supported. Only languages ${Object.values(
+        QueryLanguage,
+      ).join(", ")} are supported.`,
+    );
+    return undefined;
+  }
+
+  return language;
+}

extensions/ql-vscode/src/codeql-cli/query-metadata.ts

@@ -0,0 +1,22 @@
+import { CodeQLCliServer } from "./cli";
+import { QueryMetadata } from "../common/interface-types";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Gets metadata for a query, if it exists.
+ * @param cliServer The CLI server.
+ * @param queryPath The path to the query.
+ * @returns A promise that resolves to the query metadata, if available.
+ */
+export async function tryGetQueryMetadata(
+  cliServer: CodeQLCliServer,
+  queryPath: string,
+): Promise<QueryMetadata | undefined> {
+  try {
+    return await cliServer.resolveMetadata(queryPath);
+  } catch (e) {
+    // Ignore errors and provide no metadata.
+    void extLogger.log(`Couldn't resolve metadata for ${queryPath}: ${e}`);
+    return;
+  }
+}

extensions/ql-vscode/src/common/app.ts

@@ -1,26 +1,21 @@
 import { Credentials } from "./authentication";
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 import { AppEventEmitter } from "./events";
-import { Logger } from "./logging";
+import { NotificationLogger } from "./logging";
 import { Memento } from "./memento";
 import { AppCommandManager } from "./commands";
-import type {
-  WorkspaceFolder,
-  Event,
-  WorkspaceFoldersChangeEvent,
-} from "vscode";
+import { AppTelemetry } from "./telemetry";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
   readonly mode: AppMode;
-  readonly logger: Logger;
+  readonly logger: NotificationLogger;
+  readonly telemetry?: AppTelemetry;
   readonly subscriptions: Disposable[];
   readonly extensionPath: string;
   readonly globalStoragePath: string;
   readonly workspaceStoragePath?: string;
   readonly workspaceState: Memento;
-  readonly workspaceFolders: readonly WorkspaceFolder[] | undefined;
-  readonly onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>;
   readonly credentials: Credentials;
   readonly commands: AppCommandManager;
   readonly environment: EnvironmentContext;

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -1,5 +1,3 @@
-export const PAGE_SIZE = 1000;
-
 /**
  * The single-character codes used in the bqrs format for the the kind
  * of a result column. This namespace is intentionally not an enum, see
@@ -15,7 +13,7 @@ export namespace ColumnKindCode {
   export const ENTITY = "e";
 }
 
-export type ColumnKind =
+type ColumnKind =
   | typeof ColumnKindCode.FLOAT
   | typeof ColumnKindCode.INTEGER
   | typeof ColumnKindCode.STRING
@@ -46,7 +44,7 @@ export function getResultSetSchema(
   }
   return undefined;
 }
-export interface PaginationInfo {
+interface PaginationInfo {
   "step-size": number;
   offsets: number[];
 }

extensions/ql-vscode/src/common/bqrs-utils.ts

@@ -4,7 +4,7 @@ import {
   LineColumnLocation,
   WholeFileLocation,
 } from "./bqrs-cli-types";
-import { createRemoteFileRef } from "./location-link-utils";
+import { createRemoteFileRef } from "../common/location-link-utils";
 
 /**
  * The CodeQL filesystem libraries use this pattern in `getURL()` predicates
@@ -142,5 +142,7 @@ export function tryGetRemoteLocation(
     fileLink,
     resolvableLocation.startLine,
     resolvableLocation.endLine,
+    resolvableLocation.startColumn,
+    resolvableLocation.endColumn,
   );
 }

extensions/ql-vscode/src/common/commands.ts

@@ -4,7 +4,6 @@ import type { AstItem } from "../language-support";
 import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
 import type { DatabaseItem } from "../databases/local-databases";
 import type { QueryHistoryInfo } from "../query-history/query-history-info";
-import type { RepositoriesFilterSortStateWithIds } from "../pure/variant-analysis-filter-sort";
 import type { TestTreeNode } from "../query-testing/test-tree-node";
 import type {
   VariantAnalysis,
@@ -12,6 +11,8 @@ import type {
   VariantAnalysisScannedRepositoryResult,
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
+import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import type { Usage } from "../data-extensions-editor/external-api-usage";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -55,10 +56,11 @@ export type ExplorerSelectionCommandFunction<Item> = (
 
 // Builtin commands where the implementation is provided by VS Code and not by this extension.
 // See https://code.visualstudio.com/api/references/commands
-export type BuiltInVsCodeCommands = {
+type BuiltInVsCodeCommands = {
   // The codeQLDatabases.focus command is provided by VS Code because we've registered the custom view
   "codeQLDatabases.focus": () => Promise<void>;
   "markdown.showPreviewToSide": (uri: Uri) => Promise<void>;
+  "workbench.action.closeActiveEditor": () => Promise<void>;
   revealFileInOS: (uri: Uri) => Promise<void>;
   setContext: (
     key: `${"codeql" | "codeQL"}${string}`,
@@ -129,8 +131,14 @@ export type LocalQueryCommands = {
   "codeQL.runQueryOnMultipleDatabasesContextEditor": (
     uri?: Uri,
   ) => Promise<void>;
+  "codeQLQueries.runLocalQueryFromQueriesPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueryContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesFromPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise<void>;
   "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
   "codeQL.quickEval": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalCount": (uri: Uri) => Promise<void>;
   "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
   "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
   "codeQL.quickQuery": () => Promise<void>;
@@ -237,10 +245,6 @@ export type VariantAnalysisCommands = {
     scannedRepo: VariantAnalysisScannedRepository,
     variantAnalysisSummary: VariantAnalysis,
   ) => Promise<void>;
-  "codeQL.copyVariantAnalysisRepoList": (
-    variantAnalysisId: number,
-    filterSort?: RepositoriesFilterSortStateWithIds,
-  ) => Promise<void>;
   "codeQL.loadVariantAnalysisRepoResults": (
     variantAnalysisId: number,
     repositoryFullName: string,
@@ -262,6 +266,7 @@ export type VariantAnalysisCommands = {
   ) => Promise<void>;
   "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
   "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQLQueries.runVariantAnalysisContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
 };
 
 export type DatabasePanelCommands = {
@@ -299,6 +304,10 @@ export type PackagingCommands = {
 
 export type DataExtensionsEditorCommands = {
   "codeQL.openDataExtensionsEditor": () => Promise<void>;
+  "codeQLDataExtensionsEditor.jumpToUsageLocation": (
+    usage: Usage,
+    databaseItem: DatabaseItem,
+  ) => Promise<void>;
 };
 
 export type EvalLogViewerCommands = {

extensions/ql-vscode/src/common/discovery.ts

@@ -1,5 +1,5 @@
-import { DisposableObject } from "../pure/disposable-object";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { DisposableObject } from "./disposable-object";
+import { getErrorMessage } from "./helpers-pure";
 import { Logger } from "./logging";
 
 /**
@@ -7,11 +7,14 @@ import { Logger } from "./logging";
  * files. This class automatically prevents more than one discovery operation from running at the
  * same time.
  */
-export abstract class Discovery<T> extends DisposableObject {
+export abstract class Discovery extends DisposableObject {
   private restartWhenFinished = false;
   private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string, private readonly logger: Logger) {
+  constructor(
+    protected readonly name: string,
+    private readonly logger: Logger,
+  ) {
     super();
   }
 
@@ -64,14 +67,12 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery.
    */
   private async launchDiscovery(): Promise<void> {
-    let results: T | undefined;
     try {
-      results = await this.discover();
+      await this.discover();
     } catch (err) {
       void this.logger.log(
         `${this.name} failed. Reason: ${getErrorMessage(err)}`,
       );
-      results = undefined;
     }
 
     if (this.restartWhenFinished) {
@@ -82,24 +83,11 @@ export abstract class Discovery<T> extends DisposableObject {
       // succeeded or failed.
       this.restartWhenFinished = false;
       await this.launchDiscovery();
-    } else {
-      // If the discovery was successful, then update any listeners with the results.
-      if (results !== undefined) {
-        this.update(results);
-      }
     }
   }
 
   /**
    * Overridden by the derived class to spawn the actual discovery operation, returning the results.
    */
-  protected abstract discover(): Promise<T>;
-
-  /**
-   * Overridden by the derived class to atomically update the `Discovery` object with the results of
-   * the discovery operation, and to notify any listeners that the discovery results may have
-   * changed.
-   * @param results The discovery results returned by the `discover` function.
-   */
-  protected abstract update(results: T): void;
+  protected abstract discover(): Promise<void>;
 }

extensions/ql-vscode/src/common/events.ts

@@ -1,4 +1,4 @@
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 
 export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;

extensions/ql-vscode/src/common/files.ts

@@ -1,5 +1,6 @@
-import { pathExists, stat, readdir } from "fs-extra";
-import { join, resolve } from "path";
+import { pathExists, stat, readdir, opendir } from "fs-extra";
+import { isAbsolute, join, relative, resolve } from "path";
+import { tmpdir as osTmpdir } from "os";
 
 /**
  * Recursively finds all .ql files in this set of Uris.
@@ -51,36 +52,32 @@ export async function getDirectoryNamesInsidePath(
   return dirNames;
 }
 
-function normalizePath(path: string, platform: NodeJS.Platform): string {
+export function normalizePath(path: string): string {
   // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
   // to normalize the paths to ensure they all get resolved to the
   // same format. On Windows, we also need to do the comparison
   // case-insensitively.
   path = resolve(path);
-  if (platform === "win32") {
+  if (process.platform === "win32") {
     path = path.toLowerCase();
   }
   return path;
 }
 
-export function pathsEqual(
-  path1: string,
-  path2: string,
-  platform: NodeJS.Platform,
-): boolean {
-  return normalizePath(path1, platform) === normalizePath(path2, platform);
+export function pathsEqual(path1: string, path2: string): boolean {
+  return normalizePath(path1) === normalizePath(path2);
 }
 
 /**
- * Returns true if path1 contains path2.
+ * Returns true if `parent` contains `child`, or if they are equal.
  */
-export function containsPath(
-  path1: string,
-  path2: string,
-  platform: NodeJS.Platform,
-): boolean {
-  return normalizePath(path2, platform).startsWith(
-    normalizePath(path1, platform),
+export function containsPath(parent: string, child: string): boolean {
+  const relativePath = relative(parent, child);
+  return (
+    !relativePath.startsWith("..") &&
+    // On windows, if the two paths are in different drives, then the
+    // relative path will be an absolute path to the other drive.
+    !isAbsolute(relativePath)
   );
 }
 
@@ -88,3 +85,45 @@ export async function readDirFullPaths(path: string): Promise<string[]> {
   const baseNames = await readdir(path);
   return baseNames.map((baseName) => join(path, baseName));
 }
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(
+  dir: string,
+): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await opendir(dir)) {
+    const entry = join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}
+
+/**
+ * Error thrown from methods from the `fs` module.
+ *
+ * In practice, any error matching this is likely an instance of `NodeJS.ErrnoException`.
+ * If desired in the future, we could model more fields or use `NodeJS.ErrnoException` directly.
+ */
+export interface IOError {
+  readonly code: string;
+}
+
+export function isIOError(e: any): e is IOError {
+  return e.code !== undefined && typeof e.code === "string";
+}
+
+// This function is a wrapper around `os.tmpdir()` to make it easier to mock in tests.
+export function tmpdir(): string {
+  return osTmpdir();
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -1,4 +1,4 @@
-import { OWNER_REGEX, REPO_REGEX } from "../pure/helpers-pure";
+import { OWNER_REGEX, REPO_REGEX } from "./helpers-pure";
 
 /**
  * Checks if a string is a valid GitHub NWO.

extensions/ql-vscode/src/common/index.ts

@@ -1 +0,0 @@
-export * from "./logging";

extensions/ql-vscode/src/common/interface-types.ts

@@ -5,7 +5,7 @@ import {
   ResultSetSchema,
   Column,
   ResolvableLocationValue,
-} from "./bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import {
   VariantAnalysis,
   VariantAnalysisScannedRepositoryResult,
@@ -14,12 +14,16 @@ import {
 import {
   RepositoriesFilterSortState,
   RepositoriesFilterSortStateWithIds,
-} from "./variant-analysis-filter-sort";
-import { ErrorLike } from "./errors";
+} from "../variant-analysis/shared/variant-analysis-filter-sort";
+import { ErrorLike } from "../common/errors";
 import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
-import { ExternalApiUsage } from "../data-extensions-editor/external-api-usage";
+import {
+  ExternalApiUsage,
+  Usage,
+} from "../data-extensions-editor/external-api-usage";
 import { ModeledMethod } from "../data-extensions-editor/modeled-method";
 import { DataExtensionEditorViewState } from "../data-extensions-editor/shared/view-state";
+import { Mode } from "../data-extensions-editor/shared/mode";
 
 /**
  * This module contains types and code that are shared between
@@ -75,11 +79,9 @@ export type GraphInterpretationData = {
   dot: string[];
 };
 
-export type InterpretationData =
-  | SarifInterpretationData
-  | GraphInterpretationData;
+type InterpretationData = SarifInterpretationData | GraphInterpretationData;
 
-export interface InterpretationT<T> {
+interface InterpretationT<T> {
   sourceLocationPrefix: string;
   numTruncatedResults: number;
   numTotalResults: number;
@@ -105,7 +107,7 @@ export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
  *
  * As a result of receiving this message, listeners might want to display a loading indicator.
  */
-export interface ResultsUpdatingMsg {
+interface ResultsUpdatingMsg {
   t: "resultsUpdating";
 }
 
@@ -113,7 +115,7 @@ export interface ResultsUpdatingMsg {
  * Message to set the initial state of the results view with a new
  * query.
  */
-export interface SetStateMsg {
+interface SetStateMsg {
   t: "setState";
   resultsPath: string;
   origResultsPaths: ResultsPaths;
@@ -142,7 +144,7 @@ export interface SetStateMsg {
  * Message indicating that the results view should display interpreted
  * results.
  */
-export interface ShowInterpretedPageMsg {
+interface ShowInterpretedPageMsg {
   t: "showInterpretedPage";
   interpretation: Interpretation;
   database: DatabaseInfo;
@@ -172,7 +174,7 @@ export interface NavigateMsg {
  * A message indicating that the results view should untoggle the
  * "Show results in Problems view" checkbox.
  */
-export interface UntoggleShowProblemsMsg {
+interface UntoggleShowProblemsMsg {
   t: "untoggleShowProblems";
 }
 
@@ -202,7 +204,7 @@ export type FromResultsViewMsg =
  * Message from the results view to open a database source
  * file at the provided location.
  */
-export interface ViewSourceFileMsg {
+interface ViewSourceFileMsg {
   t: "viewSourceFile";
   loc: ResolvableLocationValue;
   databaseUri: string;
@@ -211,7 +213,7 @@ export interface ViewSourceFileMsg {
 /**
  * Message from the results view to open a file in an editor.
  */
-export interface OpenFileMsg {
+interface OpenFileMsg {
   t: "openFile";
   /* Full path to the file to open. */
   filePath: string;
@@ -273,7 +275,7 @@ export interface RawResultsSortState {
   sortDirection: SortDirection;
 }
 
-export type InterpretedResultsSortColumn = "alert-message";
+type InterpretedResultsSortColumn = "alert-message";
 
 export interface InterpretedResultsSortState {
   sortBy: InterpretedResultsSortColumn;
@@ -317,7 +319,7 @@ export type FromCompareViewMessage =
 /**
  * Message from the compare view to request opening a query.
  */
-export interface OpenQueryMessage {
+interface OpenQueryMessage {
   readonly t: "openQuery";
   readonly kind: "from" | "to";
 }
@@ -405,12 +407,12 @@ export interface ParsedResultSets {
   resultSet: ResultSet;
 }
 
-export interface SetVariantAnalysisMessage {
+interface SetVariantAnalysisMessage {
   t: "setVariantAnalysis";
   variantAnalysis: VariantAnalysis;
 }
 
-export interface SetFilterSortStateMessage {
+interface SetFilterSortStateMessage {
   t: "setFilterSortState";
   filterSortState: RepositoriesFilterSortState;
 }
@@ -419,48 +421,48 @@ export type VariantAnalysisState = {
   variantAnalysisId: number;
 };
 
-export interface SetRepoResultsMessage {
+interface SetRepoResultsMessage {
   t: "setRepoResults";
   repoResults: VariantAnalysisScannedRepositoryResult[];
 }
 
-export interface SetRepoStatesMessage {
+interface SetRepoStatesMessage {
   t: "setRepoStates";
   repoStates: VariantAnalysisScannedRepositoryState[];
 }
 
-export interface RequestRepositoryResultsMessage {
+interface RequestRepositoryResultsMessage {
   t: "requestRepositoryResults";
   repositoryFullName: string;
 }
 
-export interface OpenQueryFileMessage {
+interface OpenQueryFileMessage {
   t: "openQueryFile";
 }
 
-export interface OpenQueryTextMessage {
+interface OpenQueryTextMessage {
   t: "openQueryText";
 }
 
-export interface CopyRepositoryListMessage {
+interface CopyRepositoryListMessage {
   t: "copyRepositoryList";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface ExportResultsMessage {
+interface ExportResultsMessage {
   t: "exportResults";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface OpenLogsMessage {
+interface OpenLogsMessage {
   t: "openLogs";
 }
 
-export interface CancelVariantAnalysisMessage {
+interface CancelVariantAnalysisMessage {
   t: "cancelVariantAnalysis";
 }
 
-export interface ShowDataFlowPathsMessage {
+interface ShowDataFlowPathsMessage {
   t: "showDataFlowPaths";
   dataFlowPaths: DataFlowPaths;
 }
@@ -482,7 +484,7 @@ export type FromVariantAnalysisMessage =
   | CancelVariantAnalysisMessage
   | ShowDataFlowPathsMessage;
 
-export interface SetDataFlowPathsMessage {
+interface SetDataFlowPathsMessage {
   t: "setDataFlowPaths";
   dataFlowPaths: DataFlowPaths;
 }
@@ -491,76 +493,96 @@ export type ToDataFlowPathsMessage = SetDataFlowPathsMessage;
 
 export type FromDataFlowPathsMessage = CommonFromViewMessages;
 
-export interface SetExtensionPackStateMessage {
+interface SetExtensionPackStateMessage {
   t: "setDataExtensionEditorViewState";
   viewState: DataExtensionEditorViewState;
 }
 
-export interface SetExternalApiUsagesMessage {
+interface SetExternalApiUsagesMessage {
   t: "setExternalApiUsages";
   externalApiUsages: ExternalApiUsage[];
 }
 
-export interface ShowProgressMessage {
-  t: "showProgress";
-  step: number;
-  maxStep: number;
-  message: string;
+interface LoadModeledMethodsMessage {
+  t: "loadModeledMethods";
+  modeledMethods: Record<string, ModeledMethod>;
 }
 
-export interface AddModeledMethodsMessage {
+interface AddModeledMethodsMessage {
   t: "addModeledMethods";
   modeledMethods: Record<string, ModeledMethod>;
-
-  /**
-   * If true, then any existing modeled methods set to "none" will be
-   * overwritten by the new modeled methods. Otherwise, the "none" modeled
-   * methods will not be overwritten, even if the new modeled methods
-   * contain a better model.
-   */
-  overrideNone?: boolean;
 }
 
-export interface JumpToUsageMessage {
+interface SetInProgressMethodsMessage {
+  t: "setInProgressMethods";
+  packageName: string;
+  inProgressMethods: string[];
+}
+
+interface SwitchModeMessage {
+  t: "switchMode";
+  mode: Mode;
+}
+
+interface JumpToUsageMessage {
   t: "jumpToUsage";
-  location: ResolvableLocationValue;
+  usage: Usage;
 }
 
-export interface OpenExtensionPackMessage {
+interface OpenDatabaseMessage {
+  t: "openDatabase";
+}
+
+interface OpenExtensionPackMessage {
   t: "openExtensionPack";
 }
 
-export interface OpenModelFileMessage {
-  t: "openModelFile";
+interface RefreshExternalApiUsages {
+  t: "refreshExternalApiUsages";
 }
 
-export interface SaveModeledMethods {
+interface SaveModeledMethods {
   t: "saveModeledMethods";
   externalApiUsages: ExternalApiUsage[];
   modeledMethods: Record<string, ModeledMethod>;
 }
 
-export interface GenerateExternalApiMessage {
+interface GenerateExternalApiMessage {
   t: "generateExternalApi";
 }
 
-export interface GenerateExternalApiFromLlmMessage {
+interface GenerateExternalApiFromLlmMessage {
   t: "generateExternalApiFromLlm";
+  packageName: string;
   externalApiUsages: ExternalApiUsage[];
   modeledMethods: Record<string, ModeledMethod>;
 }
 
+interface StopGeneratingExternalApiFromLlmMessage {
+  t: "stopGeneratingExternalApiFromLlm";
+  packageName: string;
+}
+
+interface ModelDependencyMessage {
+  t: "modelDependency";
+}
+
 export type ToDataExtensionsEditorMessage =
   | SetExtensionPackStateMessage
   | SetExternalApiUsagesMessage
-  | ShowProgressMessage
-  | AddModeledMethodsMessage;
+  | LoadModeledMethodsMessage
+  | AddModeledMethodsMessage
+  | SetInProgressMethodsMessage;
 
 export type FromDataExtensionsEditorMessage =
   | ViewLoadedMsg
-  | OpenModelFileMessage
+  | SwitchModeMessage
+  | RefreshExternalApiUsages
+  | OpenDatabaseMessage
   | OpenExtensionPackMessage
   | JumpToUsageMessage
   | SaveModeledMethods
   | GenerateExternalApiMessage
-  | GenerateExternalApiFromLlmMessage;
+  | GenerateExternalApiFromLlmMessage
+  | StopGeneratingExternalApiFromLlmMessage
+  | ModelDependencyMessage;

extensions/ql-vscode/src/common/location-link-utils.ts

@@ -0,0 +1,27 @@
+import { FileLink } from "../variant-analysis/shared/analysis-result";
+
+export function createRemoteFileRef(
+  fileLink: FileLink,
+  startLine?: number,
+  endLine?: number,
+  startColumn?: number,
+  endColumn?: number,
+): string {
+  if (
+    startColumn &&
+    endColumn &&
+    startLine &&
+    endLine &&
+    // Verify that location information is valid; otherwise highlighting might be broken
+    ((startLine === endLine && startColumn < endColumn) || startLine < endLine)
+  ) {
+    // This relies on column highlighting of new code view on GitHub
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}C${startColumn}-L${endLine}C${endColumn}`;
+  } else if (startLine && endLine && startLine < endLine) {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}-L${endLine}`;
+  } else if (startLine) {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}`;
+  } else {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}`;
+  }
+}

extensions/ql-vscode/src/common/logging/index.ts

@@ -1,4 +1,4 @@
 export * from "./logger";
+export * from "./notification-logger";
+export * from "./notifications";
 export * from "./tee-logger";
-export * from "./vscode/loggers";
-export * from "./vscode/output-channel-logger";

extensions/ql-vscode/src/common/logging/notification-logger.ts

@@ -0,0 +1,7 @@
+import { Logger } from "./logger";
+
+export interface NotificationLogger extends Logger {
+  showErrorMessage(message: string): Promise<void>;
+  showWarningMessage(message: string): Promise<void>;
+  showInformationMessage(message: string): Promise<void>;
+}

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -0,0 +1,116 @@
+import { NotificationLogger } from "./notification-logger";
+import { AppTelemetry } from "../telemetry";
+import { RedactableError } from "../errors";
+
+interface ShowAndLogOptions {
+  /**
+   * An alternate message that is added to the log, but not displayed in the popup.
+   * This is useful for adding extra detail to the logs that would be too noisy for the popup.
+   */
+  fullMessage?: string;
+}
+
+/**
+ * Show an error message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogErrorMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    dropLinesExceptInitial(message),
+    logger.showErrorMessage,
+    { fullMessage: message, ...options },
+  );
+}
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join("\n");
+}
+
+/**
+ * Show a warning message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogWarningMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showWarningMessage,
+    options,
+  );
+}
+
+/**
+ * Show an information message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogInformationMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showInformationMessage,
+    options,
+  );
+}
+
+async function internalShowAndLog(
+  logger: NotificationLogger,
+  message: string,
+  fn: (message: string) => Promise<void>,
+  { fullMessage }: ShowAndLogOptions = {},
+): Promise<void> {
+  void logger.log(fullMessage || message);
+  await fn.bind(logger)(message);
+}
+
+interface ShowAndLogExceptionOptions extends ShowAndLogOptions {
+  /** Custom properties to include in the telemetry report. */
+  extraTelemetryProperties?: { [key: string]: string };
+}
+
+/**
+ * Show an error message, log it to the console, and emit redacted information as telemetry
+ *
+ * @param logger The logger that will receive the message.
+ * @param telemetry The telemetry instance to use for reporting.
+ * @param error The error to show. Only redacted information will be included in the telemetry.
+ * @param options See individual fields on `ShowAndLogExceptionOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogExceptionWithTelemetry(
+  logger: NotificationLogger,
+  telemetry: AppTelemetry | undefined,
+  error: RedactableError,
+  options: ShowAndLogExceptionOptions = {},
+): Promise<void> {
+  telemetry?.sendError(error, options.extraTelemetryProperties);
+  return showAndLogErrorMessage(logger, error.fullMessage, options);
+}

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -1,6 +1,6 @@
 import { appendFile, ensureFile } from "fs-extra";
 import { isAbsolute } from "path";
-import { getErrorMessage } from "../../pure/helpers-pure";
+import { getErrorMessage } from "../helpers-pure";
 import { Logger, LogOptions } from "./logger";
 
 /**

extensions/ql-vscode/src/common/logging/vscode/index.ts

@@ -0,0 +1,2 @@
+export * from "./loggers";
+export * from "./output-channel-logger";

extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts

@@ -1,11 +1,15 @@
 import { window as Window, OutputChannel, Progress } from "vscode";
 import { Logger, LogOptions } from "../logger";
-import { DisposableObject } from "../../../pure/disposable-object";
+import { DisposableObject } from "../../disposable-object";
+import { NotificationLogger } from "../notification-logger";
 
 /**
  * A logger that writes messages to an output channel in the VS Code Output tab.
  */
-export class OutputChannelLogger extends DisposableObject implements Logger {
+export class OutputChannelLogger
+  extends DisposableObject
+  implements Logger, NotificationLogger
+{
   public readonly outputChannel: OutputChannel;
   isCustomLogDirectory: boolean;
 
@@ -42,6 +46,30 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   show(preserveFocus?: boolean): void {
     this.outputChannel.show(preserveFocus);
   }
+
+  async showErrorMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showErrorMessage);
+  }
+
+  async showInformationMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showInformationMessage);
+  }
+
+  async showWarningMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showWarningMessage);
+  }
+
+  private async showMessage(
+    message: string,
+    show: (message: string, ...items: string[]) => Thenable<string | undefined>,
+  ): Promise<void> {
+    const label = "Show Log";
+    const result = await show(message, label);
+
+    if (result === label) {
+      this.show();
+    }
+  }
 }
 
 export type ProgressReporter = Progress<{ message: string }>;

extensions/ql-vscode/src/common/path.ts

@@ -19,3 +19,11 @@ export const basename = (path: string): string => {
   const index = path.lastIndexOf("\\");
   return index === -1 ? path : path.slice(index + 1);
 };
+
+// Returns the extension of a path, including the leading dot.
+export const extname = (path: string): string => {
+  const name = basename(path);
+
+  const index = name.lastIndexOf(".");
+  return index === -1 ? "" : name.slice(index);
+};

extensions/ql-vscode/src/common/query-language.ts

@@ -9,6 +9,29 @@ export enum QueryLanguage {
   Swift = "swift",
 }
 
+export function getLanguageDisplayName(language: string): string {
+  switch (language) {
+    case QueryLanguage.CSharp:
+      return "C#";
+    case QueryLanguage.Cpp:
+      return "C / C++";
+    case QueryLanguage.Go:
+      return "Go";
+    case QueryLanguage.Java:
+      return "Java";
+    case QueryLanguage.Javascript:
+      return "JavaScript";
+    case QueryLanguage.Python:
+      return "Python";
+    case QueryLanguage.Ruby:
+      return "Ruby";
+    case QueryLanguage.Swift:
+      return "Swift";
+    default:
+      return language;
+  }
+}
+
 export const PACKS_BY_QUERY_LANGUAGE = {
   [QueryLanguage.Cpp]: ["codeql/cpp-queries"],
   [QueryLanguage.CSharp]: [
@@ -25,13 +48,17 @@ export const PACKS_BY_QUERY_LANGUAGE = {
   [QueryLanguage.Ruby]: ["codeql/ruby-queries"],
 };
 
-export const dbSchemeToLanguage = {
-  "semmlecode.javascript.dbscheme": "javascript",
-  "semmlecode.cpp.dbscheme": "cpp",
-  "semmlecode.dbscheme": "java",
-  "semmlecode.python.dbscheme": "python",
-  "semmlecode.csharp.dbscheme": "csharp",
-  "go.dbscheme": "go",
-  "ruby.dbscheme": "ruby",
-  "swift.dbscheme": "swift",
+export const dbSchemeToLanguage: Record<string, QueryLanguage> = {
+  "semmlecode.javascript.dbscheme": QueryLanguage.Javascript,
+  "semmlecode.cpp.dbscheme": QueryLanguage.Cpp,
+  "semmlecode.dbscheme": QueryLanguage.Java,
+  "semmlecode.python.dbscheme": QueryLanguage.Python,
+  "semmlecode.csharp.dbscheme": QueryLanguage.CSharp,
+  "go.dbscheme": QueryLanguage.Go,
+  "ruby.dbscheme": QueryLanguage.Ruby,
+  "swift.dbscheme": QueryLanguage.Swift,
 };
+
+export function isQueryLanguage(language: string): language is QueryLanguage {
+  return Object.values(QueryLanguage).includes(language as QueryLanguage);
+}

extensions/ql-vscode/src/common/sarif-parser.ts

@@ -1,7 +1,7 @@
 import * as Sarif from "sarif";
 import { createReadStream } from "fs-extra";
 import { connectTo } from "stream-json/Assembler";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { getErrorMessage } from "./helpers-pure";
 import { withParser } from "stream-json/filters/Pick";
 
 const DUMMY_TOOL: Sarif.Tool = { driver: { name: "" } };

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -1,6 +1,6 @@
 import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
-import { ResolvableLocationValue } from "./bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 
 export interface SarifLink {
   dest: number;
@@ -24,7 +24,7 @@ type ParsedSarifLocation =
   // that, and is appropriate for display in the UI.
   | NoLocation;
 
-export type SarifMessageComponent = string | SarifLink;
+type SarifMessageComponent = string | SarifLink;
 
 /**
  * Unescape "[", "]" and "\\" like in sarif plain text messages
@@ -203,7 +203,7 @@ export function shouldHighlightLine(
  * A line of code split into: plain text before the highlighted section, the highlighted
  * text itself, and plain text after the highlighted section.
  */
-export interface PartiallyHighlightedLine {
+interface PartiallyHighlightedLine {
   plainSection1: string;
   highlightedSection: string;
   plainSection2: string;

extensions/ql-vscode/src/common/telemetry.ts

@@ -0,0 +1,10 @@
+import { RedactableError } from "./errors";
+
+export interface AppTelemetry {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void;
+  sendUIInteraction(name: string): void;
+  sendError(
+    error: RedactableError,
+    extraProperties?: { [key: string]: string },
+  ): void;
+}

extensions/ql-vscode/src/common/time.ts

@@ -2,16 +2,16 @@
  * Contains an assortment of helper constants and functions for working with time, dates, and durations.
  */
 
-export const ONE_SECOND_IN_MS = 1000;
-export const ONE_MINUTE_IN_MS = ONE_SECOND_IN_MS * 60;
+const ONE_SECOND_IN_MS = 1000;
+const ONE_MINUTE_IN_MS = ONE_SECOND_IN_MS * 60;
 export const ONE_HOUR_IN_MS = ONE_MINUTE_IN_MS * 60;
 export const TWO_HOURS_IN_MS = ONE_HOUR_IN_MS * 2;
 export const THREE_HOURS_IN_MS = ONE_HOUR_IN_MS * 3;
 export const ONE_DAY_IN_MS = ONE_HOUR_IN_MS * 24;
 
 // These are approximations
-export const ONE_MONTH_IN_MS = ONE_DAY_IN_MS * 30;
-export const ONE_YEAR_IN_MS = ONE_DAY_IN_MS * 365;
+const ONE_MONTH_IN_MS = ONE_DAY_IN_MS * 30;
+const ONE_YEAR_IN_MS = ONE_DAY_IN_MS * 365;
 
 const durationFormatter = new Intl.RelativeTimeFormat("en", {
   numeric: "auto",

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -9,13 +9,9 @@ import {
 } from "vscode";
 import { join } from "path";
 
-import { DisposableObject, DisposeHandler } from "../../pure/disposable-object";
-import { tmpDir } from "../../helpers";
-import {
-  getHtmlForWebview,
-  WebviewMessage,
-  WebviewView,
-} from "../../interface-utils";
+import { DisposableObject, DisposeHandler } from "../disposable-object";
+import { tmpDir } from "../../tmp-dir";
+import { getHtmlForWebview, WebviewMessage, WebviewView } from "./webview-html";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -23,6 +19,7 @@ export type WebviewPanelConfig = {
   viewColumn: ViewColumn;
   view: WebviewView;
   preserveFocus?: boolean;
+  iconPath?: Uri | { dark: Uri; light: Uri };
   additionalOptions?: WebviewPanelOptions & WebviewOptions;
   allowWasmEval?: boolean;
 };
@@ -90,6 +87,8 @@ export abstract class AbstractWebview<
       );
       this.panel = panel;
 
+      this.panel.iconPath = config.iconPath;
+
       this.setupPanel(panel, config);
 
       this.panelResolves.forEach((resolve) => resolve(panel));

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -1,20 +1,23 @@
 import { pathExists } from "fs-extra";
 import * as unzipper from "unzipper";
 import * as vscode from "vscode";
-import { extLogger } from "..";
+import { extLogger } from "../logging/vscode";
 
 // All path operations in this file must be on paths *within* the zip
 // archive.
 import { posix } from "path";
 const path = posix;
 
-export class File implements vscode.FileStat {
+class File implements vscode.FileStat {
   type: vscode.FileType;
   ctime: number;
   mtime: number;
   size: number;
 
-  constructor(public name: string, public data: Uint8Array) {
+  constructor(
+    public name: string,
+    public data: Uint8Array,
+  ) {
     this.type = vscode.FileType.File;
     this.ctime = Date.now();
     this.mtime = Date.now();
@@ -23,7 +26,7 @@ export class File implements vscode.FileStat {
   }
 }
 
-export class Directory implements vscode.FileStat {
+class Directory implements vscode.FileStat {
   type: vscode.FileType;
   ctime: number;
   mtime: number;
@@ -38,7 +41,7 @@ export class Directory implements vscode.FileStat {
   }
 }
 
-export type Entry = File | Directory;
+type Entry = File | Directory;
 
 /**
  * A map containing directory hierarchy information in a convenient form.
@@ -49,7 +52,7 @@ export type Entry = File | Directory;
  * dirMap['/foo'] = {'bar': vscode.FileType.Directory}
  * dirMap['/foo/bar'] = {'baz': vscode.FileType.File}
  */
-export type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
+type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
 
 export type ZipFileReference = {
   sourceArchiveZipPath: string;

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -1,18 +1,20 @@
 import { commands, Disposable } from "vscode";
 import { CommandFunction, CommandManager } from "../../packages/commands";
-import { extLogger, OutputChannelLogger } from "../logging";
+import {
+  NotificationLogger,
+  showAndLogWarningMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../logging";
+import { extLogger } from "../logging/vscode";
 import {
   asError,
   getErrorMessage,
   getErrorStack,
-} from "../../pure/helpers-pure";
-import { redactableError } from "../../pure/errors";
+} from "../../common/helpers-pure";
+import { redactableError } from "../../common/errors";
 import { UserCancellationException } from "./progress";
-import {
-  showAndLogExceptionWithTelemetry,
-  showAndLogWarningMessage,
-} from "../../helpers";
-import { telemetryListener } from "../../telemetry";
+import { telemetryListener } from "./telemetry";
+import { AppTelemetry } from "../telemetry";
 
 /**
  * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -20,9 +22,12 @@ import { telemetryListener } from "../../telemetry";
  */
 export function createVSCodeCommandManager<
   Commands extends Record<string, CommandFunction>,
->(outputLogger?: OutputChannelLogger): CommandManager<Commands> {
+>(
+  logger?: NotificationLogger,
+  telemetry?: AppTelemetry,
+): CommandManager<Commands> {
   return new CommandManager((commandId, task) => {
-    return registerCommandWithErrorHandling(commandId, task, outputLogger);
+    return registerCommandWithErrorHandling(commandId, task, logger, telemetry);
   }, wrapExecuteCommand);
 }
 
@@ -32,11 +37,14 @@ export function createVSCodeCommandManager<
  * @param commandId The ID of the command to register.
  * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
  * arguments to the command handler are passed on to the task.
+ * @param logger The logger to use for error reporting.
+ * @param telemetry The telemetry listener to use for error reporting.
  */
 export function registerCommandWithErrorHandling(
   commandId: string,
   task: (...args: any[]) => Promise<any>,
-  outputLogger = extLogger,
+  logger: NotificationLogger = extLogger,
+  telemetry: AppTelemetry | undefined = telemetryListener,
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     const startTime = Date.now();
@@ -52,11 +60,9 @@ export function registerCommandWithErrorHandling(
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          void outputLogger.log(errorMessage.fullMessage);
+          void logger.log(errorMessage.fullMessage);
         } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage, {
-            outputLogger,
-          });
+          void showAndLogWarningMessage(logger, errorMessage.fullMessage);
         }
       } else {
         // Include the full stack in the error log only.
@@ -64,8 +70,7 @@ export function registerCommandWithErrorHandling(
         const fullMessage = errorStack
           ? `${errorMessage.fullMessage}\n${errorStack}`
           : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          outputLogger,
+        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
           fullMessage,
           extraTelemetryProperties: {
             command: commandId,

extensions/ql-vscode/src/common/vscode/dialog.ts

@@ -0,0 +1,135 @@
+import { env, Uri, window } from "vscode";
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+): Promise<boolean | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: false };
+  const noItem = { title: noTitle, isCloseAffordance: true };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+  );
+  if (!chosenItem) {
+    return undefined;
+  }
+  return chosenItem?.title === yesItem.title;
+}
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(
+  message: string,
+  url: string,
+): Promise<boolean | undefined> {
+  const urlItem = { title: "More Information", isCloseAffordance: false };
+  const yesItem = { title: "Yes", isCloseAffordance: false };
+  const noItem = { title: "No", isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await window.showInformationMessage(
+      message,
+      { modal: true },
+      urlItem,
+      yesItem,
+      noItem,
+    );
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
+/**
+ * Show an information message with a customisable action.
+ * @param message The message to show.
+ * @param actionMessage The call to action message.
+ *
+ * @return `true` if the user clicks the action, `false` if the user cancels the dialog.
+ */
+export async function showInformationMessageWithAction(
+  message: string,
+  actionMessage: string,
+): Promise<boolean> {
+  const actionItem = { title: actionMessage, isCloseAffordance: false };
+  const chosenItem = await window.showInformationMessage(message, actionItem);
+  return chosenItem === actionItem;
+}
+
+/**
+ * Opens a modal dialog for the user to make a choice between yes/no/never be asked again.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ * @param neverTitle The text in the box indicating the opt out choice.
+ *
+ * @return
+ *  `Yes` if the user clicks 'Yes',
+ *  `No` if the user clicks 'No' or cancels the dialog,
+ *  `No, and never ask me again` if the user clicks 'No, and never ask me again',
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showNeverAskAgainDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+  neverAskAgainTitle = "No, and never ask me again",
+): Promise<string | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: true };
+  const noItem = { title: noTitle, isCloseAffordance: false };
+  const neverAskAgainItem = {
+    title: neverAskAgainTitle,
+    isCloseAffordance: false,
+  };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+    neverAskAgainItem,
+  );
+
+  return chosenItem?.title;
+}

extensions/ql-vscode/src/common/vscode/external-files.ts

@@ -1,15 +1,15 @@
 import { Uri, window } from "vscode";
 import { AppCommandManager } from "../commands";
-import {
-  showAndLogExceptionWithTelemetry,
-  showBinaryChoiceDialog,
-} from "../../helpers";
-import { redactableError } from "../../pure/errors";
+import { showBinaryChoiceDialog } from "./dialog";
+import { redactableError } from "../../common/errors";
 import {
   asError,
   getErrorMessage,
   getErrorStack,
-} from "../../pure/helpers-pure";
+} from "../../common/helpers-pure";
+import { showAndLogExceptionWithTelemetry } from "../logging";
+import { extLogger } from "../logging/vscode";
+import { telemetryListener } from "./telemetry";
 
 export async function tryOpenExternalFile(
   commandManager: AppCommandManager,
@@ -36,6 +36,8 @@ the file in the file explorer and dragging it into the workspace.`,
           await commandManager.execute("revealFileInOS", uri);
         } catch (e) {
           void showAndLogExceptionWithTelemetry(
+            extLogger,
+            telemetryListener,
             redactableError(
               asError(e),
             )`Failed to reveal file in OS: ${getErrorMessage(e)}`,
@@ -44,6 +46,8 @@ the file in the file explorer and dragging it into the workspace.`,
       }
     } else {
       void showAndLogExceptionWithTelemetry(
+        extLogger,
+        telemetryListener,
         redactableError(asError(e))`Could not open file ${fileLocation}`,
         {
           fullMessage: `${getErrorMessage(e)}\n${getErrorStack(e)}`,

extensions/ql-vscode/src/common/vscode/file-path-discovery.ts

@@ -0,0 +1,272 @@
+import { Discovery } from "../discovery";
+import {
+  Event,
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFoldersChangeEvent,
+  workspace,
+} from "vscode";
+import { MultiFileSystemWatcher } from "./multi-file-system-watcher";
+import { AppEventEmitter } from "../events";
+import { extLogger } from "../logging/vscode";
+import { lstat } from "fs-extra";
+import { containsPath, isIOError } from "../files";
+import {
+  getOnDiskWorkspaceFolders,
+  getOnDiskWorkspaceFoldersObjects,
+} from "./workspace-folders";
+import { getErrorMessage } from "../../common/helpers-pure";
+
+interface PathData {
+  path: string;
+}
+
+/**
+ * Discovers and watches for changes to all files matching a given filter
+ * contained in the workspace. Also allows computing extra data about each
+ * file path, and only recomputing the data when the file changes.
+ *
+ * Scans the whole workspace on startup, and then watches for changes to files
+ * to do the minimum work to keep up with changes.
+ *
+ * Can configure which changes it watches for, which files are considered
+ * relevant, and what extra data to compute for each file.
+ */
+export abstract class FilePathDiscovery<T extends PathData> extends Discovery {
+  /**
+   * Has `discover` been called. This allows distinguishing between
+   * "no paths found" and not having scanned yet.
+   */
+  private discoverHasCompletedOnce = false;
+
+  /** The set of known paths and associated data that we are tracking */
+  private pathData: T[] = [];
+
+  /** Event that fires whenever the contents of `pathData` changes */
+  private readonly onDidChangePathDataEmitter: AppEventEmitter<void>;
+
+  /**
+   * The set of file paths that may have changed on disk since the last time
+   * refresh was run. Whenever a watcher reports some change to a file we add
+   * it to this set, and then during the next refresh we will process all
+   * file paths from this set and update our internal state to match whatever
+   * we find on disk (i.e. the file exists, doesn't exist, computed data has
+   * changed).
+   */
+  private readonly changedFilePaths = new Set<string>();
+
+  /**
+   * Watches for changes to files and directories in all workspace folders.
+   */
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  /**
+   * @param name Name of the discovery operation, for logging purposes.
+   * @param fileWatchPattern Passed to `vscode.RelativePattern` to determine the files to watch for changes to.
+   */
+  constructor(
+    name: string,
+    private readonly fileWatchPattern: string,
+  ) {
+    super(name, extLogger);
+
+    this.onDidChangePathDataEmitter = this.push(new EventEmitter<void>());
+    this.push(
+      workspace.onDidChangeWorkspaceFolders(
+        this.workspaceFoldersChanged.bind(this),
+      ),
+    );
+    this.push(this.watcher.onDidChange(this.fileChanged.bind(this)));
+  }
+
+  protected getPathData(): ReadonlyArray<Readonly<T>> | undefined {
+    if (!this.discoverHasCompletedOnce) {
+      return undefined;
+    }
+    return this.pathData;
+  }
+
+  protected get onDidChangePathData(): Event<void> {
+    return this.onDidChangePathDataEmitter.event;
+  }
+
+  /**
+   * Compute any extra data to be stored regarding the given path.
+   */
+  protected abstract getDataForPath(path: string): Promise<T>;
+
+  /**
+   * Is the given path relevant to this discovery operation?
+   */
+  protected abstract pathIsRelevant(path: string): boolean;
+
+  /**
+   * Should the given new data overwrite the existing data we have stored?
+   */
+  protected abstract shouldOverwriteExistingData(
+    newData: T,
+    existingData: T,
+  ): boolean;
+
+  /**
+   * Update the data for every path by calling `getDataForPath`.
+   */
+  protected async recomputeAllData() {
+    this.pathData = await Promise.all(
+      this.pathData.map((p) => this.getDataForPath(p.path)),
+    );
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  /**
+   * Do the initial scan of the entire workspace and set up watchers for future changes.
+   */
+  public async initialRefresh() {
+    getOnDiskWorkspaceFolders().forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder);
+    });
+
+    this.updateWatchers();
+    await this.refresh();
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  private workspaceFoldersChanged(event: WorkspaceFoldersChangeEvent) {
+    event.added.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+    event.removed.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+
+    this.updateWatchers();
+    void this.refresh();
+  }
+
+  private updateWatchers() {
+    this.watcher.clear();
+    for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
+      // Watch for changes to individual files
+      this.watcher.addWatch(
+        new RelativePattern(workspaceFolder, this.fileWatchPattern),
+      );
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(workspaceFolder, "**/"));
+    }
+  }
+
+  private fileChanged(uri: Uri) {
+    this.changedFilePaths.add(uri.fsPath);
+    void this.refresh();
+  }
+
+  protected async discover() {
+    let pathsUpdated = false;
+    for (const path of this.changedFilePaths) {
+      try {
+        this.changedFilePaths.delete(path);
+        if (await this.handleChangedPath(path)) {
+          pathsUpdated = true;
+        }
+      } catch (e) {
+        // If we get an error while processing a path, just log it and continue.
+        // There aren't any network operations happening here or anything else
+        // that's likely to succeed on a retry, so don't bother adding it back
+        // to the changedFilePaths set.
+        void extLogger.log(
+          `${
+            this.name
+          } failed while processing path "${path}": ${getErrorMessage(e)}`,
+        );
+      }
+    }
+
+    this.discoverHasCompletedOnce = true;
+    if (pathsUpdated) {
+      this.onDidChangePathDataEmitter.fire();
+    }
+  }
+
+  private async handleChangedPath(path: string): Promise<boolean> {
+    try {
+      // If the path is not in the workspace then we don't want to be
+      // tracking or displaying it, so treat it as if it doesn't exist.
+      if (!this.pathIsInWorkspace(path)) {
+        return this.handleRemovedPath(path);
+      }
+
+      if ((await lstat(path)).isDirectory()) {
+        return await this.handleChangedDirectory(path);
+      } else {
+        return this.handleChangedFile(path);
+      }
+    } catch (e) {
+      if (isIOError(e) && e.code === "ENOENT") {
+        return this.handleRemovedPath(path);
+      }
+      throw e;
+    }
+  }
+
+  private pathIsInWorkspace(path: string): boolean {
+    return getOnDiskWorkspaceFolders().some((workspaceFolder) =>
+      containsPath(workspaceFolder, path),
+    );
+  }
+
+  private handleRemovedPath(path: string): boolean {
+    const oldLength = this.pathData.length;
+    this.pathData = this.pathData.filter(
+      (existingPathData) => !containsPath(path, existingPathData.path),
+    );
+    return this.pathData.length !== oldLength;
+  }
+
+  private async handleChangedDirectory(path: string): Promise<boolean> {
+    const newPaths = await workspace.findFiles(
+      new RelativePattern(path, this.fileWatchPattern),
+    );
+
+    let pathsUpdated = false;
+    for (const path of newPaths) {
+      if (await this.addOrUpdatePath(path.fsPath)) {
+        pathsUpdated = true;
+      }
+    }
+    return pathsUpdated;
+  }
+
+  private async handleChangedFile(path: string): Promise<boolean> {
+    if (this.pathIsRelevant(path)) {
+      return await this.addOrUpdatePath(path);
+    } else {
+      return false;
+    }
+  }
+
+  private async addOrUpdatePath(path: string): Promise<boolean> {
+    const data = await this.getDataForPath(path);
+    const existingPathDataIndex = this.pathData.findIndex(
+      (existingPathData) => existingPathData.path === path,
+    );
+    if (existingPathDataIndex !== -1) {
+      if (
+        this.shouldOverwriteExistingData(
+          data,
+          this.pathData[existingPathDataIndex],
+        )
+      ) {
+        this.pathData.splice(existingPathDataIndex, 1, data);
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      this.pathData.push(data);
+      return true;
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from "../../pure/disposable-object";
+import { DisposableObject } from "../disposable-object";
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from "vscode";
 
 /**

extensions/ql-vscode/src/common/vscode/progress.ts

@@ -10,7 +10,10 @@ export class UserCancellationException extends Error {
    * @param message The error message
    * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
    */
-  constructor(message?: string, public readonly silent = false) {
+  constructor(
+    message?: string,
+    public readonly silent = false,
+  ) {
     super(message);
   }
 }
@@ -35,7 +38,7 @@ export type ProgressCallback = (p: ProgressUpdate) => void;
 // Make certain properties within a type optional
 type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
 
-export type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
+type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
 
 /**
  * A task that reports progress.

extensions/ql-vscode/src/common/vscode/selection-commands.ts

@@ -1,9 +1,9 @@
-import { showAndLogErrorMessage } from "../../helpers";
 import {
   ExplorerSelectionCommandFunction,
   TreeViewContextMultiSelectionCommandFunction,
   TreeViewContextSingleSelectionCommandFunction,
 } from "../commands";
+import { showAndLogErrorMessage, NotificationLogger } from "../logging";
 
 // A hack to match types that are not an array, which is useful to help avoid
 // misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
@@ -25,6 +25,7 @@ type SelectionCommand<T extends NotArray> = CreateSupertypeOf<
 >;
 
 export function createSingleSelectionCommand<T extends NotArray>(
+  logger: NotificationLogger,
   f: (argument: T) => Promise<void>,
   itemName: string,
 ): SelectionCommand<T> {
@@ -32,7 +33,10 @@ export function createSingleSelectionCommand<T extends NotArray>(
     if (multiSelect === undefined || multiSelect.length === 1) {
       return f(singleItem);
     } else {
-      void showAndLogErrorMessage(`Please select a single ${itemName}.`);
+      void showAndLogErrorMessage(
+        logger,
+        `Please select a single ${itemName}.`,
+      );
       return;
     }
   };

extensions/ql-vscode/src/common/vscode/telemetry.ts

@@ -13,18 +13,19 @@ import {
   LOG_TELEMETRY,
   isIntegrationTestMode,
   isCanary,
-} from "./config";
+} from "../../config";
 import * as appInsights from "applicationinsights";
-import { extLogger } from "./common";
-import { UserCancellationException } from "./common/vscode/progress";
-import { showBinaryChoiceWithUrlDialog } from "./helpers";
-import { RedactableError } from "./pure/errors";
+import { extLogger } from "../logging/vscode";
+import { UserCancellationException } from "./progress";
+import { showBinaryChoiceWithUrlDialog } from "./dialog";
+import { RedactableError } from "../errors";
 import { SemVer } from "semver";
+import { AppTelemetry } from "../telemetry";
 
 // Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
 const key = "REPLACE-APP-INSIGHTS-KEY";
 
-export enum CommandCompletion {
+enum CommandCompletion {
   Success = "Success",
   Failed = "Failed",
   Cancelled = "Cancelled",
@@ -54,7 +55,10 @@ const baseDataPropertiesToRemove = [
 
 const NOT_SET_CLI_VERSION = "not-set";
 
-export class TelemetryListener extends ConfigListener {
+export class ExtensionTelemetryListener
+  extends ConfigListener
+  implements AppTelemetry
+{
   static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
 
   private reporter?: TelemetryReporter;
@@ -152,7 +156,7 @@ export class TelemetryListener extends ConfigListener {
     void this.reporter?.dispose();
   }
 
-  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void {
     if (!this.reporter) {
       return;
     }
@@ -174,7 +178,7 @@ export class TelemetryListener extends ConfigListener {
     );
   }
 
-  sendUIInteraction(name: string) {
+  sendUIInteraction(name: string): void {
     if (!this.reporter) {
       return;
     }
@@ -193,7 +197,7 @@ export class TelemetryListener extends ConfigListener {
   sendError(
     error: RedactableError,
     extraProperties?: { [key: string]: string },
-  ) {
+  ): void {
     if (!this.reporter) {
       return;
     }
@@ -272,16 +276,16 @@ export class TelemetryListener extends ConfigListener {
 /**
  * The global Telemetry instance
  */
-export let telemetryListener: TelemetryListener | undefined;
+export let telemetryListener: ExtensionTelemetryListener | undefined;
 
 export async function initializeTelemetry(
   extension: Extension<any>,
   ctx: ExtensionContext,
-): Promise<TelemetryListener> {
+): Promise<ExtensionTelemetryListener> {
   if (telemetryListener !== undefined) {
     throw new Error("Telemetry is already initialized");
   }
-  telemetryListener = new TelemetryListener(
+  telemetryListener = new ExtensionTelemetryListener(
     extension.id,
     extension.packageJSON.version,
     key,

extensions/ql-vscode/src/common/vscode/vscode-app.ts

@@ -1,14 +1,17 @@
 import * as vscode from "vscode";
 import { VSCodeCredentials } from "./authentication";
-import { Disposable } from "../../pure/disposable-object";
+import { Disposable } from "../disposable-object";
 import { App, AppMode, EnvironmentContext } from "../app";
 import { AppEventEmitter } from "../events";
-import { extLogger, Logger, queryServerLogger } from "../logging";
+import { NotificationLogger } from "../logging";
+import { extLogger, queryServerLogger } from "../logging/vscode";
 import { Memento } from "../memento";
 import { VSCodeAppEventEmitter } from "./events";
 import { AppCommandManager, QueryServerCommandManager } from "../commands";
 import { createVSCodeCommandManager } from "./commands";
 import { AppEnvironmentContext } from "./environment-context";
+import { AppTelemetry } from "../telemetry";
+import { telemetryListener } from "./telemetry";
 
 export class ExtensionApp implements App {
   public readonly credentials: VSCodeCredentials;
@@ -40,14 +43,6 @@ export class ExtensionApp implements App {
     return this.extensionContext.workspaceState;
   }
 
-  public get workspaceFolders(): readonly vscode.WorkspaceFolder[] | undefined {
-    return vscode.workspace.workspaceFolders;
-  }
-
-  public get onDidChangeWorkspaceFolders(): vscode.Event<vscode.WorkspaceFoldersChangeEvent> {
-    return vscode.workspace.onDidChangeWorkspaceFolders;
-  }
-
   public get subscriptions(): Disposable[] {
     return this.extensionContext.subscriptions;
   }
@@ -63,10 +58,14 @@ export class ExtensionApp implements App {
     }
   }
 
-  public get logger(): Logger {
+  public get logger(): NotificationLogger {
     return extLogger;
   }
 
+  public get telemetry(): AppTelemetry | undefined {
+    return telemetryListener;
+  }
+
   public createEventEmitter<T>(): AppEventEmitter<T> {
     return new VSCodeAppEventEmitter<T>();
   }

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -0,0 +1,101 @@
+import { ExtensionContext, Uri, Webview } from "vscode";
+import { randomBytes } from "crypto";
+import { EOL } from "os";
+
+export type WebviewView =
+  | "results"
+  | "compare"
+  | "variant-analysis"
+  | "data-flow-paths"
+  | "data-extensions-editor";
+
+export interface WebviewMessage {
+  t: string;
+}
+
+/**
+ * Returns HTML to populate the given webview.
+ * Uses a content security policy that only loads the given script.
+ */
+export function getHtmlForWebview(
+  ctx: ExtensionContext,
+  webview: Webview,
+  view: WebviewView,
+  {
+    allowInlineStyles,
+    allowWasmEval,
+  }: {
+    allowInlineStyles?: boolean;
+    allowWasmEval?: boolean;
+  } = {
+    allowInlineStyles: false,
+    allowWasmEval: false,
+  },
+): string {
+  const scriptUriOnDisk = Uri.file(ctx.asAbsolutePath("out/webview.js"));
+
+  const stylesheetUrisOnDisk = [
+    Uri.file(ctx.asAbsolutePath("out/webview.css")),
+  ];
+
+  // Convert the on-disk URIs into webview URIs.
+  const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
+  const stylesheetWebviewUris = stylesheetUrisOnDisk.map(
+    (stylesheetUriOnDisk) => webview.asWebviewUri(stylesheetUriOnDisk),
+  );
+
+  // Use a nonce in the content security policy to uniquely identify the above resources.
+  const nonce = getNonce();
+
+  const stylesheetsHtmlLines = allowInlineStyles
+    ? stylesheetWebviewUris.map((uri) => createStylesLinkWithoutNonce(uri))
+    : stylesheetWebviewUris.map((uri) => createStylesLinkWithNonce(nonce, uri));
+
+  const styleSrc = allowInlineStyles
+    ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
+    : `'nonce-${nonce}'`;
+
+  const fontSrc = webview.cspSource;
+
+  /*
+   * Content security policy:
+   * default-src: allow nothing by default.
+   * script-src:
+   *   - allow the given script, using the nonce.
+   *   - 'wasm-unsafe-eval: allow loading WebAssembly modules if necessary.
+   * style-src: allow only the given stylesheet, using the nonce.
+   * connect-src: only allow fetch calls to webview resource URIs
+   * (this is used to load BQRS result files).
+   */
+  return `
+<html>
+  <head>
+    <meta http-equiv="Content-Security-Policy"
+          content="default-src 'none'; script-src 'nonce-${nonce}'${
+            allowWasmEval ? " 'wasm-unsafe-eval'" : ""
+          }; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${
+            webview.cspSource
+          };">
+        ${stylesheetsHtmlLines.join(`    ${EOL}`)}
+  </head>
+  <body>
+    <div id=root data-view="${view}">
+    </div>
+      <script nonce="${nonce}" src="${scriptWebviewUri}">
+    </script>
+  </body>
+</html>`;
+}
+
+/** Gets a nonce string created with 128 bits of entropy. */
+function getNonce(): string {
+  return randomBytes(16).toString("base64");
+}
+
+function createStylesLinkWithNonce(nonce: string, uri: Uri): string {
+  return `<link nonce="${nonce}" rel="stylesheet" href="${uri}">`;
+}
+
+function createStylesLinkWithoutNonce(uri: Uri): string {
+  return `<link rel="stylesheet" href="${uri}">`;
+}

extensions/ql-vscode/src/common/vscode/workspace-folders.ts

@@ -0,0 +1,64 @@
+import { dirname, join } from "path";
+import { workspace, WorkspaceFolder } from "vscode";
+
+/** Returns true if the specified workspace folder is on the file system. */
+export function isWorkspaceFolderOnDisk(
+  workspaceFolder: WorkspaceFolder,
+): boolean {
+  return workspaceFolder.uri.scheme === "file";
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFoldersObjects() {
+  const workspaceFolders = workspace.workspaceFolders ?? [];
+  return workspaceFolders.filter(isWorkspaceFolderOnDisk);
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFolders() {
+  return getOnDiskWorkspaceFoldersObjects().map((folder) => folder.uri.fsPath);
+}
+
+/** Check if folder is already present in workspace */
+export function isFolderAlreadyInWorkspace(folderName: string) {
+  const workspaceFolders = workspace.workspaceFolders || [];
+
+  return !!workspaceFolders.find(
+    (workspaceFolder) => workspaceFolder.name === folderName,
+  );
+}
+
+/**
+ * Returns the path of the first folder in the workspace.
+ * This is used to decide where to create skeleton QL packs.
+ *
+ * If the first folder is a QL pack, then the parent folder is returned.
+ * This is because the vscode-codeql-starter repo contains a ql pack in
+ * the first folder.
+ *
+ * This is a temporary workaround until we can retire the
+ * vscode-codeql-starter repo.
+ */
+export function getFirstWorkspaceFolder() {
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+
+  if (!workspaceFolders || workspaceFolders.length === 0) {
+    throw new Error("No workspace folders found");
+  }
+
+  const firstFolderFsPath = workspaceFolders[0];
+
+  // For the vscode-codeql-starter repo, the first folder will be a ql pack
+  // so we need to get the parent folder
+  if (
+    firstFolderFsPath.includes(
+      join("vscode-codeql-starter", "codeql-custom-queries"),
+    )
+  ) {
+    // return the parent folder
+    return dirname(firstFolderFsPath);
+  } else {
+    // if the first folder is not a ql pack, then we are in a normal workspace
+    return firstFolderFsPath;
+  }
+}

extensions/ql-vscode/src/common/word.ts

@@ -7,8 +7,9 @@ export function pluralize(
   numItems: number | undefined,
   singular: string,
   plural: string,
+  numberFormatter: (value: number) => string = (value) => value.toString(),
 ): string {
   return numItems !== undefined
-    ? `${numItems} ${numItems === 1 ? singular : plural}`
+    ? `${numberFormatter(numItems)} ${numItems === 1 ? singular : plural}`
     : "";
 }

extensions/ql-vscode/src/common/zlib.ts

@@ -0,0 +1,12 @@
+import { promisify } from "util";
+import { gzip, gunzip } from "zlib";
+
+/**
+ * Promisified version of zlib.gzip
+ */
+export const gzipEncode = promisify(gzip);
+
+/**
+ * Promisified version of zlib.gunzip
+ */
+export const gzipDecode = promisify(gunzip);

extensions/ql-vscode/src/compare/compare-view.ts

@@ -4,27 +4,27 @@ import {
   FromCompareViewMessage,
   ToCompareViewMessage,
   QueryCompareResult,
-} from "../pure/interface-types";
-import { Logger } from "../common";
+} from "../common/interface-types";
+import { Logger, showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseManager } from "../databases/local-databases";
-import { jumpToLocation } from "../interface-utils";
+import { jumpToLocation } from "../databases/local-databases/locations";
 import {
   transformBqrsResultSet,
   RawResultSet,
   BQRSInfo,
-} from "../pure/bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import resultsDiff from "./resultsDiff";
 import { CompletedLocalQueryInfo } from "../query-results";
-import { assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { assertNever, getErrorMessage } from "../common/helpers-pure";
 import { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
 import {
   AbstractWebview,
   WebviewPanelConfig,
 } from "../common/vscode/abstract-webview";
-import { telemetryListener } from "../telemetry";
-import { redactableError } from "../pure/errors";
-import { showAndLogExceptionWithTelemetry } from "../helpers";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
@@ -130,7 +130,12 @@ export class CompareView extends AbstractWebview<
         break;
 
       case "viewSourceFile":
-        await jumpToLocation(msg, this.databaseManager, this.logger);
+        await jumpToLocation(
+          msg.databaseUri,
+          msg.loc,
+          this.databaseManager,
+          this.logger,
+        );
         break;
 
       case "openQuery":
@@ -146,6 +151,8 @@ export class CompareView extends AbstractWebview<
 
       case "unhandledError":
         void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
           redactableError(
             msg.error,
           )`Unhandled error in result comparison view: ${msg.error.message}`,

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -1,5 +1,5 @@
-import { RawResultSet } from "../pure/bqrs-cli-types";
-import { QueryCompareResult } from "../pure/interface-types";
+import { RawResultSet } from "../common/bqrs-cli-types";
+import { QueryCompareResult } from "../common/interface-types";
 
 /**
  * Compare the rows of two queries. Use deep equality to determine if

extensions/ql-vscode/src/config.ts

@@ -1,19 +1,20 @@
-import { DisposableObject } from "./pure/disposable-object";
+import { DisposableObject } from "./common/disposable-object";
 import {
   workspace,
   Event,
   EventEmitter,
   ConfigurationChangeEvent,
   ConfigurationTarget,
+  ConfigurationScope,
 } from "vscode";
 import { DistributionManager } from "./codeql-cli/distribution";
-import { extLogger } from "./common";
-import { ONE_DAY_IN_MS } from "./pure/time";
+import { extLogger } from "./common/logging/vscode";
+import { ONE_DAY_IN_MS } from "./common/time";
 import {
   FilterKey,
   SortKey,
   defaultFilterSortState,
-} from "./pure/variant-analysis-filter-sort";
+} from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -44,12 +45,12 @@ export class Setting {
     }
   }
 
-  getValue<T>(): T {
+  getValue<T>(scope?: ConfigurationScope | null): T {
     if (this.parent === undefined) {
       throw new Error("Cannot get the value of a root setting.");
     }
     return workspace
-      .getConfiguration(this.parent.qualifiedName)
+      .getConfiguration(this.parent.qualifiedName, scope)
       .get<T>(this.name)!;
   }
 
@@ -63,11 +64,11 @@ export class Setting {
   }
 }
 
-export interface InspectionResult<T> {
-  globalValue?: T;
-  workspaceValue?: T;
-  workspaceFolderValue?: T;
-}
+const VSCODE_DEBUG_SETTING = new Setting("debug", undefined);
+export const VSCODE_SAVE_BEFORE_START_SETTING = new Setting(
+  "saveBeforeStart",
+  VSCODE_DEBUG_SETTING,
+);
 
 const ROOT_SETTING = new Setting("codeQL");
 
@@ -160,10 +161,6 @@ export const NUMBER_OF_TEST_THREADS_SETTING = new Setting(
   RUNNING_TESTS_SETTING,
 );
 export const MAX_QUERIES = new Setting("maxQueries", RUNNING_QUERIES_SETTING);
-export const AUTOSAVE_SETTING = new Setting(
-  "autoSave",
-  RUNNING_QUERIES_SETTING,
-);
 export const PAGE_SIZE = new Setting("pageSize", RESULTS_DISPLAY_SETTING);
 const CUSTOM_LOG_DIRECTORY_SETTING = new Setting(
   "customLogDirectory",
@@ -589,10 +586,6 @@ export function isIntegrationTestMode() {
   return process.env.INTEGRATION_TEST_MODE === "true";
 }
 
-export function isVariantAnalysisLiveResultsEnabled(): boolean {
-  return true;
-}
-
 // Settings for mocking the GitHub API.
 const MOCK_GH_API_SERVER = new Setting("mockGitHubApiServer", ROOT_SETTING);
 
@@ -656,10 +649,7 @@ export function isCodespacesTemplate() {
 
 const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING);
 
-export const ALLOW_HTTP_SETTING = new Setting(
-  "allowHttp",
-  DATABASE_DOWNLOAD_SETTING,
-);
+const ALLOW_HTTP_SETTING = new Setting("allowHttp", DATABASE_DOWNLOAD_SETTING);
 
 export function allowHttp(): boolean {
   return ALLOW_HTTP_SETTING.getValue<boolean>() || false;
@@ -692,7 +682,7 @@ const AUTOGENERATE_QL_PACKS = new Setting(
 );
 
 const AutogenerateQLPacksValues = ["ask", "never"] as const;
-type AutogenerateQLPacks = typeof AutogenerateQLPacksValues[number];
+type AutogenerateQLPacks = (typeof AutogenerateQLPacksValues)[number];
 
 export function getAutogenerateQlPacks(): AutogenerateQLPacks {
   const value = AUTOGENERATE_QL_PACKS.getValue<AutogenerateQLPacks>();
@@ -714,7 +704,35 @@ export function showQueriesPanel(): boolean {
 
 const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
+const FRAMEWORK_MODE = new Setting("frameworkMode", DATA_EXTENSIONS);
+const DISABLE_AUTO_NAME_EXTENSION_PACK = new Setting(
+  "disableAutoNameExtensionPack",
+  DATA_EXTENSIONS,
+);
+const EXTENSIONS_DIRECTORY = new Setting(
+  "extensionsDirectory",
+  DATA_EXTENSIONS,
+);
+const MODEL_DETAILS_VIEW = new Setting("modelDetailsView", DATA_EXTENSIONS);
+
+export function showModelDetailsView(): boolean {
+  return !!MODEL_DETAILS_VIEW.getValue<boolean>();
+}
 
 export function showLlmGeneration(): boolean {
   return !!LLM_GENERATION.getValue<boolean>();
 }
+
+export function enableFrameworkMode(): boolean {
+  return !!FRAMEWORK_MODE.getValue<boolean>();
+}
+
+export function disableAutoNameExtensionPack(): boolean {
+  return !!DISABLE_AUTO_NAME_EXTENSION_PACK.getValue<boolean>();
+}
+
+export function getExtensionsDirectory(languageId: string): string | undefined {
+  return EXTENSIONS_DIRECTORY.getValue<string>({
+    languageId,
+  });
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts

@@ -1,40 +1,20 @@
 import { Credentials } from "../common/authentication";
 import { OctokitResponse } from "@octokit/types";
 
-export enum ClassificationType {
-  Unknown = "CLASSIFICATION_TYPE_UNKNOWN",
-  Neutral = "CLASSIFICATION_TYPE_NEUTRAL",
-  Source = "CLASSIFICATION_TYPE_SOURCE",
-  Sink = "CLASSIFICATION_TYPE_SINK",
-  Summary = "CLASSIFICATION_TYPE_SUMMARY",
-}
-
-export interface Classification {
-  type: ClassificationType;
-  kind: string;
-  explanation: string;
-}
-
-export interface Method {
-  package: string;
-  type: string;
-  name: string;
-  signature: string;
-  usages: string[];
-  classification?: Classification;
-  input?: string;
-  output?: string;
+export enum AutomodelMode {
+  Unspecified = "AUTOMODEL_MODE_UNSPECIFIED",
+  Framework = "AUTOMODEL_MODE_FRAMEWORK",
+  Application = "AUTOMODEL_MODE_APPLICATION",
 }
 
 export interface ModelRequest {
-  language: string;
-  candidates: Method[];
-  samples: Method[];
+  mode: AutomodelMode;
+  // Base64-encoded GZIP-compressed SARIF log
+  candidates: string;
 }
 
 export interface ModelResponse {
-  language: string;
-  predicted: Method[];
+  models: string;
 }
 
 export async function autoModel(

extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts

@@ -0,0 +1,286 @@
+import { CodeQLCliServer, SourceInfo } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { ProgressCallback } from "../common/vscode/progress";
+import * as Sarif from "sarif";
+import { qlpackOfDatabase, resolveQueries } from "../local-queries";
+import { extLogger } from "../common/logging/vscode";
+import { Mode } from "./shared/mode";
+import { QlPacksForLanguage } from "../databases/qlpack";
+import { createLockFileForStandardQuery } from "../local-queries/standard-queries";
+import { CancellationToken, CancellationTokenSource } from "vscode";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { QueryResultType } from "../query-server/new-messages";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
+import { interpretResultsSarif } from "../query-results";
+import { join } from "path";
+import { assertNever } from "../common/helpers-pure";
+import { dir } from "tmp-promise";
+import { writeFile, outputFile } from "fs-extra";
+import { dump as dumpYaml } from "js-yaml";
+import { MethodSignature } from "./external-api-usage";
+
+type AutoModelQueryOptions = {
+  queryTag: string;
+  mode: Mode;
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  qlpack: QlPacksForLanguage;
+  sourceInfo: SourceInfo | undefined;
+  additionalPacks: string[];
+  extensionPacks: string[];
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+  token: CancellationToken;
+};
+
+function modeTag(mode: Mode): string {
+  switch (mode) {
+    case Mode.Application:
+      return "application-mode";
+    case Mode.Framework:
+      return "framework-mode";
+    default:
+      assertNever(mode);
+  }
+}
+
+async function runAutoModelQuery({
+  queryTag,
+  mode,
+  cliServer,
+  queryRunner,
+  databaseItem,
+  qlpack,
+  sourceInfo,
+  additionalPacks,
+  extensionPacks,
+  queryStorageDir,
+  progress,
+  token,
+}: AutoModelQueryOptions): Promise<Sarif.Log | undefined> {
+  // First, resolve the query that we want to run.
+  // All queries are tagged like this:
+  // internal extract automodel <mode> <queryTag>
+  // Example: internal extract automodel framework-mode candidates
+  const queries = await resolveQueries(
+    cliServer,
+    qlpack,
+    `Extract automodel ${queryTag}`,
+    {
+      kind: "problem",
+      "tags contain all": ["automodel", modeTag(mode), ...queryTag.split(" ")],
+    },
+  );
+  if (queries.length > 1) {
+    throw new Error(
+      `Found multiple auto model queries for ${mode} ${queryTag}. Can't continue`,
+    );
+  }
+  if (queries.length === 0) {
+    throw new Error(
+      `Did not found any auto model queries for ${mode} ${queryTag}. Can't continue`,
+    );
+  }
+
+  const queryPath = queries[0];
+  const { cleanup: cleanupLockFile } = await createLockFileForStandardQuery(
+    cliServer,
+    queryPath,
+  );
+
+  // Get metadata for the query. This is required to interpret the results. We already know the kind is problem
+  // (because of the constraint in resolveQueries), so we don't need any more checks on the metadata.
+  const metadata = await cliServer.resolveMetadata(queryPath);
+
+  const queryRun = queryRunner.createQueryRun(
+    databaseItem.databaseUri.fsPath,
+    {
+      queryPath,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
+    false,
+    additionalPacks,
+    extensionPacks,
+    queryStorageDir,
+    undefined,
+    undefined,
+  );
+
+  const completedQuery = await queryRun.evaluate(
+    progress,
+    token,
+    new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
+  );
+
+  await cleanupLockFile?.();
+
+  if (completedQuery.resultType !== QueryResultType.SUCCESS) {
+    void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
+      redactableError`Auto-model query ${queryTag} failed: ${
+        completedQuery.message ?? "No message"
+      }`,
+    );
+    return;
+  }
+
+  const interpretedResultsPath = join(
+    queryStorageDir,
+    `interpreted-results-${queryTag.replaceAll(" ", "-")}-${queryRun.id}.sarif`,
+  );
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars -- We only need the actual SARIF data, not the extra fields added by SarifInterpretationData
+  const { t, sortState, ...sarif } = await interpretResultsSarif(
+    cliServer,
+    metadata,
+    {
+      resultsPath: completedQuery.outputDir.bqrsPath,
+      interpretedResultsPath,
+    },
+    sourceInfo,
+    ["--sarif-add-snippets"],
+  );
+
+  return sarif;
+}
+
+type AutoModelQueriesOptions = {
+  mode: Mode;
+  candidateMethods: MethodSignature[];
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+  cancellationTokenSource: CancellationTokenSource;
+};
+
+export type AutoModelQueriesResult = {
+  candidates: Sarif.Log;
+};
+
+export async function runAutoModelQueries({
+  mode,
+  candidateMethods,
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+  cancellationTokenSource,
+}: AutoModelQueriesOptions): Promise<AutoModelQueriesResult | undefined> {
+  const qlpack = await qlpackOfDatabase(cliServer, databaseItem);
+
+  // CodeQL needs to have access to the database to be able to retrieve the
+  // snippets from it. The source location prefix is used to determine the
+  // base path of the database.
+  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
+    cliServer,
+  );
+  const sourceArchiveUri = databaseItem.sourceArchive;
+  const sourceInfo =
+    sourceArchiveUri === undefined
+      ? undefined
+      : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix,
+        };
+
+  // Generate a pack containing the candidate filters
+  const filterPackDir = await generateCandidateFilterPack(
+    databaseItem.language,
+    candidateMethods,
+  );
+
+  const additionalPacks = [...getOnDiskWorkspaceFolders(), filterPackDir];
+  const extensionPacks = Object.keys(
+    await cliServer.resolveQlpacks(additionalPacks, true),
+  );
+
+  const candidates = await runAutoModelQuery({
+    mode,
+    queryTag: "candidates",
+    cliServer,
+    queryRunner,
+    databaseItem,
+    qlpack,
+    sourceInfo,
+    additionalPacks,
+    extensionPacks,
+    queryStorageDir,
+    progress,
+    token: cancellationTokenSource.token,
+  });
+
+  if (!candidates) {
+    return undefined;
+  }
+
+  return {
+    candidates,
+  };
+}
+
+/**
+ * generateCandidateFilterPack will create a temporary extension pack.
+ * This pack will contain a filter that will restrict the automodel queries
+ * to the specified candidate methods only.
+ * This is done using the `extensible` predicate "automodelCandidateFilter".
+ * @param language
+ * @param candidateMethods
+ * @returns
+ */
+export async function generateCandidateFilterPack(
+  language: string,
+  candidateMethods: MethodSignature[],
+): Promise<string> {
+  // Pack resides in a temporary directory, to not pollute the workspace.
+  const packDir = (await dir({ unsafeCleanup: true })).path;
+
+  const syntheticConfigPack = {
+    name: "codeql/automodel-filter",
+    version: "0.0.0",
+    library: true,
+    extensionTargets: {
+      [`codeql/${language}-queries`]: "*",
+    },
+    dataExtensions: ["filter.yml"],
+  };
+
+  const qlpackFile = join(packDir, "codeql-pack.yml");
+  await outputFile(qlpackFile, dumpYaml(syntheticConfigPack), "utf8");
+
+  // The predicate has the following defintion:
+  // extensible predicate automodelCandidateFilter(string package, string type, string name, string signature)
+  const dataRows = candidateMethods.map((method) => [
+    method.packageName,
+    method.typeName,
+    method.methodName,
+    method.methodParameters,
+  ]);
+
+  const filter = {
+    extensions: [
+      {
+        addsTo: {
+          pack: `codeql/${language}-queries`,
+          extensible: "automodelCandidateFilter",
+        },
+        data: dataRows,
+      },
+    ],
+  };
+
+  const filterFile = join(packDir, "filter.yml");
+  await writeFile(filterFile, dumpYaml(filter), "utf8");
+
+  return packDir;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

@@ -1,136 +0,0 @@
-import { CancellationTokenSource } from "vscode";
-import { join } from "path";
-import { runQuery } from "./external-api-usage-query";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import { QueryRunner } from "../query-server";
-import { DatabaseItem } from "../databases/local-databases";
-import { interpretResultsSarif } from "../query-results";
-import { ProgressCallback } from "../common/vscode/progress";
-
-type Options = {
-  cliServer: CodeQLCliServer;
-  queryRunner: QueryRunner;
-  databaseItem: DatabaseItem;
-  queryStorageDir: string;
-
-  progress: ProgressCallback;
-};
-
-export type UsageSnippetsBySignature = Record<string, string[]>;
-
-export async function getAutoModelUsages({
-  cliServer,
-  queryRunner,
-  databaseItem,
-  queryStorageDir,
-  progress,
-}: Options): Promise<UsageSnippetsBySignature> {
-  const maxStep = 1500;
-
-  const cancellationTokenSource = new CancellationTokenSource();
-
-  // This will re-run the query that was already run when opening the data extensions editor. This
-  // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
-  // need to interpret the results.
-  const queryResult = await runQuery({
-    cliServer,
-    queryRunner,
-    queryStorageDir,
-    databaseItem,
-    progress: (update) =>
-      progress({
-        maxStep,
-        step: update.step,
-        message: update.message,
-      }),
-    token: cancellationTokenSource.token,
-  });
-  if (!queryResult) {
-    throw new Error("Query failed");
-  }
-
-  progress({
-    maxStep,
-    step: 1100,
-    message: "Retrieving source location prefix",
-  });
-
-  // CodeQL needs to have access to the database to be able to retrieve the
-  // snippets from it. The source location prefix is used to determine the
-  // base path of the database.
-  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
-    cliServer,
-  );
-  const sourceArchiveUri = databaseItem.sourceArchive;
-  const sourceInfo =
-    sourceArchiveUri === undefined
-      ? undefined
-      : {
-          sourceArchive: sourceArchiveUri.fsPath,
-          sourceLocationPrefix,
-        };
-
-  progress({
-    maxStep,
-    step: 1200,
-    message: "Interpreting results",
-  });
-
-  // Convert the results to SARIF so that Codeql will retrieve the snippets
-  // from the datababe. This means we don't need to do that in the extension
-  // and everything is handled by the CodeQL CLI.
-  const sarif = await interpretResultsSarif(
-    cliServer,
-    {
-      // To interpret the results we need to provide metadata about the query. We could do this using
-      // `resolveMetadata` but that would be an extra call to the CodeQL CLI server and would require
-      // us to know the path to the query on the filesystem. Since we know what the metadata should
-      // look like and the only metadata that the CodeQL CLI requires is an ID and the kind, we can
-      // simply use constants here.
-      kind: "problem",
-      id: "usage",
-    },
-    {
-      resultsPath: queryResult.outputDir.bqrsPath,
-      interpretedResultsPath: join(
-        queryStorageDir,
-        "interpreted-results.sarif",
-      ),
-    },
-    sourceInfo,
-    ["--sarif-add-snippets"],
-  );
-
-  progress({
-    maxStep,
-    step: 1400,
-    message: "Parsing results",
-  });
-
-  const snippets: UsageSnippetsBySignature = {};
-
-  const results = sarif.runs[0]?.results;
-  if (!results) {
-    throw new Error("No results");
-  }
-
-  // This will group the snippets by the method signature.
-  for (const result of results) {
-    const signature = result.message.text;
-
-    const snippet =
-      result.locations?.[0]?.physicalLocation?.contextRegion?.snippet?.text;
-
-    if (!signature || !snippet) {
-      continue;
-    }
-
-    if (!(signature in snippets)) {
-      snippets[signature] = [];
-    }
-
-    snippets[signature].push(snippet);
-  }
-
-  return snippets;
-}

extensions/ql-vscode/src/data-extensions-editor/auto-model.ts

@@ -1,221 +1,89 @@
-import { ExternalApiUsage } from "./external-api-usage";
-import { ModeledMethod, ModeledMethodType } from "./modeled-method";
-import {
-  Classification,
-  ClassificationType,
-  Method,
-  ModelRequest,
-} from "./auto-model-api";
-import type { UsageSnippetsBySignature } from "./auto-model-usages-query";
+import { AutomodelMode, ModelRequest } from "./auto-model-api";
+import { Mode } from "./shared/mode";
+import { AutoModelQueriesResult } from "./auto-model-codeml-queries";
+import { assertNever } from "../common/helpers-pure";
+import * as Sarif from "sarif";
+import { gzipEncode } from "../common/zlib";
+import { ExternalApiUsage, MethodSignature } from "./external-api-usage";
+import { ModeledMethod } from "./modeled-method";
+import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
 
-export function createAutoModelRequest(
-  language: string,
+/**
+ * Return the candidates that the model should be run on. This includes limiting the number of
+ * candidates to the candidate limit and filtering out anything that is already modeled and respecting
+ * the order in the UI.
+ * @param mode Whether it is application or framework mode.
+ * @param externalApiUsages all external API usages.
+ * @param modeledMethods the currently modeled methods.
+ * @returns list of modeled methods that are candidates for modeling.
+ */
+export function getCandidates(
+  mode: Mode,
   externalApiUsages: ExternalApiUsage[],
   modeledMethods: Record<string, ModeledMethod>,
-  usages: UsageSnippetsBySignature,
-): ModelRequest {
-  const request: ModelRequest = {
-    language,
-    samples: [],
-    candidates: [],
-  };
+): MethodSignature[] {
+  // Sort the same way as the UI so we send the first ones listed in the UI first
+  const grouped = groupMethods(externalApiUsages, mode);
+  const sortedGroupNames = sortGroupNames(grouped);
+  const sortedExternalApiUsages = sortedGroupNames.flatMap((name) =>
+    sortMethods(grouped[name]),
+  );
 
-  // Sort by number of usages so we always send the most used methods first
-  externalApiUsages = [...externalApiUsages];
-  externalApiUsages.sort((a, b) => b.usages.length - a.usages.length);
+  const candidates: MethodSignature[] = [];
 
-  for (const externalApiUsage of externalApiUsages) {
+  for (const externalApiUsage of sortedExternalApiUsages) {
     const modeledMethod: ModeledMethod = modeledMethods[
       externalApiUsage.signature
     ] ?? {
       type: "none",
     };
 
-    const usagesForMethod =
-      usages[externalApiUsage.signature] ??
-      externalApiUsage.usages.map((usage) => usage.label);
-
-    const numberOfArguments =
-      externalApiUsage.methodParameters === "()"
-        ? 0
-        : externalApiUsage.methodParameters.split(",").length;
-
-    for (
-      let argumentIndex = 0;
-      argumentIndex < numberOfArguments;
-      argumentIndex++
-    ) {
-      const method: Method = {
-        package: externalApiUsage.packageName,
-        type: externalApiUsage.typeName,
-        name: externalApiUsage.methodName,
-        signature: externalApiUsage.methodParameters,
-        classification:
-          modeledMethod.type === "none"
-            ? undefined
-            : toMethodClassification(modeledMethod),
-        usages: usagesForMethod.slice(0, 10),
-        input: `Argument[${argumentIndex}]`,
-      };
-
-      if (modeledMethod.type === "none") {
-        request.candidates.push(method);
-      } else {
-        request.samples.push(method);
-      }
-    }
-  }
-
-  request.candidates = request.candidates.slice(0, 20);
-  request.samples = request.samples.slice(0, 100);
-
-  return request;
-}
-
-/**
- * For now, we have a simplified model that only models methods as sinks. It does not model methods as neutral,
- * so we aren't actually able to correctly determine that a method is neutral; it could still be a source or summary.
- * However, to keep this method simple and give output to the user, we will model any method for which none of its
- * arguments are modeled as sinks as neutral.
- *
- * If there are multiple arguments which are modeled as sinks, we will only model the first one.
- */
-export function parsePredictedClassifications(
-  predicted: Method[],
-): Record<string, ModeledMethod> {
-  const predictedBySignature: Record<string, Method[]> = {};
-  for (const method of predicted) {
-    const signature = toFullMethodSignature(method);
-
-    if (!(signature in predictedBySignature)) {
-      predictedBySignature[signature] = [];
-    }
-
-    predictedBySignature[signature].push(method);
-  }
-
-  const modeledMethods: Record<string, ModeledMethod> = {};
-
-  for (const signature in predictedBySignature) {
-    const predictedMethods = predictedBySignature[signature];
-
-    const sinks = predictedMethods.filter(
-      (method) => method.classification?.type === ClassificationType.Sink,
-    );
-    if (sinks.length === 0) {
-      // For now, model any method for which none of its arguments are modeled as sinks as neutral
-      modeledMethods[signature] = {
-        type: "neutral",
-        kind: "summary",
-        input: "",
-        output: "",
-        provenance: "ai-generated",
-      };
+    // Anything that is modeled is not a candidate
+    if (modeledMethod.type !== "none") {
       continue;
     }
 
-    // Order the sinks by the input alphabetically. This will ensure that the first argument is always
-    // first in the list of sinks, the second argument is always second, etc.
-    // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first
-    sinks.sort((a, b) => compareInputOutput(a.input ?? "", b.input ?? ""));
+    // A method that is supported is modeled outside of the model file, so it is not a candidate.
+    if (externalApiUsage.supported) {
+      continue;
+    }
 
-    const sink = sinks[0];
-
-    modeledMethods[signature] = {
-      type: "sink",
-      kind: sink.classification?.kind ?? "",
-      input: sink.input ?? "",
-      output: sink.output ?? "",
-      provenance: "ai-generated",
-    };
+    // The rest are candidates
+    candidates.push(externalApiUsage);
   }
-
-  return modeledMethods;
+  return candidates;
 }
 
-function toMethodClassificationType(
-  type: ModeledMethodType,
-): ClassificationType {
-  switch (type) {
-    case "source":
-      return ClassificationType.Source;
-    case "sink":
-      return ClassificationType.Sink;
-    case "summary":
-      return ClassificationType.Summary;
-    case "neutral":
-      return ClassificationType.Neutral;
-    default:
-      return ClassificationType.Unknown;
-  }
-}
-
-function toMethodClassification(modeledMethod: ModeledMethod): Classification {
-  return {
-    type: toMethodClassificationType(modeledMethod.type),
-    kind: modeledMethod.kind,
-    explanation: "",
-  };
-}
-
-function toFullMethodSignature(method: Method): string {
-  return `${method.package}.${method.type}#${method.name}${method.signature}`;
-}
-
-const argumentRegex = /^Argument\[(\d+)]$/;
-
-// Argument[this] is before ReturnValue
-const nonNumericArgumentOrder = ["Argument[this]", "ReturnValue"];
-
 /**
- * Compare two inputs or outputs matching `Argument[<number>]`, `Argument[this]`, or `ReturnValue`.
- * If they are the same, return 0. If a is less than b, returns a negative number.
- * If a is greater than b, returns a positive number.
+ * Encode a SARIF log to the format expected by the server: JSON, GZIP-compressed, base64-encoded
+ * @param log SARIF log to encode
+ * @returns base64-encoded GZIP-compressed SARIF log
  */
-export function compareInputOutput(a: string, b: string): number {
-  if (a === b) {
-    return 0;
-  }
-
-  const aMatch = a.match(argumentRegex);
-  const bMatch = b.match(argumentRegex);
-
-  // Numeric arguments are always first
-  if (aMatch && !bMatch) {
-    return -1;
-  }
-  if (!aMatch && bMatch) {
-    return 1;
-  }
-
-  // Neither is an argument
-  if (!aMatch && !bMatch) {
-    const aIndex = nonNumericArgumentOrder.indexOf(a);
-    const bIndex = nonNumericArgumentOrder.indexOf(b);
-
-    // If either one is unknown, it is sorted last
-    if (aIndex === -1 && bIndex === -1) {
-      // Use en-US because these are well-known strings that are not localized
-      return a.localeCompare(b, "en-US");
-    }
-    if (aIndex === -1) {
-      return 1;
-    }
-    if (bIndex === -1) {
-      return -1;
-    }
-
-    return aIndex - bIndex;
-  }
-
-  // This case shouldn't happen, but makes TypeScript happy
-  if (!aMatch || !bMatch) {
-    return 0;
-  }
-
-  // Both are arguments
-  const aIndex = parseInt(aMatch[1]);
-  const bIndex = parseInt(bMatch[1]);
-
-  return aIndex - bIndex;
+export async function encodeSarif(log: Sarif.Log): Promise<string> {
+  const json = JSON.stringify(log);
+  const buffer = Buffer.from(json, "utf-8");
+  const compressed = await gzipEncode(buffer);
+  return compressed.toString("base64");
+}
+
+export async function createAutoModelRequest(
+  mode: Mode,
+  result: AutoModelQueriesResult,
+): Promise<ModelRequest> {
+  let requestMode: AutomodelMode;
+  switch (mode) {
+    case Mode.Application:
+      requestMode = AutomodelMode.Application;
+      break;
+    case Mode.Framework:
+      requestMode = AutomodelMode.Framework;
+      break;
+    default:
+      assertNever(mode);
+  }
+
+  return {
+    mode: requestMode,
+    candidates: await encodeSarif(result.candidates),
+  };
 }

extensions/ql-vscode/src/data-extensions-editor/auto-modeler.ts

@@ -0,0 +1,246 @@
+import { ExternalApiUsage, MethodSignature } from "./external-api-usage";
+import { ModeledMethod } from "./modeled-method";
+import { extLogger } from "../common/logging/vscode";
+import { load as loadYaml } from "js-yaml";
+import { ProgressCallback, withProgress } from "../common/vscode/progress";
+import { createAutoModelRequest, getCandidates } from "./auto-model";
+import { runAutoModelQueries } from "./auto-model-codeml-queries";
+import { loadDataExtensionYaml } from "./yaml";
+import { ModelRequest, ModelResponse, autoModel } from "./auto-model-api";
+import { RequestError } from "@octokit/request-error";
+import { showAndLogExceptionWithTelemetry } from "../common/logging";
+import { redactableError } from "../common/errors";
+import { App } from "../common/app";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { Mode } from "./shared/mode";
+import { CancellationTokenSource } from "vscode";
+
+// Limit the number of candidates we send to the model in each request
+// to avoid long requests.
+// Note that the model may return fewer than this number of candidates.
+const candidateBatchSize = 20;
+
+/**
+ * The auto-modeler holds state around auto-modeling jobs and allows
+ * starting and stopping them.
+ */
+export class AutoModeler {
+  // Keep track of auto-modeling jobs that are in progress
+  // so that we can stop them.
+  private readonly jobs: Map<string, CancellationTokenSource>;
+
+  constructor(
+    private readonly app: App,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly queryRunner: QueryRunner,
+    private readonly queryStorageDir: string,
+    private readonly databaseItem: DatabaseItem,
+    private readonly setInProgressMethods: (
+      packageName: string,
+      inProgressMethods: string[],
+    ) => Promise<void>,
+    private readonly addModeledMethods: (
+      modeledMethods: Record<string, ModeledMethod>,
+    ) => Promise<void>,
+  ) {
+    this.jobs = new Map<string, CancellationTokenSource>();
+  }
+
+  /**
+   * Models the given package's external API usages, except
+   * the ones that are already modeled.
+   * @param packageName The name of the package to model.
+   * @param externalApiUsages The external API usages.
+   * @param modeledMethods The currently modeled methods.
+   * @param mode The mode we are modeling in.
+   */
+  public async startModeling(
+    packageName: string,
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+    mode: Mode,
+  ): Promise<void> {
+    if (this.jobs.has(packageName)) {
+      return;
+    }
+
+    const cancellationTokenSource = new CancellationTokenSource();
+    this.jobs.set(packageName, cancellationTokenSource);
+
+    try {
+      await this.modelPackage(
+        packageName,
+        externalApiUsages,
+        modeledMethods,
+        mode,
+        cancellationTokenSource,
+      );
+    } finally {
+      this.jobs.delete(packageName);
+    }
+  }
+
+  /**
+   * Stops modeling the given package.
+   * @param packageName The name of the package to stop modeling.
+   */
+  public async stopModeling(packageName: string): Promise<void> {
+    void extLogger.log(`Stopping modeling for package ${packageName}`);
+    const cancellationTokenSource = this.jobs.get(packageName);
+    if (cancellationTokenSource) {
+      cancellationTokenSource.cancel();
+    }
+  }
+
+  /**
+   * Stops all in-progress modeling jobs.
+   */
+  public async stopAllModeling(): Promise<void> {
+    for (const cancellationTokenSource of this.jobs.values()) {
+      cancellationTokenSource.cancel();
+    }
+  }
+
+  private async modelPackage(
+    packageName: string,
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+    mode: Mode,
+    cancellationTokenSource: CancellationTokenSource,
+  ): Promise<void> {
+    void extLogger.log(`Modeling package ${packageName}`);
+    await withProgress(async (progress) => {
+      // Fetch the candidates to send to the model
+      const allCandidateMethods = getCandidates(
+        mode,
+        externalApiUsages,
+        modeledMethods,
+      );
+
+      // If there are no candidates, there is nothing to model and we just return
+      if (allCandidateMethods.length === 0) {
+        void extLogger.log("No candidates to model. Stopping.");
+        return;
+      }
+
+      // Find number of slices to make
+      const batchNumber = Math.ceil(
+        allCandidateMethods.length / candidateBatchSize,
+      );
+      try {
+        for (let i = 0; i < batchNumber; i++) {
+          // Check if we should stop
+          if (cancellationTokenSource.token.isCancellationRequested) {
+            break;
+          }
+
+          const start = i * candidateBatchSize;
+          const end = start + candidateBatchSize;
+          const candidatesToProcess = allCandidateMethods.slice(start, end);
+
+          // Let the UI know which candidates we are modeling
+          await this.setInProgressMethods(
+            packageName,
+            candidatesToProcess.map((c) => c.signature),
+          );
+
+          // Kick off the process to model the slice of candidates
+          await this.modelCandidates(
+            candidatesToProcess,
+            mode,
+            progress,
+            cancellationTokenSource,
+          );
+        }
+      } finally {
+        // Clear out in progress methods
+        await this.setInProgressMethods(packageName, []);
+      }
+    });
+  }
+
+  private async modelCandidates(
+    candidateMethods: MethodSignature[],
+    mode: Mode,
+    progress: ProgressCallback,
+    cancellationTokenSource: CancellationTokenSource,
+  ): Promise<void> {
+    void extLogger.log("Executing auto-model queries");
+
+    const usages = await runAutoModelQueries({
+      mode,
+      candidateMethods,
+      cliServer: this.cliServer,
+      queryRunner: this.queryRunner,
+      queryStorageDir: this.queryStorageDir,
+      databaseItem: this.databaseItem,
+      progress: (update) => progress({ ...update }),
+      cancellationTokenSource,
+    });
+    if (!usages) {
+      return;
+    }
+
+    const request = await createAutoModelRequest(mode, usages);
+
+    void extLogger.log("Calling auto-model API");
+
+    const response = await this.callAutoModelApi(request);
+    if (!response) {
+      return;
+    }
+
+    const models = loadYaml(response.models, {
+      filename: "auto-model.yml",
+    });
+
+    const loadedMethods = loadDataExtensionYaml(models);
+    if (!loadedMethods) {
+      return;
+    }
+
+    // Any candidate that was part of the response is a negative result
+    // meaning that the canidate is not a sink for the kinds that the LLM is checking for.
+    // For now we model this as a sink neutral method, however this is subject
+    // to discussion.
+    for (const candidate of candidateMethods) {
+      if (!(candidate.signature in loadedMethods)) {
+        loadedMethods[candidate.signature] = {
+          type: "neutral",
+          kind: "sink",
+          input: "",
+          output: "",
+          provenance: "ai-generated",
+          signature: candidate.signature,
+          packageName: candidate.packageName,
+          typeName: candidate.typeName,
+          methodName: candidate.methodName,
+          methodParameters: candidate.methodParameters,
+        };
+      }
+    }
+
+    await this.addModeledMethods(loadedMethods);
+  }
+
+  private async callAutoModelApi(
+    request: ModelRequest,
+  ): Promise<ModelResponse | null> {
+    try {
+      return await autoModel(this.app.credentials, request);
+    } catch (e) {
+      if (e instanceof RequestError && e.status === 429) {
+        void showAndLogExceptionWithTelemetry(
+          this.app.logger,
+          this.app.telemetry,
+          redactableError(e)`Rate limit hit, please try again soon.`,
+        );
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
+}

extensions/ql-vscode/src/data-extensions-editor/bqrs.ts

@@ -1,5 +1,11 @@
-import { DecodedBqrsChunk } from "../pure/bqrs-cli-types";
-import { Call, ExternalApiUsage } from "./external-api-usage";
+import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import {
+  Call,
+  CallClassification,
+  ExternalApiUsage,
+} from "./external-api-usage";
+import { ModeledMethodType } from "./modeled-method";
+import { parseLibraryFilename } from "./library";
 
 export function decodeBqrsToExternalApiUsages(
   chunk: DecodedBqrsChunk,
@@ -10,6 +16,10 @@ export function decodeBqrsToExternalApiUsages(
     const usage = tuple[0] as Call;
     const signature = tuple[1] as string;
     const supported = (tuple[2] as string) === "true";
+    let library = tuple[4] as string;
+    let libraryVersion: string | undefined = tuple[5] as string;
+    const type = tuple[6] as ModeledMethodType;
+    const classification = tuple[8] as CallClassification;
 
     const [packageWithType, methodDeclaration] = signature.split("#");
 
@@ -29,33 +39,42 @@ export function decodeBqrsToExternalApiUsages(
       methodDeclaration.indexOf("("),
     );
 
+    // For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
+    // properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
+    // for Java.
+    if (library.endsWith(".jar") || libraryVersion === "") {
+      const { name, version } = parseLibraryFilename(library);
+      library = name;
+      if (version) {
+        libraryVersion = version;
+      }
+    }
+
+    if (libraryVersion === "") {
+      libraryVersion = undefined;
+    }
+
     if (!methodsByApiName.has(signature)) {
       methodsByApiName.set(signature, {
+        library,
+        libraryVersion,
         signature,
         packageName,
         typeName,
         methodName,
         methodParameters,
         supported,
+        supportedType: type,
         usages: [],
       });
     }
 
     const method = methodsByApiName.get(signature)!;
-    method.usages.push(usage);
+    method.usages.push({
+      ...usage,
+      classification,
+    });
   });
 
-  const externalApiUsages = Array.from(methodsByApiName.values());
-  externalApiUsages.sort((a, b) => {
-    // Sort first by supported, putting unmodeled methods first.
-    if (a.supported && !b.supported) {
-      return 1;
-    }
-    if (!a.supported && b.supported) {
-      return -1;
-    }
-    // Then sort by number of usages descending
-    return b.usages.length - a.usages.length;
-  });
-  return externalApiUsages;
+  return Array.from(methodsByApiName.values());
 }

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -3,18 +3,34 @@ import { DataExtensionsEditorView } from "./data-extensions-editor-view";
 import { DataExtensionsEditorCommands } from "../common/commands";
 import { CliVersionConstraint, CodeQLCliServer } from "../codeql-cli/cli";
 import { QueryRunner } from "../query-server";
-import { DatabaseManager } from "../databases/local-databases";
+import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { App } from "../common/app";
-import { showAndLogErrorMessage } from "../helpers";
 import { withProgress } from "../common/vscode/progress";
-import { pickExtensionPackModelFile } from "./extension-pack-picker";
+import { pickExtensionPack } from "./extension-pack-picker";
+import {
+  showAndLogErrorMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../common/logging";
+import { dir } from "tmp-promise";
+import { fetchExternalApiQueries } from "./queries";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
+import { extLogger } from "../common/logging/vscode";
+import { isQueryLanguage } from "../common/query-language";
+import { setUpPack } from "./external-api-usage-query";
+import { DisposableObject } from "../common/disposable-object";
+import { ModelDetailsPanel } from "./model-details/model-details-panel";
+import { Mode } from "./shared/mode";
+import { showResolvableLocation } from "../databases/local-databases/locations";
+import { Usage } from "./external-api-usage";
 
 const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
 
-export class DataExtensionsEditorModule {
+export class DataExtensionsEditorModule extends DisposableObject {
   private readonly queryStorageDir: string;
+  private readonly modelDetailsPanel: ModelDetailsPanel;
 
   private constructor(
     private readonly ctx: ExtensionContext,
@@ -24,10 +40,12 @@ export class DataExtensionsEditorModule {
     private readonly queryRunner: QueryRunner,
     baseQueryStorageDir: string,
   ) {
+    super();
     this.queryStorageDir = join(
       baseQueryStorageDir,
       "data-extensions-editor-results",
     );
+    this.modelDetailsPanel = this.push(new ModelDetailsPanel(cliServer));
   }
 
   public static async initialize(
@@ -56,13 +74,18 @@ export class DataExtensionsEditorModule {
       "codeQL.openDataExtensionsEditor": async () => {
         const db = this.databaseManager.currentDatabaseItem;
         if (!db) {
-          void showAndLogErrorMessage("No database selected");
+          void showAndLogErrorMessage(this.app.logger, "No database selected");
           return;
         }
 
-        if (!SUPPORTED_LANGUAGES.includes(db.language)) {
+        const language = db.language;
+        if (
+          !SUPPORTED_LANGUAGES.includes(language) ||
+          !isQueryLanguage(language)
+        ) {
           void showAndLogErrorMessage(
-            `The data extensions editor is not supported for ${db.language} databases.`,
+            this.app.logger,
+            `The data extensions editor is not supported for ${language} databases.`,
           );
           return;
         }
@@ -71,14 +94,26 @@ export class DataExtensionsEditorModule {
           async (progress, token) => {
             if (!(await this.cliServer.cliConstraints.supportsQlpacksKind())) {
               void showAndLogErrorMessage(
+                this.app.logger,
                 `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND.format()} or later.`,
               );
               return;
             }
 
-            const modelFile = await pickExtensionPackModelFile(
+            if (
+              !(await this.cliServer.cliConstraints.supportsResolveExtensions())
+            ) {
+              void showAndLogErrorMessage(
+                this.app.logger,
+                `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS.format()} or later.`,
+              );
+              return;
+            }
+
+            const modelFile = await pickExtensionPack(
               this.cliServer,
               db,
+              this.app.logger,
               progress,
               token,
             );
@@ -86,6 +121,21 @@ export class DataExtensionsEditorModule {
               return;
             }
 
+            const query = fetchExternalApiQueries[language];
+            if (!query) {
+              void showAndLogExceptionWithTelemetry(
+                extLogger,
+                telemetryListener,
+                redactableError`No external API usage query found for language ${language}`,
+              );
+              return;
+            }
+
+            // Create new temporary directory for query files and pack dependencies
+            const queryDir = (await dir({ unsafeCleanup: true })).path;
+            await setUpPack(queryDir, query, language);
+            await this.cliServer.packInstall(queryDir);
+
             const view = new DataExtensionsEditorView(
               this.ctx,
               this.app,
@@ -93,8 +143,12 @@ export class DataExtensionsEditorModule {
               this.cliServer,
               this.queryRunner,
               this.queryStorageDir,
+              queryDir,
               db,
               modelFile,
+              Mode.Application,
+              this.modelDetailsPanel.setState.bind(this.modelDetailsPanel),
+              this.modelDetailsPanel.revealItem.bind(this.modelDetailsPanel),
             );
             await view.openView();
           },
@@ -103,6 +157,12 @@ export class DataExtensionsEditorModule {
           },
         );
       },
+      "codeQLDataExtensionsEditor.jumpToUsageLocation": async (
+        usage: Usage,
+        databaseItem: DatabaseItem,
+      ) => {
+        await showResolvableLocation(usage.url, databaseItem, this.app.logger);
+      },
     };
   }
 

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -4,9 +4,7 @@ import {
   Uri,
   ViewColumn,
   window,
-  workspace,
 } from "vscode";
-import { RequestError } from "@octokit/request-error";
 import {
   AbstractWebview,
   WebviewPanelConfig,
@@ -14,43 +12,44 @@ import {
 import {
   FromDataExtensionsEditorMessage,
   ToDataExtensionsEditorMessage,
-} from "../pure/interface-types";
-import { ProgressUpdate } from "../common/vscode/progress";
+} from "../common/interface-types";
+import { ProgressCallback, withProgress } from "../common/vscode/progress";
 import { QueryRunner } from "../query-server";
 import {
-  showAndLogErrorMessage,
   showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { extLogger } from "../common";
-import { outputFile, pathExists, readFile } from "fs-extra";
-import { load as loadYaml } from "js-yaml";
+  showAndLogErrorMessage,
+} from "../common/logging";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { asError, assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
 import { generateFlowModel } from "./generate-flow-model";
 import { promptImportGithubDatabase } from "../databases/database-fetcher";
 import { App } from "../common/app";
-import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
-import { showResolvableLocation } from "../interface-utils";
+import { showResolvableLocation } from "../databases/local-databases/locations";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
-import { redactableError } from "../pure/errors";
+import { redactableError } from "../common/errors";
 import { readQueryResults, runQuery } from "./external-api-usage-query";
-import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
-import { ExternalApiUsage } from "./external-api-usage";
+import { ExternalApiUsage, Usage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
-import { ExtensionPackModelFile } from "./shared/extension-pack";
-import { autoModel, ModelRequest, ModelResponse } from "./auto-model-api";
+import { ExtensionPack } from "./shared/extension-pack";
 import {
-  createAutoModelRequest,
-  parsePredictedClassifications,
-} from "./auto-model";
-import { showLlmGeneration } from "../config";
-import { getAutoModelUsages } from "./auto-model-usages-query";
+  enableFrameworkMode,
+  showLlmGeneration,
+  showModelDetailsView,
+} from "../config";
+import { Mode } from "./shared/mode";
+import { loadModeledMethods, saveModeledMethods } from "./modeled-method-fs";
+import { join } from "path";
+import { pickExtensionPack } from "./extension-pack-picker";
+import { getLanguageDisplayName } from "../common/query-language";
+import { AutoModeler } from "./auto-modeler";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
   FromDataExtensionsEditorMessage
 > {
+  private readonly autoModeler: AutoModeler;
+
   public constructor(
     ctx: ExtensionContext,
     private readonly app: App,
@@ -58,10 +57,35 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly cliServer: CodeQLCliServer,
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
+    private readonly queryDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly modelFile: ExtensionPackModelFile,
+    private readonly extensionPack: ExtensionPack,
+    private mode: Mode,
+    private readonly updateModelDetailsPanelState: (
+      externalApiUsages: ExternalApiUsage[],
+      databaseItem: DatabaseItem,
+    ) => Promise<void>,
+    private readonly revealItemInDetailsPanel: (usage: Usage) => Promise<void>,
   ) {
     super(ctx);
+
+    this.autoModeler = new AutoModeler(
+      app,
+      cliServer,
+      queryRunner,
+      queryStorageDir,
+      databaseItem,
+      async (packageName, inProgressMethods) => {
+        await this.postMessage({
+          t: "setInProgressMethods",
+          packageName,
+          inProgressMethods,
+        });
+      },
+      async (modeledMethods) => {
+        await this.postMessage({ t: "addModeledMethods", modeledMethods });
+      },
+    );
   }
 
   public async openView() {
@@ -74,10 +98,20 @@ export class DataExtensionsEditorView extends AbstractWebview<
   protected async getPanelConfig(): Promise<WebviewPanelConfig> {
     return {
       viewId: "data-extensions-editor",
-      title: "Data Extensions Editor",
+      title: `Modeling ${getLanguageDisplayName(
+        this.extensionPack.language,
+      )} (${this.extensionPack.name})`,
       viewColumn: ViewColumn.Active,
       preserveFocus: true,
       view: "data-extensions-editor",
+      iconPath: {
+        dark: Uri.file(
+          join(this.ctx.extensionPath, "media/dark/symbol-misc.svg"),
+        ),
+        light: Uri.file(
+          join(this.ctx.extensionPath, "media/light/symbol-misc.svg"),
+        ),
+      },
     };
   }
 
@@ -92,28 +126,39 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "viewLoaded":
         await this.onWebViewLoaded();
 
+        break;
+      case "openDatabase":
+        await this.app.commands.execute(
+          "revealInExplorer",
+          this.databaseItem.getSourceArchiveExplorerUri(),
+        );
+
         break;
       case "openExtensionPack":
         await this.app.commands.execute(
           "revealInExplorer",
-          Uri.file(this.modelFile.extensionPack.path),
+          Uri.file(this.extensionPack.path),
         );
 
         break;
-      case "openModelFile":
-        await window.showTextDocument(
-          await workspace.openTextDocument(this.modelFile.filename),
-        );
+      case "refreshExternalApiUsages":
+        await this.loadExternalApiUsages();
 
         break;
       case "jumpToUsage":
-        await this.jumpToUsage(msg.location);
+        await this.handleJumpToUsage(msg.usage);
 
         break;
       case "saveModeledMethods":
-        await this.saveModeledMethods(
+        await saveModeledMethods(
+          this.extensionPack,
+          this.databaseItem.name,
+          this.databaseItem.language,
           msg.externalApiUsages,
           msg.modeledMethods,
+          this.mode,
+          this.cliServer,
+          this.app.logger,
         );
         await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);
 
@@ -124,9 +169,21 @@ export class DataExtensionsEditorView extends AbstractWebview<
         break;
       case "generateExternalApiFromLlm":
         await this.generateModeledMethodsFromLlm(
+          msg.packageName,
           msg.externalApiUsages,
           msg.modeledMethods,
         );
+        break;
+      case "stopGeneratingExternalApiFromLlm":
+        await this.autoModeler.stopModeling(msg.packageName);
+        break;
+      case "modelDependency":
+        await this.modelDependency();
+        break;
+      case "switchMode":
+        this.mode = msg.mode;
+
+        await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);
 
         break;
       default:
@@ -145,341 +202,287 @@ export class DataExtensionsEditorView extends AbstractWebview<
   }
 
   private async setViewState(): Promise<void> {
+    const showLlmButton =
+      this.databaseItem.language === "java" && showLlmGeneration();
+
     await this.postMessage({
       t: "setDataExtensionEditorViewState",
       viewState: {
-        extensionPackModelFile: this.modelFile,
-        modelFileExists: await pathExists(this.modelFile.filename),
-        showLlmButton: showLlmGeneration(),
+        extensionPack: this.extensionPack,
+        enableFrameworkMode: enableFrameworkMode(),
+        showLlmButton,
+        mode: this.mode,
       },
     });
   }
 
-  protected async jumpToUsage(
-    location: ResolvableLocationValue,
-  ): Promise<void> {
-    try {
-      await showResolvableLocation(location, this.databaseItem);
-    } catch (e) {
-      if (e instanceof Error) {
-        if (e.message.match(/File not found/)) {
-          void window.showErrorMessage(
-            "Original file of this result is not in the database's source archive.",
-          );
-        } else {
-          void extLogger.log(`Unable to handleMsgFromView: ${e.message}`);
-        }
-      } else {
-        void extLogger.log(`Unable to handleMsgFromView: ${e}`);
-      }
+  protected async handleJumpToUsage(usage: Usage) {
+    if (showModelDetailsView()) {
+      await this.revealItemInDetailsPanel(usage);
     }
-  }
-
-  protected async saveModeledMethods(
-    externalApiUsages: ExternalApiUsage[],
-    modeledMethods: Record<string, ModeledMethod>,
-  ): Promise<void> {
-    const yaml = createDataExtensionYaml(
-      this.databaseItem.language,
-      externalApiUsages,
-      modeledMethods,
-    );
-
-    await outputFile(this.modelFile.filename, yaml);
-
-    void extLogger.log(
-      `Saved data extension YAML to ${this.modelFile.filename}`,
-    );
+    await showResolvableLocation(usage.url, this.databaseItem, this.app.logger);
   }
 
   protected async loadExistingModeledMethods(): Promise<void> {
     try {
-      if (!(await pathExists(this.modelFile.filename))) {
-        return;
-      }
-
-      const yaml = await readFile(this.modelFile.filename, "utf8");
-
-      const data = loadYaml(yaml, {
-        filename: this.modelFile.filename,
-      });
-
-      const existingModeledMethods = loadDataExtensionYaml(data);
-
-      if (!existingModeledMethods) {
-        void showAndLogErrorMessage(
-          `Failed to parse data extension YAML ${this.modelFile.filename}.`,
-        );
-        return;
-      }
-
+      const modeledMethods = await loadModeledMethods(
+        this.extensionPack,
+        this.cliServer,
+        this.app.logger,
+      );
       await this.postMessage({
-        t: "addModeledMethods",
-        modeledMethods: existingModeledMethods,
+        t: "loadModeledMethods",
+        modeledMethods,
       });
     } catch (e: unknown) {
       void showAndLogErrorMessage(
-        `Unable to read data extension YAML ${
-          this.modelFile.filename
-        }: ${getErrorMessage(e)}`,
+        this.app.logger,
+        `Unable to read data extension YAML: ${getErrorMessage(e)}`,
       );
     }
   }
 
   protected async loadExternalApiUsages(): Promise<void> {
-    const cancellationTokenSource = new CancellationTokenSource();
+    await withProgress(
+      async (progress) => {
+        try {
+          const cancellationTokenSource = new CancellationTokenSource();
+          const queryResult = await runQuery(this.mode, {
+            cliServer: this.cliServer,
+            queryRunner: this.queryRunner,
+            databaseItem: this.databaseItem,
+            queryStorageDir: this.queryStorageDir,
+            queryDir: this.queryDir,
+            progress: (update) => progress({ ...update, maxStep: 1500 }),
+            token: cancellationTokenSource.token,
+          });
+          if (!queryResult) {
+            return;
+          }
 
-    try {
-      const queryResult = await runQuery({
-        cliServer: this.cliServer,
-        queryRunner: this.queryRunner,
-        databaseItem: this.databaseItem,
-        queryStorageDir: this.queryStorageDir,
-        progress: (progressUpdate: ProgressUpdate) => {
-          void this.showProgress(progressUpdate, 1500);
-        },
-        token: cancellationTokenSource.token,
-      });
-      if (!queryResult) {
-        await this.clearProgress();
-        return;
-      }
+          progress({
+            message: "Decoding results",
+            step: 1100,
+            maxStep: 1500,
+          });
 
-      await this.showProgress({
-        message: "Decoding results",
-        step: 1100,
-        maxStep: 1500,
-      });
+          const bqrsChunk = await readQueryResults({
+            cliServer: this.cliServer,
+            bqrsPath: queryResult.outputDir.bqrsPath,
+          });
+          if (!bqrsChunk) {
+            return;
+          }
 
-      const bqrsChunk = await readQueryResults({
-        cliServer: this.cliServer,
-        bqrsPath: queryResult.outputDir.bqrsPath,
-      });
-      if (!bqrsChunk) {
-        await this.clearProgress();
-        return;
-      }
+          progress({
+            message: "Finalizing results",
+            step: 1450,
+            maxStep: 1500,
+          });
 
-      await this.showProgress({
-        message: "Finalizing results",
-        step: 1450,
-        maxStep: 1500,
-      });
+          const externalApiUsages = decodeBqrsToExternalApiUsages(bqrsChunk);
 
-      const externalApiUsages = decodeBqrsToExternalApiUsages(bqrsChunk);
-
-      await this.postMessage({
-        t: "setExternalApiUsages",
-        externalApiUsages,
-      });
-
-      await this.clearProgress();
-    } catch (err) {
-      void showAndLogExceptionWithTelemetry(
-        redactableError(
-          asError(err),
-        )`Failed to load external API usages: ${getErrorMessage(err)}`,
-      );
-    }
+          await this.postMessage({
+            t: "setExternalApiUsages",
+            externalApiUsages,
+          });
+          await this.updateModelDetailsPanelState(
+            externalApiUsages,
+            this.databaseItem,
+          );
+        } catch (err) {
+          void showAndLogExceptionWithTelemetry(
+            this.app.logger,
+            this.app.telemetry,
+            redactableError(
+              asError(err),
+            )`Failed to load external API usages: ${getErrorMessage(err)}`,
+          );
+        }
+      },
+      { cancellable: false },
+    );
   }
 
   protected async generateModeledMethods(): Promise<void> {
-    const tokenSource = new CancellationTokenSource();
+    await withProgress(
+      async (progress) => {
+        const tokenSource = new CancellationTokenSource();
 
-    const selectedDatabase = this.databaseManager.currentDatabaseItem;
+        let addedDatabase: DatabaseItem | undefined;
 
+        // In application mode, we need the database of a specific library to generate
+        // the modeled methods. In framework mode, we'll use the current database.
+        if (this.mode === Mode.Application) {
+          addedDatabase = await this.promptChooseNewOrExistingDatabase(
+            progress,
+          );
+          if (!addedDatabase) {
+            return;
+          }
+        }
+
+        progress({
+          step: 0,
+          maxStep: 4000,
+          message: "Generating modeled methods for library",
+        });
+
+        try {
+          await generateFlowModel({
+            cliServer: this.cliServer,
+            queryRunner: this.queryRunner,
+            queryStorageDir: this.queryStorageDir,
+            databaseItem: addedDatabase ?? this.databaseItem,
+            onResults: async (modeledMethods) => {
+              const modeledMethodsByName: Record<string, ModeledMethod> = {};
+
+              for (const modeledMethod of modeledMethods) {
+                modeledMethodsByName[modeledMethod.signature] = modeledMethod;
+              }
+
+              await this.postMessage({
+                t: "addModeledMethods",
+                modeledMethods: modeledMethodsByName,
+              });
+            },
+            progress,
+            token: tokenSource.token,
+          });
+        } catch (e: unknown) {
+          void showAndLogExceptionWithTelemetry(
+            this.app.logger,
+            this.app.telemetry,
+            redactableError(
+              asError(e),
+            )`Failed to generate flow model: ${getErrorMessage(e)}`,
+          );
+        }
+      },
+      { cancellable: false },
+    );
+  }
+
+  private async generateModeledMethodsFromLlm(
+    packageName: string,
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+  ): Promise<void> {
+    await this.autoModeler.startModeling(
+      packageName,
+      externalApiUsages,
+      modeledMethods,
+      this.mode,
+    );
+  }
+
+  private async modelDependency(): Promise<void> {
+    return withProgress(async (progress, token) => {
+      const addedDatabase = await this.promptChooseNewOrExistingDatabase(
+        progress,
+      );
+      if (!addedDatabase || token.isCancellationRequested) {
+        return;
+      }
+
+      const modelFile = await pickExtensionPack(
+        this.cliServer,
+        addedDatabase,
+        this.app.logger,
+        progress,
+        token,
+      );
+      if (!modelFile) {
+        return;
+      }
+
+      const view = new DataExtensionsEditorView(
+        this.ctx,
+        this.app,
+        this.databaseManager,
+        this.cliServer,
+        this.queryRunner,
+        this.queryStorageDir,
+        this.queryDir,
+        addedDatabase,
+        modelFile,
+        Mode.Framework,
+        this.updateModelDetailsPanelState,
+        this.revealItemInDetailsPanel,
+      );
+      await view.openView();
+    });
+  }
+
+  private async promptChooseNewOrExistingDatabase(
+    progress: ProgressCallback,
+  ): Promise<DatabaseItem | undefined> {
+    const language = this.databaseItem.language;
+    const databases = this.databaseManager.databaseItems.filter(
+      (db) => db.language === language,
+    );
+    if (databases.length === 0) {
+      return await this.promptImportDatabase(progress);
+    } else {
+      const local = {
+        label: "$(database) Use existing database",
+        detail: "Use database from the workspace",
+      };
+      const github = {
+        label: "$(repo) Import database",
+        detail: "Choose database from GitHub",
+      };
+      const newOrExistingDatabase = await window.showQuickPick([local, github]);
+
+      if (!newOrExistingDatabase) {
+        void this.app.logger.log("No database chosen");
+        return;
+      }
+
+      if (newOrExistingDatabase === local) {
+        const pickedDatabase = await window.showQuickPick(
+          databases.map((database) => ({
+            label: database.name,
+            description: database.language,
+            database,
+          })),
+          {
+            placeHolder: "Pick a database",
+          },
+        );
+        if (!pickedDatabase) {
+          void this.app.logger.log("No database chosen");
+          return;
+        }
+        return pickedDatabase.database;
+      } else {
+        return await this.promptImportDatabase(progress);
+      }
+    }
+  }
+
+  private async promptImportDatabase(
+    progress: ProgressCallback,
+  ): Promise<DatabaseItem | undefined> {
     // The external API methods are in the library source code, so we need to ask
     // the user to import the library database. We need to have the database
     // imported to the query server, so we need to register it to our workspace.
-    const database = await promptImportGithubDatabase(
+    const makeSelected = false;
+    const addedDatabase = await promptImportGithubDatabase(
       this.app.commands,
       this.databaseManager,
       this.app.workspaceStoragePath ?? this.app.globalStoragePath,
       this.app.credentials,
-      (update) => this.showProgress(update),
-      tokenSource.token,
+      progress,
       this.cliServer,
-    );
-    if (!database) {
-      await this.clearProgress();
-      void extLogger.log("No database chosen");
-
-      return;
-    }
-
-    // The library database was set as the current database by importing it,
-    // but we need to set it back to the originally selected database.
-    await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
-
-    await this.showProgress({
-      step: 0,
-      maxStep: 4000,
-      message: "Generating modeled methods for library",
-    });
-
-    try {
-      await generateFlowModel({
-        cliServer: this.cliServer,
-        queryRunner: this.queryRunner,
-        queryStorageDir: this.queryStorageDir,
-        databaseItem: database,
-        onResults: async (results) => {
-          const modeledMethodsByName: Record<string, ModeledMethod> = {};
-
-          for (const result of results) {
-            modeledMethodsByName[result.signature] = result.modeledMethod;
-          }
-
-          await this.postMessage({
-            t: "addModeledMethods",
-            modeledMethods: modeledMethodsByName,
-            overrideNone: true,
-          });
-        },
-        progress: (update) => this.showProgress(update),
-        token: tokenSource.token,
-      });
-    } catch (e: unknown) {
-      void showAndLogExceptionWithTelemetry(
-        redactableError(
-          asError(e),
-        )`Failed to generate flow model: ${getErrorMessage(e)}`,
-      );
-    }
-
-    // After the flow model has been generated, we can remove the temporary database
-    // which we used for generating the flow model.
-    await this.databaseManager.removeDatabaseItem(
-      () =>
-        this.showProgress({
-          step: 3900,
-          maxStep: 4000,
-          message: "Removing temporary database",
-        }),
-      tokenSource.token,
-      database,
-    );
-
-    await this.clearProgress();
-  }
-
-  private async generateModeledMethodsFromLlm(
-    externalApiUsages: ExternalApiUsage[],
-    modeledMethods: Record<string, ModeledMethod>,
-  ): Promise<void> {
-    const maxStep = 3000;
-
-    await this.showProgress({
-      step: 0,
-      maxStep,
-      message: "Retrieving usages",
-    });
-
-    const usages = await getAutoModelUsages({
-      cliServer: this.cliServer,
-      queryRunner: this.queryRunner,
-      queryStorageDir: this.queryStorageDir,
-      databaseItem: this.databaseItem,
-      progress: (update) => this.showProgress(update, maxStep),
-    });
-
-    await this.showProgress({
-      step: 1800,
-      maxStep,
-      message: "Creating request",
-    });
-
-    const request = createAutoModelRequest(
       this.databaseItem.language,
-      externalApiUsages,
-      modeledMethods,
-      usages,
+      makeSelected,
     );
-
-    await this.showProgress({
-      step: 2000,
-      maxStep,
-      message: "Sending request",
-    });
-
-    const response = await this.callAutoModelApi(request);
-    if (!response) {
+    if (!addedDatabase) {
+      void this.app.logger.log("No database chosen");
       return;
     }
 
-    await this.showProgress({
-      step: 2500,
-      maxStep,
-      message: "Parsing response",
-    });
-
-    const predictedModeledMethods = parsePredictedClassifications(
-      response.predicted,
-    );
-
-    await this.showProgress({
-      step: 2800,
-      maxStep,
-      message: "Applying results",
-    });
-
-    await this.postMessage({
-      t: "addModeledMethods",
-      modeledMethods: predictedModeledMethods,
-      overrideNone: true,
-    });
-
-    await this.clearProgress();
-  }
-
-  /*
-   * Progress in this class is a bit weird. Most of the progress is based on running the query.
-   * Query progress is always between 0 and 1000. However, we still have some steps that need
-   * to be done after the query has finished. Therefore, the maximum step is 1500. This captures
-   * that there's 1000 steps of the query progress since that takes the most time, and then
-   * an additional 500 steps for the rest of the work. The progress doesn't need to be 100%
-   * accurate, so this is just a rough estimate.
-   *
-   * For generating the modeled methods for an external library, the max step is 4000. This is
-   * based on the following steps:
-   * - 1000 for the summary model
-   * - 1000 for the sink model
-   * - 1000 for the source model
-   * - 1000 for the neutral model
-   */
-  private async showProgress(update: ProgressUpdate, maxStep?: number) {
-    await this.postMessage({
-      t: "showProgress",
-      step: update.step,
-      maxStep: maxStep ?? update.maxStep,
-      message: update.message,
-    });
-  }
-
-  private async clearProgress() {
-    await this.showProgress({
-      step: 0,
-      maxStep: 0,
-      message: "",
-    });
-  }
-
-  private async callAutoModelApi(
-    request: ModelRequest,
-  ): Promise<ModelResponse | null> {
-    try {
-      return await autoModel(this.app.credentials, request);
-    } catch (e) {
-      await this.clearProgress();
-
-      if (e instanceof RequestError && e.status === 429) {
-        void showAndLogExceptionWithTelemetry(
-          redactableError(e)`Rate limit hit, please try again soon.`,
-        );
-        return null;
-      } else {
-        throw e;
-      }
-    }
+    return addedDatabase;
   }
 }

extensions/ql-vscode/src/data-extensions-editor/extension-pack-name.ts

@@ -0,0 +1,93 @@
+const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
+const packNameRegex = new RegExp(
+  `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
+);
+const packNameLength = 128;
+
+export interface ExtensionPackName {
+  scope: string;
+  name: string;
+}
+
+export function formatPackName(packName: ExtensionPackName): string {
+  return `${packName.scope}/${packName.name}`;
+}
+
+export function autoNameExtensionPack(
+  name: string,
+  language: string,
+): ExtensionPackName | undefined {
+  let packName = `${name}-${language}`;
+  if (!packName.includes("/")) {
+    packName = `pack/${packName}`;
+  }
+
+  const parts = packName.split("/");
+  const sanitizedParts = parts.map((part) => sanitizeExtensionPackName(part));
+
+  // If the scope is empty (e.g. if the given name is "-/b"), then we need to still set a scope
+  if (sanitizedParts[0].length === 0) {
+    sanitizedParts[0] = "pack";
+  }
+
+  return {
+    scope: sanitizedParts[0],
+    // This will ensure there's only 1 slash
+    name: sanitizedParts.slice(1).join("-"),
+  };
+}
+
+export function sanitizeExtensionPackName(name: string) {
+  // Lowercase everything
+  name = name.toLowerCase();
+
+  // Replace all spaces, dots, and underscores with hyphens
+  name = name.replaceAll(/[\s._]+/g, "-");
+
+  // Replace all characters which are not allowed by empty strings
+  name = name.replaceAll(/[^a-z0-9-]/g, "");
+
+  // Remove any leading or trailing hyphens
+  name = name.replaceAll(/^-|-$/g, "");
+
+  // Remove any duplicate hyphens
+  name = name.replaceAll(/-{2,}/g, "-");
+
+  return name;
+}
+
+export function parsePackName(packName: string): ExtensionPackName | undefined {
+  const matches = packNameRegex.exec(packName);
+  if (!matches?.groups) {
+    return;
+  }
+
+  const scope = matches.groups.scope;
+  const name = matches.groups.name;
+
+  return {
+    scope,
+    name,
+  };
+}
+
+export function validatePackName(name: string): string | undefined {
+  if (!name) {
+    return "Pack name must not be empty";
+  }
+
+  if (name.length > packNameLength) {
+    return `Pack name must be no longer than ${packNameLength} characters`;
+  }
+
+  const matches = packNameRegex.exec(name);
+  if (!matches?.groups) {
+    if (!name.includes("/")) {
+      return "Invalid package name: a pack name must contain a slash to separate the scope from the pack name";
+    }
+
+    return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
+  }
+
+  return undefined;
+}

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -1,64 +1,37 @@
-import { join, relative, resolve, sep } from "path";
+import { join } from "path";
 import { outputFile, pathExists, readFile } from "fs-extra";
 import { dump as dumpYaml, load as loadYaml } from "js-yaml";
-import { minimatch } from "minimatch";
-import { CancellationToken, window } from "vscode";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import {
-  getOnDiskWorkspaceFolders,
-  getOnDiskWorkspaceFoldersObjects,
-  showAndLogErrorMessage,
-} from "../helpers";
+import { CancellationToken, Uri, window } from "vscode";
+import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { ProgressCallback } from "../common/vscode/progress";
 import { DatabaseItem } from "../databases/local-databases";
-import { getQlPackPath, QLPACK_FILENAMES } from "../pure/ql";
-import { getErrorMessage } from "../pure/helpers-pure";
-import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
+import { getQlPackPath, QLPACK_FILENAMES } from "../common/ql";
+import { getErrorMessage } from "../common/helpers-pure";
+import { ExtensionPack } from "./shared/extension-pack";
+import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
+import {
+  disableAutoNameExtensionPack,
+  getExtensionsDirectory,
+} from "../config";
+import {
+  autoNameExtensionPack,
+  ExtensionPackName,
+  formatPackName,
+  parsePackName,
+  validatePackName,
+} from "./extension-pack-name";
+import {
+  askForWorkspaceFolder,
+  autoPickExtensionsDirectory,
+} from "./extensions-workspace-folder";
 
 const maxStep = 3;
 
-const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
-const packNameRegex = new RegExp(
-  `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
-);
-const packNameLength = 128;
-
-export async function pickExtensionPackModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name" | "language">,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<ExtensionPackModelFile | undefined> {
-  const extensionPack = await pickExtensionPack(
-    cliServer,
-    databaseItem,
-    progress,
-    token,
-  );
-  if (!extensionPack) {
-    return undefined;
-  }
-
-  const modelFile = await pickModelFile(
-    cliServer,
-    databaseItem,
-    extensionPack,
-    progress,
-    token,
-  );
-  if (!modelFile) {
-    return;
-  }
-
-  return {
-    filename: modelFile,
-    extensionPack,
-  };
-}
-
-async function pickExtensionPack(
+export async function pickExtensionPack(
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
   databaseItem: Pick<DatabaseItem, "name" | "language">,
+  logger: NotificationLogger,
   progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<ExtensionPack | undefined> {
@@ -70,11 +43,30 @@ async function pickExtensionPack(
 
   // Get all existing extension packs in the workspace
   const additionalPacks = getOnDiskWorkspaceFolders();
+  // the CLI doesn't check packs in the .github folder, so we need to add it manually
+  if (additionalPacks.length === 1) {
+    additionalPacks.push(`${additionalPacks[0]}/.github`);
+  }
   const extensionPacksInfo = await cliServer.resolveQlpacks(
     additionalPacks,
     true,
   );
 
+  if (!disableAutoNameExtensionPack()) {
+    progress({
+      message: "Creating extension pack...",
+      step: 2,
+      maxStep,
+    });
+
+    return autoCreateExtensionPack(
+      databaseItem.name,
+      databaseItem.language,
+      extensionPacksInfo,
+      logger,
+    );
+  }
+
   if (Object.keys(extensionPacksInfo).length === 0) {
     return pickNewExtensionPack(databaseItem, token);
   }
@@ -84,6 +76,7 @@ async function pickExtensionPack(
       Object.entries(extensionPacksInfo).map(async ([name, paths]) => {
         if (paths.length !== 1) {
           void showAndLogErrorMessage(
+            logger,
             `Extension pack ${name} resolves to multiple paths`,
             {
               fullMessage: `Extension pack ${name} resolves to multiple paths: ${paths.join(
@@ -99,13 +92,17 @@ async function pickExtensionPack(
 
         let extensionPack: ExtensionPack;
         try {
-          extensionPack = await readExtensionPack(path);
+          extensionPack = await readExtensionPack(path, databaseItem.language);
         } catch (e: unknown) {
-          void showAndLogErrorMessage(`Could not read extension pack ${name}`, {
-            fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
-              e,
-            )}`,
-          });
+          void showAndLogErrorMessage(
+            logger,
+            `Could not read extension pack ${name}`,
+            {
+              fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
+                e,
+              )}`,
+            },
+          );
 
           return undefined;
         }
@@ -163,118 +160,39 @@ async function pickExtensionPack(
   return extensionPackOption.extensionPack;
 }
 
-async function pickModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<string | undefined> {
-  // Find the existing model files in the extension pack
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensions = await cliServer.resolveExtensions(
-    extensionPack.path,
-    additionalPacks,
-  );
-
-  const modelFiles = new Set<string>();
-
-  if (extensionPack.path in extensions.data) {
-    for (const extension of extensions.data[extensionPack.path]) {
-      modelFiles.add(extension.file);
-    }
-  }
-
-  if (modelFiles.size === 0) {
-    return pickNewModelFile(databaseItem, extensionPack, token);
-  }
-
-  const fileOptions: Array<{ label: string; file: string | null }> = [];
-  for (const file of modelFiles) {
-    fileOptions.push({
-      label: relative(extensionPack.path, file).replaceAll(sep, "/"),
-      file,
-    });
-  }
-  fileOptions.push({
-    label: "Create new model file",
-    file: null,
-  });
-
-  progress({
-    message: "Choosing model file...",
-    step: 3,
-    maxStep,
-  });
-
-  const fileOption = await window.showQuickPick(
-    fileOptions,
-    {
-      title: "Select model file to use",
-    },
-    token,
-  );
-
-  if (!fileOption) {
-    return undefined;
-  }
-
-  if (fileOption.file) {
-    return fileOption.file;
-  }
-
-  return pickNewModelFile(databaseItem, extensionPack, token);
-}
-
 async function pickNewExtensionPack(
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   token: CancellationToken,
 ): Promise<ExtensionPack | undefined> {
-  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
-  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
-    label: folder.name,
-    detail: folder.uri.fsPath,
-    path: folder.uri.fsPath,
-  }));
-
-  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
-  // we only want to include on-disk workspace folders.
-  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
-    title: "Select workspace folder to create extension pack in",
-  });
+  const workspaceFolder = await askForWorkspaceFolder();
   if (!workspaceFolder) {
     return undefined;
   }
 
-  let examplePackName = `${databaseItem.name}-extensions`;
-  if (!examplePackName.includes("/")) {
-    examplePackName = `pack/${examplePackName}`;
-  }
+  const examplePackName = autoNameExtensionPack(
+    databaseItem.name,
+    databaseItem.language,
+  );
 
-  const packName = await window.showInputBox(
+  const name = await window.showInputBox(
     {
       title: "Create new extension pack",
       prompt: "Enter name of extension pack",
-      placeHolder: `e.g. ${examplePackName}`,
+      placeHolder: examplePackName
+        ? `e.g. ${formatPackName(examplePackName)}`
+        : "",
       validateInput: async (value: string): Promise<string | undefined> => {
-        if (!value) {
-          return "Pack name must not be empty";
+        const message = validatePackName(value);
+        if (message) {
+          return message;
         }
 
-        if (value.length > packNameLength) {
-          return `Pack name must be no longer than ${packNameLength} characters`;
+        const packName = parsePackName(value);
+        if (!packName) {
+          return "Invalid pack name";
         }
 
-        const matches = packNameRegex.exec(value);
-        if (!matches?.groups) {
-          if (!value.includes("/")) {
-            return "Invalid package name: a pack name must contain a slash to separate the scope from the pack name";
-          }
-
-          return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
-        }
-
-        const packPath = join(workspaceFolder.path, matches.groups.name);
+        const packPath = join(workspaceFolder.uri.fsPath, packName.name);
         if (await pathExists(packPath)) {
           return `A pack already exists at ${packPath}`;
         }
@@ -284,31 +202,131 @@ async function pickNewExtensionPack(
     },
     token,
   );
+  if (!name) {
+    return undefined;
+  }
+
+  const packName = parsePackName(name);
   if (!packName) {
     return undefined;
   }
 
-  const matches = packNameRegex.exec(packName);
-  if (!matches?.groups) {
-    return;
-  }
-
-  const name = matches.groups.name;
-  const packPath = join(workspaceFolder.path, name);
+  const packPath = join(workspaceFolder.uri.fsPath, packName.name);
 
   if (await pathExists(packPath)) {
     return undefined;
   }
 
+  return writeExtensionPack(packPath, packName, databaseItem.language);
+}
+
+async function autoCreateExtensionPack(
+  name: string,
+  language: string,
+  extensionPacksInfo: QlpacksInfo,
+  logger: NotificationLogger,
+): Promise<ExtensionPack | undefined> {
+  // Get the `codeQL.dataExtensions.extensionsDirectory` setting for the language
+  const userExtensionsDirectory = getExtensionsDirectory(language);
+
+  // If the setting is not set, automatically pick a suitable directory
+  const extensionsDirectory = userExtensionsDirectory
+    ? Uri.file(userExtensionsDirectory)
+    : await autoPickExtensionsDirectory();
+
+  if (!extensionsDirectory) {
+    return undefined;
+  }
+
+  // Generate the name of the extension pack
+  const packName = autoNameExtensionPack(name, language);
+  if (!packName) {
+    void showAndLogErrorMessage(
+      logger,
+      `Could not automatically name extension pack for database ${name}`,
+    );
+
+    return undefined;
+  }
+
+  // Find any existing locations of this extension pack
+  const existingExtensionPackPaths =
+    extensionPacksInfo[formatPackName(packName)];
+
+  // If there is already an extension pack with this name, use it if it is valid
+  if (existingExtensionPackPaths?.length === 1) {
+    let extensionPack: ExtensionPack;
+    try {
+      extensionPack = await readExtensionPack(
+        existingExtensionPackPaths[0],
+        language,
+      );
+    } catch (e: unknown) {
+      void showAndLogErrorMessage(
+        logger,
+        `Could not read extension pack ${formatPackName(packName)}`,
+        {
+          fullMessage: `Could not read extension pack ${formatPackName(
+            packName,
+          )} at ${existingExtensionPackPaths[0]}: ${getErrorMessage(e)}`,
+        },
+      );
+
+      return undefined;
+    }
+
+    return extensionPack;
+  }
+
+  // If there is already an existing extension pack with this name, but it resolves
+  // to multiple paths, then we can't use it
+  if (existingExtensionPackPaths?.length > 1) {
+    void showAndLogErrorMessage(
+      logger,
+      `Extension pack ${formatPackName(packName)} resolves to multiple paths`,
+      {
+        fullMessage: `Extension pack ${formatPackName(
+          packName,
+        )} resolves to multiple paths: ${existingExtensionPackPaths.join(
+          ", ",
+        )}`,
+      },
+    );
+
+    return undefined;
+  }
+
+  const packPath = join(extensionsDirectory.fsPath, packName.name);
+
+  if (await pathExists(packPath)) {
+    void showAndLogErrorMessage(
+      logger,
+      `Directory ${packPath} already exists for extension pack ${formatPackName(
+        packName,
+      )}`,
+    );
+
+    return undefined;
+  }
+
+  return writeExtensionPack(packPath, packName, language);
+}
+
+async function writeExtensionPack(
+  packPath: string,
+  packName: ExtensionPackName,
+  language: string,
+): Promise<ExtensionPack> {
   const packYamlPath = join(packPath, "codeql-pack.yml");
 
   const extensionPack: ExtensionPack = {
     path: packPath,
     yamlPath: packYamlPath,
-    name: packName,
+    name: formatPackName(packName),
     version: "0.0.0",
+    language,
     extensionTargets: {
-      [`codeql/${databaseItem.language}-all`]: "*",
+      [`codeql/${language}-all`]: "*",
     },
     dataExtensions: ["models/**/*.yml"],
   };
@@ -327,54 +345,10 @@ async function pickNewExtensionPack(
   return extensionPack;
 }
 
-async function pickNewModelFile(
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  token: CancellationToken,
-) {
-  const filename = await window.showInputBox(
-    {
-      title: "Enter the name of the new model file",
-      value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
-      validateInput: async (value: string): Promise<string | undefined> => {
-        if (value === "") {
-          return "File name must not be empty";
-        }
-
-        const path = resolve(extensionPack.path, value);
-
-        if (await pathExists(path)) {
-          return "File already exists";
-        }
-
-        const notInExtensionPack = relative(
-          extensionPack.path,
-          path,
-        ).startsWith("..");
-        if (notInExtensionPack) {
-          return "File must be in the extension pack";
-        }
-
-        const matchesPattern = extensionPack.dataExtensions.some((pattern) =>
-          minimatch(value, pattern, { matchBase: true }),
-        );
-        if (!matchesPattern) {
-          return `File must match one of the patterns in 'dataExtensions' in ${extensionPack.yamlPath}`;
-        }
-
-        return undefined;
-      },
-    },
-    token,
-  );
-  if (!filename) {
-    return undefined;
-  }
-
-  return resolve(extensionPack.path, filename);
-}
-
-async function readExtensionPack(path: string): Promise<ExtensionPack> {
+async function readExtensionPack(
+  path: string,
+  language: string,
+): Promise<ExtensionPack> {
   const qlpackPath = await getQlPackPath(path);
   if (!qlpackPath) {
     throw new Error(
@@ -411,6 +385,7 @@ async function readExtensionPack(path: string): Promise<ExtensionPack> {
     yamlPath: qlpackPath,
     name: qlpack.name,
     version: qlpack.version,
+    language,
     extensionTargets: qlpack.extensionTargets,
     dataExtensions,
   };

extensions/ql-vscode/src/data-extensions-editor/extensions-workspace-folder.ts

@@ -0,0 +1,224 @@
+import { FileType, Uri, window, workspace, WorkspaceFolder } from "vscode";
+import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { tmpdir } from "../common/files";
+
+/**
+ * Returns the ancestors of this path in order from furthest to closest (i.e. root of filesystem to parent directory)
+ */
+function getAncestors(uri: Uri): Uri[] {
+  const ancestors: Uri[] = [];
+  let current = uri;
+  while (current.fsPath !== Uri.joinPath(current, "..").fsPath) {
+    ancestors.push(current);
+    current = Uri.joinPath(current, "..");
+  }
+
+  // The ancestors are now in order from closest to furthest, so reverse them
+  ancestors.reverse();
+
+  return ancestors;
+}
+
+async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
+  // If there is a valid workspace file, just use its directory as the directory for the extensions
+  const workspaceFile = workspace.workspaceFile;
+  if (workspaceFile?.scheme === "file") {
+    return Uri.joinPath(workspaceFile, "..");
+  }
+
+  const allWorkspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  // Get the system temp directory and convert it to a URI so it's normalized
+  const systemTmpdir = Uri.file(tmpdir());
+
+  const workspaceFolders = allWorkspaceFolders.filter((folder) => {
+    // Never use a workspace folder that is in the system temp directory
+    return !folder.uri.fsPath.startsWith(systemTmpdir.fsPath);
+  });
+
+  // Find the common root directory of all workspace folders by finding the longest common prefix
+  const commonRoot = workspaceFolders.reduce((commonRoot, folder) => {
+    const folderUri = folder.uri;
+    const ancestors = getAncestors(folderUri);
+
+    const minLength = Math.min(commonRoot.length, ancestors.length);
+    let commonLength = 0;
+    for (let i = 0; i < minLength; i++) {
+      if (commonRoot[i].fsPath === ancestors[i].fsPath) {
+        commonLength++;
+      } else {
+        break;
+      }
+    }
+
+    return commonRoot.slice(0, commonLength);
+  }, getAncestors(workspaceFolders[0].uri));
+
+  if (commonRoot.length === 0) {
+    return await findGitFolder(workspaceFolders);
+  }
+
+  // The path closest to the workspace folders is the last element of the common root
+  const commonRootUri = commonRoot[commonRoot.length - 1];
+
+  // If we are at the root of the filesystem, we can't go up any further and there's something
+  // wrong, so just return undefined
+  if (commonRootUri.fsPath === Uri.joinPath(commonRootUri, "..").fsPath) {
+    return await findGitFolder(workspaceFolders);
+  }
+
+  return commonRootUri;
+}
+
+async function findGitFolder(
+  workspaceFolders: WorkspaceFolder[],
+): Promise<Uri | undefined> {
+  // Go through all workspace folders one-by-one and try to find the closest .git folder for each one
+  const folders = await Promise.all(
+    workspaceFolders.map(async (folder) => {
+      const ancestors = getAncestors(folder.uri);
+
+      // Reverse the ancestors so we're going from closest to furthest
+      ancestors.reverse();
+
+      const gitFoldersExists = await Promise.all(
+        ancestors.map(async (uri) => {
+          const gitFolder = Uri.joinPath(uri, ".git");
+          try {
+            const stat = await workspace.fs.stat(gitFolder);
+            // Check whether it's a directory
+            return (stat.type & FileType.Directory) !== 0;
+          } catch (e) {
+            return false;
+          }
+        }),
+      );
+
+      // Find the first ancestor that has a .git folder
+      const ancestorIndex = gitFoldersExists.findIndex((exists) => exists);
+
+      if (ancestorIndex === -1) {
+        return undefined;
+      }
+
+      return [ancestorIndex, ancestors[ancestorIndex]];
+    }),
+  );
+
+  const validFolders = folders.filter(
+    (folder): folder is [number, Uri] => folder !== undefined,
+  );
+  if (validFolders.length === 0) {
+    return undefined;
+  }
+
+  // Find the .git folder which is closest to a workspace folder
+  const closestFolder = validFolders.reduce((closestFolder, folder) => {
+    if (folder[0] < closestFolder[0]) {
+      return folder;
+    }
+    return closestFolder;
+  }, validFolders[0]);
+
+  return closestFolder?.[1];
+}
+
+/**
+ * Finds a suitable directory for extension packs to be created in. This will
+ * always be a path ending in `.github/codeql/extensions`. The parent directory
+ * will be determined heuristically based on the on-disk workspace folders.
+ *
+ * The heuristic is as follows (`.github/codeql/extensions` is added automatically unless
+ * otherwise specified):
+ * 1. If there is only 1 workspace folder, use that folder
+ * 2. If there is a workspace folder for which the path ends in `.github/codeql/extensions`, use that folder
+ *   - If there are multiple such folders, use the first one
+ *   - Does not append `.github/codeql/extensions` to the path
+ * 3. If there is a workspace file (`<basename>.code-workspace`), use the directory containing that file
+ * 4. If there is a common root directory for all workspace folders, use that directory
+ *   - Workspace folders in the system temp directory are ignored
+ *   - If the common root directory is the root of the filesystem, then it's not used
+ * 5. If there is a .git directory in any workspace folder, use the directory containing that .git directory
+ *    for which the .git directory is closest to a workspace folder
+ * 6. If none of the above apply, return `undefined`
+ */
+export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  // If there's only 1 workspace folder, use the `.github/codeql/extensions` directory in that folder
+  if (workspaceFolders.length === 1) {
+    return Uri.joinPath(
+      workspaceFolders[0].uri,
+      ".github",
+      "codeql",
+      "extensions",
+    );
+  }
+
+  // Now try to find a workspace folder for which the path ends in `.github/codeql/extensions`
+  const workspaceFolderForExtensions = workspaceFolders.find((folder) =>
+    // Using path instead of fsPath because path always uses forward slashes
+    folder.uri.path.endsWith(".github/codeql/extensions"),
+  );
+  if (workspaceFolderForExtensions) {
+    return workspaceFolderForExtensions.uri;
+  }
+
+  // Get the root workspace directory, i.e. the common root directory of all workspace folders
+  const rootDirectory = await getRootWorkspaceDirectory();
+  if (!rootDirectory) {
+    void extLogger.log("Unable to determine root workspace directory");
+
+    return undefined;
+  }
+
+  // We'll create a new workspace folder for the extensions in the root workspace directory
+  // at `.github/codeql/extensions`
+  const extensionsUri = Uri.joinPath(
+    rootDirectory,
+    ".github",
+    "codeql",
+    "extensions",
+  );
+
+  if (
+    !workspace.updateWorkspaceFolders(
+      workspace.workspaceFolders?.length ?? 0,
+      0,
+      {
+        name: "CodeQL Extension Packs",
+        uri: extensionsUri,
+      },
+    )
+  ) {
+    void extLogger.log(
+      `Failed to add workspace folder for extensions at ${extensionsUri.fsPath}`,
+    );
+    return undefined;
+  }
+
+  return extensionsUri;
+}
+
+export async function askForWorkspaceFolder(): Promise<
+  WorkspaceFolder | undefined
+> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
+    label: folder.name,
+    detail: folder.uri.fsPath,
+    folder,
+  }));
+
+  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
+  // we only want to include on-disk workspace folders.
+  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
+    title: "Select workspace folder to create extension pack in",
+  });
+  if (!workspaceFolder) {
+    return undefined;
+  }
+
+  return workspaceFolder.folder;
+}

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts

@@ -1,64 +1,44 @@
 import { CoreCompletedQuery, QueryRunner } from "../query-server";
-import { dir } from "tmp-promise";
-import { writeFile } from "fs-extra";
-import { dump as dumpYaml } from "js-yaml";
-import {
-  getOnDiskWorkspaceFolders,
-  isQueryLanguage,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { TeeLogger } from "../common";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
 import { CancellationToken } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseItem } from "../databases/local-databases";
 import { ProgressCallback } from "../common/vscode/progress";
-import { fetchExternalApiQueries } from "./queries";
-import { QueryResultType } from "../pure/new-messages";
+import { QueryResultType } from "../query-server/new-messages";
+import { redactableError } from "../common/errors";
+import { telemetryListener } from "../common/vscode/telemetry";
 import { join } from "path";
-import { redactableError } from "../pure/errors";
+import { Mode } from "./shared/mode";
+import { writeFile } from "fs-extra";
+import { Query } from "./queries/query";
+import { QueryLanguage } from "../common/query-language";
+import { dump } from "js-yaml";
 
-export type RunQueryOptions = {
+type RunQueryOptions = {
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
   queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
   databaseItem: Pick<DatabaseItem, "contents" | "databaseUri" | "language">;
   queryStorageDir: string;
+  queryDir: string;
 
   progress: ProgressCallback;
   token: CancellationToken;
 };
 
-export async function runQuery({
-  cliServer,
-  queryRunner,
-  databaseItem,
-  queryStorageDir,
-  progress,
-  token,
-}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
-  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
-  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
-  // This is intentionally not pretty code, as it will be removed soon.
-  // For a reference of what this should do in the future, see the previous implementation in
-  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
-
-  if (!isQueryLanguage(databaseItem.language)) {
-    void showAndLogExceptionWithTelemetry(
-      redactableError`Unsupported database language ${databaseItem.language}`,
+export async function setUpPack(
+  queryDir: string,
+  query: Query,
+  language: QueryLanguage,
+) {
+  Object.values(Mode).map(async (mode) => {
+    const queryFile = join(
+      queryDir,
+      `FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`,
     );
-    return;
-  }
-
-  const query = fetchExternalApiQueries[databaseItem.language];
-  if (!query) {
-    void showAndLogExceptionWithTelemetry(
-      redactableError`No external API usage query found for language ${databaseItem.language}`,
-    );
-    return;
-  }
-
-  const queryDir = (await dir({ unsafeCleanup: true })).path;
-  const queryFile = join(queryDir, "FetchExternalApis.ql");
-  await writeFile(queryFile, query.mainQuery, "utf8");
+    await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8");
+  });
 
   if (query.dependencies) {
     for (const [filename, contents] of Object.entries(query.dependencies)) {
@@ -71,18 +51,42 @@ export async function runQuery({
     name: "codeql/external-api-usage",
     version: "0.0.0",
     dependencies: {
-      [`codeql/${databaseItem.language}-all`]: "*",
+      [`codeql/${language}-all`]: "*",
     },
   };
 
   const qlpackFile = join(queryDir, "codeql-pack.yml");
-  await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8");
+  await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
+}
+
+export async function runQuery(
+  mode: Mode,
+  {
+    cliServer,
+    queryRunner,
+    databaseItem,
+    queryStorageDir,
+    queryDir,
+    progress,
+    token,
+  }: RunQueryOptions,
+): Promise<CoreCompletedQuery | undefined> {
+  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
+  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
+  // This is intentionally not pretty code, as it will be removed soon.
+  // For a reference of what this should do in the future, see the previous implementation in
+  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
 
   const additionalPacks = getOnDiskWorkspaceFolders();
   const extensionPacks = Object.keys(
     await cliServer.resolveQlpacks(additionalPacks, true),
   );
 
+  const queryFile = join(
+    queryDir,
+    `FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`,
+  );
+
   const queryRun = queryRunner.createQueryRun(
     databaseItem.databaseUri.fsPath,
     {
@@ -106,6 +110,8 @@ export async function runQuery({
 
   if (completedQuery.resultType !== QueryResultType.SUCCESS) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`External API usage query failed: ${
         completedQuery.message ?? "No message"
       }`,
@@ -116,7 +122,7 @@ export async function runQuery({
   return completedQuery;
 }
 
-export type GetResultsOptions = {
+type GetResultsOptions = {
   cliServer: Pick<CodeQLCliServer, "bqrsInfo" | "bqrsDecode">;
   bqrsPath: string;
 };
@@ -128,6 +134,8 @@ export async function readQueryResults({
   const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
   if (bqrsInfo["result-sets"].length !== 1) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
     );
     return undefined;

extensions/ql-vscode/src/data-extensions-editor/external-api-usage.ts

@@ -1,19 +1,54 @@
-import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { ModeledMethodType } from "./modeled-method";
 
 export type Call = {
   label: string;
   url: ResolvableLocationValue;
 };
 
-export type ExternalApiUsage = {
+export enum CallClassification {
+  Unknown = "unknown",
+  Source = "source",
+  Test = "test",
+  Generated = "generated",
+}
+
+export type Usage = Call & {
+  classification: CallClassification;
+};
+
+export interface MethodSignature {
   /**
-   * Contains the full method signature, e.g. `org.sql2o.Connection#createQuery(String)`
+   * Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
+   */
+  libraryVersion?: string;
+  /**
+   * A unique signature that can be used to identify this external API usage.
+   *
+   * The signature contains the package name, type name, method name, and method parameters
+   * in the form "packageName.typeName#methodName(methodParameters)".
+   * e.g. `org.sql2o.Connection#createQuery(String)`
    */
   signature: string;
   packageName: string;
   typeName: string;
   methodName: string;
+  /**
+   * The method parameters, including enclosing parentheses, e.g. `(String, String)`
+   */
   methodParameters: string;
+}
+
+export interface ExternalApiUsage extends MethodSignature {
+  /**
+   * Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
+   */
+  library: string;
+  /**
+   * Is this method already supported by CodeQL standard libraries.
+   * If so, there is no need for the user to model it themselves.
+   */
   supported: boolean;
-  usages: Call[];
-};
+  supportedType: ModeledMethodType;
+  usages: Usage[];
+}

extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts

@@ -3,23 +3,19 @@ import { DatabaseItem } from "../databases/local-databases";
 import { basename } from "path";
 import { QueryRunner } from "../query-server";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { TeeLogger } from "../common";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
 import { extensiblePredicateDefinitions } from "./predicates";
 import { ProgressCallback } from "../common/vscode/progress";
-import {
-  getOnDiskWorkspaceFolders,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import {
-  ModeledMethodType,
-  ModeledMethodWithSignature,
-} from "./modeled-method";
-import { redactableError } from "../pure/errors";
-import { QueryResultType } from "../pure/new-messages";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+import { redactableError } from "../common/errors";
+import { QueryResultType } from "../query-server/new-messages";
 import { file } from "tmp-promise";
 import { writeFile } from "fs-extra";
 import { dump } from "js-yaml";
-import { qlpackOfDatabase } from "../language-support";
+import { qlpackOfDatabase } from "../local-queries";
+import { telemetryListener } from "../common/vscode/telemetry";
 
 type FlowModelOptions = {
   cliServer: CodeQLCliServer;
@@ -28,7 +24,7 @@ type FlowModelOptions = {
   databaseItem: DatabaseItem;
   progress: ProgressCallback;
   token: CancellationToken;
-  onResults: (results: ModeledMethodWithSignature[]) => void | Promise<void>;
+  onResults: (results: ModeledMethod[]) => void | Promise<void>;
 };
 
 async function resolveQueries(
@@ -80,9 +76,11 @@ async function getModeledMethodsFromFlow(
     progress,
     token,
   }: Omit<FlowModelOptions, "onResults">,
-): Promise<ModeledMethodWithSignature[]> {
+): Promise<ModeledMethod[]> {
   if (queryPath === undefined) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Failed to find ${type} query`,
     );
     return [];
@@ -117,6 +115,8 @@ async function getModeledMethodsFromFlow(
   );
   if (queryResult.resultType !== QueryResultType.SUCCESS) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Failed to run ${basename(queryPath)} query: ${
         queryResult.message ?? "No message"
       }`,
@@ -129,6 +129,8 @@ async function getModeledMethodsFromFlow(
   const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
   if (bqrsInfo["result-sets"].length !== 1) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Expected exactly one result set, got ${
         bqrsInfo["result-sets"].length
       } for ${basename(queryPath)}`,

extensions/ql-vscode/src/data-extensions-editor/library.ts

@@ -0,0 +1,58 @@
+import { basename, extname } from "../common/path";
+
+// From the semver package using
+// const { re, t } = require("semver/internal/re");
+// console.log(re[t.LOOSE]);
+// Modifications:
+// - Added version named group which does not capture the v prefix
+// - Removed the ^ and $ anchors
+// - Made the minor and patch versions optional
+// - Added a hyphen to the start of the version
+// - Added a dot as a valid separator between the version and the label
+// - Made the patch version optional even if a label is given
+// This will match any semver string at the end of a larger string
+const semverRegex =
+  /-[v=\s]*(?<version>([0-9]+)(\.([0-9]+)(?:(\.([0-9]+))?(?:[-.]?((?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*)(?:\.(?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?)?)?)/g;
+
+interface Library {
+  name: string;
+  version?: string;
+}
+
+export function parseLibraryFilename(filename: string): Library {
+  let libraryName = basename(filename);
+  const extension = extname(libraryName);
+  libraryName = libraryName.slice(0, -extension.length);
+
+  let libraryVersion: string | undefined;
+
+  let match: RegExpMatchArray | null = null;
+
+  // Reset the regex
+  semverRegex.lastIndex = 0;
+
+  // Find the last occurence of the regex within the library name
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const currentMatch = semverRegex.exec(libraryName);
+    if (currentMatch === null) {
+      break;
+    }
+
+    match = currentMatch;
+  }
+
+  if (match?.groups) {
+    libraryVersion = match.groups?.version;
+    // Remove everything after the start of the match
+    libraryName = libraryName.slice(0, match.index);
+  }
+
+  // Remove any leading or trailing hyphens or dots
+  libraryName = libraryName.replaceAll(/^[.-]+|[.-]+$/g, "");
+
+  return {
+    name: libraryName,
+    version: libraryVersion,
+  };
+}

extensions/ql-vscode/src/data-extensions-editor/model-details/model-details-data-provider.ts

@@ -0,0 +1,132 @@
+import {
+  Event,
+  EventEmitter,
+  ThemeColor,
+  ThemeIcon,
+  TreeDataProvider,
+  TreeItem,
+  TreeItemCollapsibleState,
+  Uri,
+} from "vscode";
+import { DisposableObject } from "../../common/disposable-object";
+import { ExternalApiUsage, Usage } from "../external-api-usage";
+import { DatabaseItem } from "../../databases/local-databases";
+import { relative } from "path";
+import { CodeQLCliServer } from "../../codeql-cli/cli";
+
+export class ModelDetailsDataProvider
+  extends DisposableObject
+  implements TreeDataProvider<ModelDetailsTreeViewItem>
+{
+  private externalApiUsages: ExternalApiUsage[] = [];
+  private databaseItem: DatabaseItem | undefined = undefined;
+  private sourceLocationPrefix: string | undefined = undefined;
+
+  private readonly onDidChangeTreeDataEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+
+  public constructor(private readonly cliServer: CodeQLCliServer) {
+    super();
+  }
+
+  public get onDidChangeTreeData(): Event<void> {
+    return this.onDidChangeTreeDataEmitter.event;
+  }
+
+  public async setState(
+    externalApiUsages: ExternalApiUsage[],
+    databaseItem: DatabaseItem,
+  ): Promise<void> {
+    this.externalApiUsages = externalApiUsages;
+    this.databaseItem = databaseItem;
+    this.sourceLocationPrefix = await this.databaseItem.getSourceLocationPrefix(
+      this.cliServer,
+    );
+    this.onDidChangeTreeDataEmitter.fire();
+  }
+
+  getTreeItem(item: ModelDetailsTreeViewItem): TreeItem {
+    if (isExternalApiUsage(item)) {
+      return {
+        label: `${item.packageName}.${item.typeName}.${item.methodName}${item.methodParameters}`,
+        collapsibleState: TreeItemCollapsibleState.Collapsed,
+        iconPath: new ThemeIcon("symbol-method"),
+      };
+    } else {
+      return {
+        label: item.label,
+        description: `${this.relativePathWithinDatabase(item.url.uri)} [${
+          item.url.startLine
+        }, ${item.url.endLine}]`,
+        collapsibleState: TreeItemCollapsibleState.None,
+        command: {
+          title: "Show usage",
+          command: "codeQLDataExtensionsEditor.jumpToUsageLocation",
+          arguments: [item, this.databaseItem],
+        },
+        iconPath: new ThemeIcon("error", new ThemeColor("errorForeground")),
+      };
+    }
+  }
+
+  private relativePathWithinDatabase(uri: string): string {
+    const parsedUri = Uri.parse(uri);
+    if (this.sourceLocationPrefix) {
+      return relative(this.sourceLocationPrefix, parsedUri.fsPath);
+    } else {
+      return parsedUri.fsPath;
+    }
+  }
+
+  getChildren(item?: ModelDetailsTreeViewItem): ModelDetailsTreeViewItem[] {
+    if (item === undefined) {
+      return this.externalApiUsages;
+    } else if (isExternalApiUsage(item)) {
+      return item.usages;
+    } else {
+      return [];
+    }
+  }
+
+  getParent(
+    item: ModelDetailsTreeViewItem,
+  ): ModelDetailsTreeViewItem | undefined {
+    if (isExternalApiUsage(item)) {
+      return undefined;
+    } else {
+      return this.externalApiUsages.find((e) => e.usages.includes(item));
+    }
+  }
+
+  public resolveCanonicalUsage(usage: Usage): Usage | undefined {
+    for (const externalApiUsage of this.externalApiUsages) {
+      for (const u of externalApiUsage.usages) {
+        if (usagesAreEqual(u, usage)) {
+          return u;
+        }
+      }
+    }
+    return undefined;
+  }
+}
+
+export type ModelDetailsTreeViewItem = ExternalApiUsage | Usage;
+
+function isExternalApiUsage(
+  item: ModelDetailsTreeViewItem,
+): item is ExternalApiUsage {
+  return (item as any).usages !== undefined;
+}
+
+function usagesAreEqual(u1: Usage, u2: Usage): boolean {
+  return (
+    u1.label === u2.label &&
+    u1.classification === u2.classification &&
+    u1.url.uri === u2.url.uri &&
+    u1.url.startLine === u2.url.startLine &&
+    u1.url.startColumn === u2.url.startColumn &&
+    u1.url.endLine === u2.url.endLine &&
+    u1.url.endColumn === u2.url.endColumn
+  );
+}

extensions/ql-vscode/src/data-extensions-editor/model-details/model-details-panel.ts

@@ -0,0 +1,43 @@
+import { TreeView, window } from "vscode";
+import { DisposableObject } from "../../common/disposable-object";
+import {
+  ModelDetailsDataProvider,
+  ModelDetailsTreeViewItem,
+} from "./model-details-data-provider";
+import { ExternalApiUsage, Usage } from "../external-api-usage";
+import { DatabaseItem } from "../../databases/local-databases";
+import { CodeQLCliServer } from "../../codeql-cli/cli";
+
+export class ModelDetailsPanel extends DisposableObject {
+  private readonly dataProvider: ModelDetailsDataProvider;
+  private readonly treeView: TreeView<ModelDetailsTreeViewItem>;
+
+  public constructor(cliServer: CodeQLCliServer) {
+    super();
+
+    this.dataProvider = new ModelDetailsDataProvider(cliServer);
+
+    this.treeView = window.createTreeView("codeQLModelDetails", {
+      treeDataProvider: this.dataProvider,
+    });
+    this.push(this.treeView);
+  }
+
+  public async setState(
+    externalApiUsages: ExternalApiUsage[],
+    databaseItem: DatabaseItem,
+  ): Promise<void> {
+    await this.dataProvider.setState(externalApiUsages, databaseItem);
+    this.treeView.badge = {
+      value: externalApiUsages.length,
+      tooltip: "Number of external APIs",
+    };
+  }
+
+  public async revealItem(usage: Usage): Promise<void> {
+    const canonicalUsage = this.dataProvider.resolveCanonicalUsage(usage);
+    if (canonicalUsage !== undefined) {
+      await this.treeView.reveal(canonicalUsage);
+    }
+  }
+}

extensions/ql-vscode/src/data-extensions-editor/modeled-method-fs.ts

@@ -0,0 +1,118 @@
+import { outputFile, readFile } from "fs-extra";
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod } from "./modeled-method";
+import { Mode } from "./shared/mode";
+import { createDataExtensionYamls, loadDataExtensionYaml } from "./yaml";
+import { join, relative } from "path";
+import { ExtensionPack } from "./shared/extension-pack";
+import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { load as loadYaml } from "js-yaml";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { pathsEqual } from "../common/files";
+
+export async function saveModeledMethods(
+  extensionPack: ExtensionPack,
+  databaseName: string,
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+  mode: Mode,
+  cliServer: CodeQLCliServer,
+  logger: NotificationLogger,
+): Promise<void> {
+  const existingModeledMethods = await loadModeledMethodFiles(
+    extensionPack,
+    cliServer,
+    logger,
+  );
+
+  const yamls = createDataExtensionYamls(
+    databaseName,
+    language,
+    externalApiUsages,
+    modeledMethods,
+    existingModeledMethods,
+    mode,
+  );
+
+  for (const [filename, yaml] of Object.entries(yamls)) {
+    await outputFile(join(extensionPack.path, filename), yaml);
+  }
+
+  void logger.log(`Saved data extension YAML`);
+}
+
+async function loadModeledMethodFiles(
+  extensionPack: ExtensionPack,
+  cliServer: CodeQLCliServer,
+  logger: NotificationLogger,
+): Promise<Record<string, Record<string, ModeledMethod>>> {
+  const modelFiles = await listModelFiles(extensionPack.path, cliServer);
+
+  const modeledMethodsByFile: Record<
+    string,
+    Record<string, ModeledMethod>
+  > = {};
+
+  for (const modelFile of modelFiles) {
+    const yaml = await readFile(join(extensionPack.path, modelFile), "utf8");
+
+    const data = loadYaml(yaml, {
+      filename: modelFile,
+    });
+
+    const modeledMethods = loadDataExtensionYaml(data);
+    if (!modeledMethods) {
+      void showAndLogErrorMessage(
+        logger,
+        `Failed to parse data extension YAML ${modelFile}.`,
+      );
+      continue;
+    }
+    modeledMethodsByFile[modelFile] = modeledMethods;
+  }
+
+  return modeledMethodsByFile;
+}
+
+export async function loadModeledMethods(
+  extensionPack: ExtensionPack,
+  cliServer: CodeQLCliServer,
+  logger: NotificationLogger,
+): Promise<Record<string, ModeledMethod>> {
+  const existingModeledMethods: Record<string, ModeledMethod> = {};
+
+  const modeledMethodsByFile = await loadModeledMethodFiles(
+    extensionPack,
+    cliServer,
+    logger,
+  );
+  for (const modeledMethods of Object.values(modeledMethodsByFile)) {
+    for (const [key, value] of Object.entries(modeledMethods)) {
+      existingModeledMethods[key] = value;
+    }
+  }
+
+  return existingModeledMethods;
+}
+
+export async function listModelFiles(
+  extensionPackPath: string,
+  cliServer: CodeQLCliServer,
+): Promise<Set<string>> {
+  const result = await cliServer.resolveExtensions(
+    extensionPackPath,
+    getOnDiskWorkspaceFolders(),
+  );
+
+  const modelFiles = new Set<string>();
+  for (const [path, extensions] of Object.entries(result.data)) {
+    if (pathsEqual(path, extensionPackPath)) {
+      for (const extension of extensions) {
+        modelFiles.add(relative(extensionPackPath, extension.file));
+      }
+    }
+  }
+  return modelFiles;
+}

extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts

@@ -1,3 +1,5 @@
+import { MethodSignature } from "./external-api-usage";
+
 export type ModeledMethodType =
   | "none"
   | "source"
@@ -17,15 +19,10 @@ export type Provenance =
   // Entered by the user in the editor manually
   | "manual";
 
-export type ModeledMethod = {
+export interface ModeledMethod extends MethodSignature {
   type: ModeledMethodType;
   input: string;
   output: string;
   kind: string;
   provenance: Provenance;
-};
-
-export type ModeledMethodWithSignature = {
-  signature: string;
-  modeledMethod: ModeledMethod;
-};
+}

extensions/ql-vscode/src/data-extensions-editor/predicates.ts

@@ -1,20 +1,9 @@
-import { ExternalApiUsage } from "./external-api-usage";
-import {
-  ModeledMethod,
-  ModeledMethodType,
-  ModeledMethodWithSignature,
-  Provenance,
-} from "./modeled-method";
-
-export type ExternalApiUsageByType = {
-  externalApiUsage: ExternalApiUsage;
-  modeledMethod: ModeledMethod;
-};
+import { ModeledMethod, ModeledMethodType, Provenance } from "./modeled-method";
 
 export type ExtensiblePredicateDefinition = {
   extensiblePredicate: string;
-  generateMethodDefinition: (method: ExternalApiUsageByType) => Tuple[];
-  readModeledMethod: (row: Tuple[]) => ModeledMethodWithSignature;
+  generateMethodDefinition: (method: ModeledMethod) => Tuple[];
+  readModeledMethod: (row: Tuple[]) => ModeledMethod;
 
   supportedKinds?: string[];
 };
@@ -36,27 +25,29 @@ export const extensiblePredicateDefinitions: Record<
     //   string output, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
+      method.packageName,
+      method.typeName,
       true,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
+      method.methodName,
+      method.methodParameters,
       "",
-      method.modeledMethod.output,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.output,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "source",
+      input: "",
+      output: row[6] as string,
+      kind: row[7] as string,
+      provenance: row[8] as Provenance,
       signature: readRowToMethod(row),
-      modeledMethod: {
-        type: "source",
-        input: "",
-        output: row[6] as string,
-        kind: row[7] as string,
-        provenance: row[8] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[3] as string,
+      methodParameters: row[4] as string,
     }),
-    supportedKinds: ["remote"],
+    supportedKinds: ["local", "remote"],
   },
   sink: {
     extensiblePredicate: "sinkModel",
@@ -65,27 +56,41 @@ export const extensiblePredicateDefinitions: Record<
     //   string input, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
+      method.packageName,
+      method.typeName,
       true,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
+      method.methodName,
+      method.methodParameters,
       "",
-      method.modeledMethod.input,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.input,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "sink",
+      input: row[6] as string,
+      output: "",
+      kind: row[7] as string,
+      provenance: row[8] as Provenance,
       signature: readRowToMethod(row),
-      modeledMethod: {
-        type: "sink",
-        input: row[6] as string,
-        output: "",
-        kind: row[7] as string,
-        provenance: row[8] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[3] as string,
+      methodParameters: row[4] as string,
     }),
-    supportedKinds: ["sql", "xss", "logging"],
+    supportedKinds: [
+      "code-injection",
+      "command-injection",
+      "file-content-store",
+      "html-injection",
+      "js-injection",
+      "ldap-injection",
+      "log-injection",
+      "path-injection",
+      "request-forgery",
+      "sql-injection",
+      "url-redirection",
+    ],
   },
   summary: {
     extensiblePredicate: "summaryModel",
@@ -94,26 +99,28 @@ export const extensiblePredicateDefinitions: Record<
     //   string input, string output, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
+      method.packageName,
+      method.typeName,
       true,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
+      method.methodName,
+      method.methodParameters,
       "",
-      method.modeledMethod.input,
-      method.modeledMethod.output,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.input,
+      method.output,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "summary",
+      input: row[6] as string,
+      output: row[7] as string,
+      kind: row[8] as string,
+      provenance: row[9] as Provenance,
       signature: readRowToMethod(row),
-      modeledMethod: {
-        type: "summary",
-        input: row[6] as string,
-        output: row[7] as string,
-        kind: row[8] as string,
-        provenance: row[9] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[3] as string,
+      methodParameters: row[4] as string,
     }),
     supportedKinds: ["taint", "value"],
   },
@@ -123,22 +130,24 @@ export const extensiblePredicateDefinitions: Record<
     //   string package, string type, string name, string signature, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.packageName,
+      method.typeName,
+      method.methodName,
+      method.methodParameters,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "neutral",
+      input: "",
+      output: "",
+      kind: row[4] as string,
+      provenance: row[5] as Provenance,
       signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
-      modeledMethod: {
-        type: "neutral",
-        input: "",
-        output: "",
-        kind: row[4] as string,
-        provenance: row[5] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[2] as string,
+      methodParameters: row[3] as string,
     }),
     supportedKinds: ["summary", "source", "sink"],
   },

extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts

@@ -1,7 +1,7 @@
 import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
-  mainQuery: `/**
+  applicationModeQuery: `/**
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase.
  * @tags telemetry
@@ -9,27 +9,57 @@ export const fetchExternalApisQuery: Query = {
  * @id cs/telemetry/fetch-external-apis
  */
 
-import csharp
-import ExternalApi
+private import csharp
+private import AutomodelVsCode
+
+class ExternalApi extends CallableMethod {
+  ExternalApi() {
+    this.isUnboundDeclaration() and
+    this.fromLibrary() and
+    this.(Modifiable).isEffectivelyPublic()
+  }
+}
 
 private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
 
-private boolean isSupported(ExternalApi api) {
-  api.isSupported() and result = true
-  or
-  not api.isSupported() and
-  result = false
-}
-
-from ExternalApi api, string apiName, boolean supported, Call usage
+from
+  ExternalApi api, string apiName, boolean supported, Call usage, string type, string classification
 where
   apiName = api.getApiName() and
   supported = isSupported(api) and
-  usage = aUsage(api)
-select usage, apiName, supported.toString(), "supported"
+  usage = aUsage(api) and
+  type = supportedType(api) and
+  classification = methodClassification(usage)
+select usage, apiName, supported.toString(), "supported", api.dllName(), api.dllVersion(), type,
+  "type", classification, "classification"
+`,
+  frameworkModeQuery: `/**
+ * @name Public methods
+ * @description A list of APIs callable by consumers. Excludes test and generated code.
+ * @tags telemetry
+ * @kind problem
+ * @id cs/telemetry/fetch-public-methods
+ */
+
+private import csharp
+private import dotnet
+private import semmle.code.csharp.frameworks.Test
+private import AutomodelVsCode
+
+class PublicMethod extends CallableMethod {
+  PublicMethod() { this.fromSource() and not this.getFile() instanceof TestFile }
+}
+
+from PublicMethod publicMethod, string apiName, boolean supported, string type
+where
+  apiName = publicMethod.getApiName() and
+  supported = isSupported(publicMethod) and
+  type = supportedType(publicMethod)
+select publicMethod, apiName, supported.toString(), "supported",
+  publicMethod.getFile().getBaseName(), "library", type, "type", "unknown", "classification"
 `,
   dependencies: {
-    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+    "AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
 
 private import csharp
 private import dotnet
@@ -41,6 +71,7 @@ private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
 private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
 private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
+private import semmle.code.csharp.frameworks.Test
 private import semmle.code.csharp.security.dataflow.flowsources.Remote
 
 pragma[nomagic]
@@ -59,22 +90,31 @@ class TestLibrary extends RefType {
 }
 
 /** Holds if the given callable is not worth supporting. */
-private predicate isUninteresting(DotNet::Callable c) {
+private predicate isUninteresting(DotNet::Declaration c) {
   c.getDeclaringType() instanceof TestLibrary or
-  c.(Constructor).isParameterless()
+  c.(Constructor).isParameterless() or
+  c.getDeclaringType() instanceof AnonymousClass
 }
 
 /**
- * An external API from either the C# Standard Library or a 3rd party library.
+ * An callable method from either the C# Standard Library, a 3rd party library, or from the source.
  */
-class ExternalApi extends DotNet::Callable {
-  ExternalApi() {
-    this.isUnboundDeclaration() and
-    this.fromLibrary() and
+class CallableMethod extends DotNet::Declaration {
+  CallableMethod() {
     this.(Modifiable).isEffectivelyPublic() and
     not isUninteresting(this)
   }
 
+  /**
+   * Gets the unbound type, name and parameter types of this API.
+   */
+  bindingset[this]
+  private string getSignature() {
+    result =
+      nestedName(this.getDeclaringType().getUnboundDeclaration()) + "#" + this.getName() + "(" +
+        parameterQualifiedTypeNamesToString(this) + ")"
+  }
+
   /**
    * Gets the namespace of this API.
    */
@@ -85,8 +125,23 @@ class ExternalApi extends DotNet::Callable {
    * Gets the namespace and signature of this API.
    */
   bindingset[this]
-  string getApiName() { result = this.getNamespace() + "." + this.getDeclaringType().getUnboundDeclaration() + "#" + this.getName() + "(" +
-  parameterQualifiedTypeNamesToString(this) + ")" }
+  string getApiName() { result = this.getNamespace() + "." + this.getSignature() }
+
+  private string getDllName() { result = this.getLocation().(Assembly).getName() }
+
+  private string getDllVersion() { result = this.getLocation().(Assembly).getVersion().toString() }
+
+  string dllName() {
+    result = this.getDllName()
+    or
+    not exists(this.getDllName()) and result = this.getFile().getBaseName()
+  }
+
+  string dllVersion() {
+    result = this.getDllVersion()
+    or
+    not exists(this.getDllVersion()) and result = ""
+  }
 
   /** Gets a node that is an input to a call to this API. */
   private ArgumentNode getAnInput() {
@@ -140,47 +195,45 @@ class ExternalApi extends DotNet::Callable {
   }
 }
 
-/**
- * Gets the limit for the number of results produced by a telemetry query.
- */
-int resultLimit() { result = 1000 }
+boolean isSupported(CallableMethod callableMethod) {
+  callableMethod.isSupported() and result = true
+  or
+  not callableMethod.isSupported() and
+  result = false
+}
+
+string supportedType(CallableMethod method) {
+  method.isSink() and result = "sink"
+  or
+  method.isSource() and result = "source"
+  or
+  method.hasSummary() and result = "summary"
+  or
+  method.isNeutral() and result = "neutral"
+  or
+  not method.isSupported() and result = ""
+}
+
+string methodClassification(Call method) {
+  method.getFile() instanceof TestFile and result = "test"
+  or
+  not method.getFile() instanceof TestFile and
+  result = "source"
+}
 
 /**
- * Holds if it is relevant to count usages of "api".
+ * Gets the nested name of the declaration.
+ *
+ * If the declaration is not a nested type, the result is the same as \`getName()\`.
+ * Otherwise the name of the nested type is prefixed with a \`+\` and appended to
+ * the name of the enclosing type, which might be a nested type as well.
  */
-signature predicate relevantApi(ExternalApi api);
-
-/**
- * Given a predicate to count relevant API usages, this module provides a predicate
- * for restricting the number or returned results based on a certain limit.
- */
-module Results<relevantApi/1 getRelevantUsages> {
-  private int getUsages(string apiName) {
-    result =
-      strictcount(Call c, ExternalApi api |
-        c.getTarget().getUnboundDeclaration() = api and
-        apiName = api.getApiName() and
-        getRelevantUsages(api)
-      )
-  }
-
-  private int getOrder(string apiName) {
-    apiName =
-      rank[result](string name, int usages |
-        usages = getUsages(name)
-      |
-        name order by usages desc, name
-      )
-  }
-
-  /**
-   * Holds if there exists an API with "apiName" that is being used "usages" times
-   * and if it is in the top results (guarded by resultLimit).
-   */
-  predicate restrict(string apiName, int usages) {
-    usages = getUsages(apiName) and
-    getOrder(apiName) <= resultLimit()
-  }
+private string nestedName(Declaration declaration) {
+  not exists(declaration.getDeclaringType().getUnboundDeclaration()) and
+  result = declaration.getName()
+  or
+  nestedName(declaration.getDeclaringType().getUnboundDeclaration()) + "+" + declaration.getName() =
+    result
 }
 `,
   },