Merge pull request #2662 from github/v1.8.9

Release prep for v1.8.9

.github/codeql/queries/assert-no-vscode-dependency.ql

@@ -0,0 +1,37 @@
+/**
+ * @name Unwanted dependency on vscode API
+ * @kind path-problem
+ * @problem.severity error
+ * @id vscode-codeql/assert-no-vscode-dependency
+ * @description The modules stored under `common` should not have dependencies on the VS Code API
+ */
+
+import javascript
+
+class VSCodeImport extends ImportDeclaration {
+  VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
+}
+
+class CommonFile extends File {
+  CommonFile() {
+    this.getRelativePath().regexpMatch(".*/src/common/.*") and
+    not this.getRelativePath().regexpMatch(".*/vscode/.*")
+  }
+}
+
+Import getANonTypeOnlyImport(Module m) {
+  result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly()
+}
+
+query predicate edges(AstNode a, AstNode b) {
+  getANonTypeOnlyImport(a) = b or
+  a.(Import).getImportedModule() = b
+}
+
+from Module m, VSCodeImport v
+where
+  m.getFile() instanceof CommonFile and
+  edges+(m, v)
+select m, m, v,
+  "This module is in the 'common' directory but has a transitive dependency on the vscode API imported $@",
+  v, "here"

.github/codeql/queries/assert-pure.ql

@@ -1,21 +0,0 @@
-/**
- * @name Unwanted dependency on vscode API
- * @kind problem
- * @problem.severity error
- * @id vscode-codeql/assert-pure
- * @description The modules stored under `pure` and tested in the `pure-tests`
- * are intended to be "pure".
- */
-import javascript
-
-class VSCodeImport extends ASTNode {
-  VSCodeImport() {
-    this.(Import).getImportedPath().getValue() = "vscode"
-  }
-}
-
-from Module m, VSCodeImport v
-where
-  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
-  m.getAnImportedModule*().getAnImport() = v
-select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/qlpack.yml

@@ -1,3 +1,4 @@
 name: vscode-codeql-custom-queries-javascript
 version: 0.0.0
-libraryPathDependencies: codeql-javascript
+dependencies:
+  codeql/javascript-queries: "*"

.github/codeql/queries/unique-command-use.ql

@@ -15,134 +15,145 @@
  * that should be changed to fix the alert.
  */
 
- import javascript
+import javascript
 
- /**
-  * The name of a VS Code command.
-  */
- class CommandName extends string {
-   CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
- 
-   /**
-    * In how many ways is this command used. Will always be at least 1.
-    */
-   int getNumberOfUsages() { result = count(this.getAUse()) }
+/**
+ * The name of a VS Code command.
+ */
+class CommandName extends string {
+  CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
 
-   /**
-    * Get a usage of this command.
-    */
-   CommandUsage getAUse() { result.getCommandName() = this }
- 
-   /**
-    * Get the canonical first usage of this command, to use for the location
-    * of the alert. The implementation of this ordering of usages is arbitrary
-    * and the usage given may not be the one that should be changed when fixing
-    * the alert.
-    */
-   CommandUsage getFirstUsage() {
-     result =
-       max(CommandUsage use |
-         use = this.getAUse()
-       |
-         use
-         order by
-           use.getFile().getRelativePath(), use.getLocation().getStartLine(),
-           use.getLocation().getStartColumn()
-       )
-   }
- }
- 
- /**
-  * Represents a single usage of a command, either from within code or
-  * from the command's definition in package.json
-  */
- abstract class CommandUsage extends Locatable {
-   abstract string getCommandName();
- }
- 
- /**
-  * A usage of a command from the typescript code, by calling `executeCommand`.
-  */
- class CommandUsageCallExpr extends CommandUsage, CallExpr {
-   CommandUsageCallExpr() {
-     this.getCalleeName() = "executeCommand" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
- 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
+  /**
+   * In how many ways is this command used. Will always be at least 1.
+   */
+  int getNumberOfUsages() { result = count(this.getAUse()) }
 
- /**
-  * A usage of a command from the typescript code, by calling `CommandManager.execute`.
-  */
- class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
-   CommandUsageCommandManagerMethodCallExpr() {
-     this.getCalleeName() = "execute" and
-     this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
+  /**
+   * Get a usage of this command.
+   */
+  CommandUsage getAUse() { result.getCommandName() = this }
 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
- 
- /**
-  * A usage of a command from any menu that isn't the command palette.
-  * This means a user could invoke the command by clicking on a button in
-  * something like a menu or a dropdown.
-  */
- class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonMenuItem() {
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson, string menuName |
-       packageJson
-           .getPropValue("contributes")
-           .getPropValue("menus")
-           .getPropValue(menuName)
-           .getElementValue(_) = this and
-       menuName != "commandPalette"
-     )
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- /**
-  * Is the given command disabled for use in the command palette by
-  * a block with a `"when": "false"` field.
-  */
- predicate isDisabledInCommandPalette(string commandName) {
-   exists(PackageJson packageJson, JsonObject commandPaletteObject |
-     packageJson
-         .getPropValue("contributes")
-         .getPropValue("menus")
-         .getPropValue("commandPalette")
-         .getElementValue(_) = commandPaletteObject and
-     commandPaletteObject.getPropValue("command").getStringValue() = commandName and
-     commandPaletteObject.getPropValue("when").getStringValue() = "false"
-   )
- }
- 
- /**
-  * Represents a command being usable from the command palette.
-  * This means that a user could choose to manually invoke the command.
-  */
- class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonCommandPalette() {
-     this.getFile().getBaseName() = "package.json" and
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson |
-       packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
-     ) and
-     not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- from CommandName c
- where c.getNumberOfUsages() > 1
- select c.getFirstUsage(),
-   "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"
- 
+  /**
+   * Get the canonical first usage of this command, to use for the location
+   * of the alert. The implementation of this ordering of usages is arbitrary
+   * and the usage given may not be the one that should be changed when fixing
+   * the alert.
+   */
+  CommandUsage getFirstUsage() {
+    result =
+      max(CommandUsage use |
+        use = this.getAUse()
+      |
+        use
+        order by
+          use.getFile().getRelativePath(), use.getLocation().getStartLine(),
+          use.getLocation().getStartColumn()
+      )
+  }
+}
+
+/**
+ * Matches one of the members of `BuiltInVsCodeCommands` from `extensions/ql-vscode/src/common/commands.ts`.
+ */
+class BuiltInVSCodeCommand extends string {
+  BuiltInVSCodeCommand() {
+    exists(TypeAliasDeclaration tad |
+      tad.getIdentifier().getName() = "BuiltInVsCodeCommands" and
+      tad.getDefinition().(InterfaceTypeExpr).getAMember().getName() = this
+    )
+  }
+}
+
+/**
+ * Represents a single usage of a command, either from within code or
+ * from the command's definition in package.json
+ */
+abstract class CommandUsage extends Locatable {
+  abstract string getCommandName();
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `executeCommand`.
+ */
+class CommandUsageCallExpr extends CommandUsage, CallExpr {
+  CommandUsageCallExpr() {
+    this.getCalleeName() = "executeCommand" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `CommandManager.execute`.
+ */
+class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
+  CommandUsageCommandManagerMethodCallExpr() {
+    this.getCalleeName() = "execute" and
+    this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from any menu that isn't the command palette.
+ * This means a user could invoke the command by clicking on a button in
+ * something like a menu or a dropdown.
+ */
+class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonMenuItem() {
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson, string menuName |
+      packageJson
+          .getPropValue("contributes")
+          .getPropValue("menus")
+          .getPropValue(menuName)
+          .getElementValue(_) = this and
+      menuName != "commandPalette"
+    )
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+/**
+ * Is the given command disabled for use in the command palette by
+ * a block with a `"when": "false"` field.
+ */
+predicate isDisabledInCommandPalette(string commandName) {
+  exists(PackageJson packageJson, JsonObject commandPaletteObject |
+    packageJson
+        .getPropValue("contributes")
+        .getPropValue("menus")
+        .getPropValue("commandPalette")
+        .getElementValue(_) = commandPaletteObject and
+    commandPaletteObject.getPropValue("command").getStringValue() = commandName and
+    commandPaletteObject.getPropValue("when").getStringValue() = "false"
+  )
+}
+
+/**
+ * Represents a command being usable from the command palette.
+ * This means that a user could choose to manually invoke the command.
+ */
+class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonCommandPalette() {
+    this.getFile().getBaseName() = "package.json" and
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson |
+      packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
+    ) and
+    not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+from CommandName c
+where c.getNumberOfUsages() > 1 and not c instanceof BuiltInVSCodeCommand
+select c.getFirstUsage(),
+  "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -80,7 +80,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -110,6 +110,11 @@ jobs:
         run: |
           npm run lint:scenarios
 
+      - name: Find deadcode
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run find-deadcode
+
   unit-test:
     name: Unit Test
     runs-on: ${{ matrix.os }}
@@ -124,7 +129,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -158,7 +163,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -209,6 +214,7 @@ jobs:
     name: CLI Test
     runs-on: ${{ matrix.os }}
     needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
@@ -225,7 +231,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
 
       - name: Install dependencies
         run: |

CODEOWNERS

@@ -2,3 +2,4 @@
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
 **/data-extensions-editor/ @github/code-scanning-secexp-reviewers
+**/queries-panel/ @github/code-scanning-secexp-reviewers

docs/releasing.md

@@ -10,7 +10,7 @@
       * New telemetry events are added.
       * Deprecation or removal of commands.
       * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
+1. Double-check that the node version we're using matches the one used for VS Code. You can find this info by seleting "About Visual Studio Code" from the top menu. If it doesn't match, you will then need to update the node version in the following files:
     * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
     * `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
     * `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting

docs/test-plan.md

@@ -11,16 +11,67 @@ We don't need to test features (and permutations of features) that are covered b
 
 ## Before releasing the VS Code extension
 
-- Go through the required test cases listed below
+- Run at least one local query and MRVA using the existing version of the extension (to generate "old" query history items).
+- Go through the required test cases listed below.
 - Check major PRs since the previous release for specific one-off things to test. Based on that, you might want to
 choose to go through some of the Optional Test Cases.
-- Run a query using the existing version of the extension (to generate an "old" query history item)
 
 ## Required Test Cases
 
-### Test Case 1: MRVA - Running a problem path query and viewing results
+### Local databases
 
-1. Open the [UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+#### Test case 1: Download a database from GitHub
+
+1. Click "Download Database from GitHub" and enter `angular-cn/ng-nice` and select the javascript language if prompted
+
+#### Test case 2: Import a database from an archive
+
+1. Obtain a javascript database for `babel/babel`
+   - You can do `gh api "/repos/babel/babel/code-scanning/codeql/databases/javascript" -H "Accept: application/zip" > babel.zip` to fetch a database from GitHub.
+2. Click "Choose Database from Archive" and select the file you just downloaded above.
+
+### Local queries
+
+#### Test case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+2. Select the `angular-cn/ng-nice` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that the paths can be opened and paths are rendered correctly
+   - Check that alert locations can be clicked on
+
+#### Test case 2: Running a problem query and viewing results
+
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
+2. Select the `babel/babel` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that alert locations can be clicked on
+
+#### Test case 3: Running a non-problem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+2. Select the `google/brotli` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the results table is rendered
+   - Check that result locations can be clicked on
+
+#### Test case 3: Can use AST viewer
+
+1. Click on any code location from a previous query to open a source file from a database
+2. Open the AST viewing panel and click "View AST"
+3. Once the AST is computed:
+   - Check that it can be navigated
+
+### MRVA
+
+#### Test Case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
 2. Run a MRVA against the following repo list:
 
    ```json
@@ -47,40 +98,44 @@ choose to go through some of the Optional Test Cases.
 6. Once the query completes:
    - Check that the query history item is updated to show the query status as "complete"
 
-### Test Case 2: MRVA - Running a problem query and viewing results
+#### Test Case 2: Running a problem query and viewing results
 
-1. Open the [ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
 2. Run a MRVA against the "Top 10" repositories.
-3. Check that there is a notification message.
+3. Check that a notification message pops up and the results view is opened.
 4. Check the query history. It should:
    - Show that an item has been added to the query history
    - The item should be marked as "in progress".
-5. Once the query starts:
-   - Check that a notification is shown with a link to the results view
+5. Once the query completes:
    - Check that the results are rendered with an alert message and a highlighted code snippet:
-![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-### Test Case 3: MRVA - Running a non-problem query and viewing results
+   ![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-1. Open the [FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+#### Test Case 3: Running a non-problem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
 2. Run a MRVA against a single repository (e.g. `google/brotli`).
-3. Once the query starts:
-   - Open the query results
+3. Check that a notification message pops up and the results view is opened.
+4. Check the query history. It should:
+   - Show that an item has been added to the query history
+   - The item should be marked as "in progress".
+5. Once the query completes:
    - Check that the results show up in a table:
-![results-table](images/results-table.png)
 
-### Test Case 4: MRVA - Interacting with query history
+   ![results-table](images/results-table.png)
+
+#### Test Case 4: Interacting with query history
 
 1. Click a history item (for MRVA):
    - Check that exporting results works
    - Check that sorting results works
    - Check that copying repo lists works
-2. Open the query results directory:
+2. Click "Open Results Directory":
    - Check that the correct directory is opened and there are results in it
-3. View logs
+3. Click "View Logs":
    - Check that the correct workflow is opened
 
-### Test Case 5: MRVA - Canceling a variant analysis run
+#### Test Case 5: Canceling a variant analysis run
 
 Run one of the above MRVAs, but cancel it from within VS Code:
 
@@ -88,9 +143,11 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
-### Test Case 6: MRVA - Change to a different colour theme
+### General
 
-Open one of the above MRVAs, try changing to a different colour theme and check that everything looks sensible.
+#### Test case 1: Change to a different colour theme
+
+Open at least one of the above MRVAs and at least one local query, then try changing to a different colour theme and check that everything looks sensible.
 Are there any components that are not showing up?
 
 ## Optional Test Cases
@@ -258,10 +315,9 @@ This requires running a MRVA query and seeing the results view.
    2. When the file does not exist
 7. Can open query text
 8. Can sort repos
-   1. By name
-   2. By results
-   3. By stars
-   4. By last updated
+   1. Alphabetically
+   2. By number of results
+   3. By popularity
 9. Can filter repos
 10. Shows correct statistics
     1. Total number of results
@@ -321,6 +377,7 @@ This requires running a MRVA query and seeing the results view.
 1. Make changes via config file (ensure JSON schema is helping out)
 1. Close and re-open VS Code (ensure lists are there)
 1. Collapse/expand tree nodes
+1. Create a new list, right click and select "Add repositories with GitHub Code Search". Enter the language 'python' and the query "UserMixin". This should show a rate limiting notification after a while but eventually populate the list with roughly 770 items.
 
 Error cases that trigger an error notification:
 

docs/testing.md

@@ -133,4 +133,4 @@ Once the scenario has been recorded, it's often useful to remove some of the req
 
 ### Scenario data location
 
-Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.
+Pre-recorded scenarios are stored in `./src/variant-analysis/github-api/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.

extensions/ql-vscode/.eslintrc.js

@@ -65,10 +65,6 @@ const baseConfig = {
     "import/no-namespace": "off",
     "import/no-unresolved": "off",
     "import/no-webpack-loader-syntax": "off",
-    "jsx-a11y/anchor-is-valid": "off",
-    "jsx-a11y/no-noninteractive-element-interactions": "off",
-    "jsx-a11y/no-static-element-interactions": "off",
-    "jsx-a11y/click-events-have-key-events": "off",
     "no-invalid-this": "off",
     "no-fallthrough": "off",
     "no-console": "off",

extensions/ql-vscode/.nvmrc

@@ -1 +1 @@
-v16.14.2
+v16.17.1

extensions/ql-vscode/.storybook/main.ts

@@ -1,4 +1,4 @@
-import type { StorybookConfig } from "@storybook/core-common";
+import type { StorybookConfig } from "@storybook/react-webpack5";
 
 const config: StorybookConfig = {
   stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
@@ -8,13 +8,13 @@ const config: StorybookConfig = {
     "@storybook/addon-interactions",
     "./vscode-theme-addon/preset.ts",
   ],
-  framework: "@storybook/react",
-  core: {
-    builder: "@storybook/builder-webpack5",
+  framework: {
+    name: "@storybook/react-webpack5",
+    options: {},
   },
-  features: {
-    babelModeV7: true,
+  docs: {
+    autodocs: "tag",
   },
 };
 
-module.exports = config;
+export default config;

extensions/ql-vscode/.storybook/manager.ts

@@ -1,4 +1,4 @@
-import { addons } from "@storybook/addons";
+import { addons } from "@storybook/manager-api";
 import { themes } from "@storybook/theming";
 
 addons.setConfig({

extensions/ql-vscode/.storybook/preview.ts

@@ -1,31 +1,36 @@
+import { Preview } from "@storybook/react";
 import { themes } from "@storybook/theming";
 import { action } from "@storybook/addon-actions";
 
 // Allow all stories/components to use Codicons
 import "@vscode/codicons/dist/codicon.css";
 
-// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
-export const parameters = {
-  // All props starting with `on` will automatically receive an action as a prop
-  actions: { argTypesRegex: "^on[A-Z].*" },
-  // All props matching these names will automatically get the correct control
-  controls: {
-    matchers: {
-      color: /(background|color)$/i,
-      date: /Date$/,
-    },
-  },
-  // Use a dark theme to be aligned with VSCode
-  docs: {
-    theme: themes.dark,
-  },
-  backgrounds: {
-    // The background is injected by our theme CSS files
-    disable: true,
-  },
-};
-
 (window as any).acquireVsCodeApi = () => ({
   postMessage: action("post-vscode-message"),
   setState: action("set-vscode-state"),
 });
+
+// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
+const preview: Preview = {
+  parameters: {
+    // All props starting with `on` will automatically receive an action as a prop
+    actions: { argTypesRegex: "^on[A-Z].*" },
+    // All props matching these names will automatically get the correct control
+    controls: {
+      matchers: {
+        color: /(background|color)$/i,
+        date: /Date$/,
+      },
+    },
+    // Use a dark theme to be aligned with VSCode
+    docs: {
+      theme: themes.dark,
+    },
+    backgrounds: {
+      // The background is injected by our theme CSS files
+      disable: true,
+    },
+  },
+};
+
+export default preview;

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -1,14 +1,12 @@
 import * as React from "react";
 import { FunctionComponent, useCallback } from "react";
 
-import { useGlobals } from "@storybook/api";
+import { useGlobals } from "@storybook/manager-api";
 import {
   IconButton,
   Icons,
-  WithTooltip,
   TooltipLinkList,
-  Link,
-  WithHideFn,
+  WithTooltip,
 } from "@storybook/components";
 
 import { themeNames, VSCodeTheme } from "./theme";
@@ -26,7 +24,7 @@ export const ThemeSelector: FunctionComponent = () => {
   );
 
   const createLinks = useCallback(
-    (onHide: () => void): Link[] =>
+    (onHide: () => void) =>
       Object.values(VSCodeTheme).map((theme) => ({
         id: theme,
         onClick() {
@@ -44,8 +42,8 @@ export const ThemeSelector: FunctionComponent = () => {
     <WithTooltip
       placement="top"
       trigger="click"
-      closeOnClick
-      tooltip={({ onHide }: WithHideFn) => (
+      closeOnOutsideClick
+      tooltip={({ onHide }: { onHide: () => void }) => (
         <TooltipLinkList links={createLinks(onHide)} />
       )}
     >

extensions/ql-vscode/.storybook/vscode-theme-addon/manager.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { addons, types } from "@storybook/addons";
+import { addons, types } from "@storybook/manager-api";
 import { ThemeSelector } from "./ThemeSelector";
 
 const ADDON_ID = "vscode-theme-addon";

extensions/ql-vscode/.storybook/vscode-theme-addon/preset.ts

@@ -1,4 +1,4 @@
-export function config(entry = []) {
+export function previewAnnotations(entry = []) {
   return [...entry, require.resolve("./preview.ts")];
 }
 

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -1,6 +1,5 @@
-import { useEffect, useGlobals } from "@storybook/addons";
+import { useEffect } from "react";
 import type {
-  AnyFramework,
   PartialStoryFn as StoryFunction,
   StoryContext,
 } from "@storybook/csf";
@@ -34,11 +33,8 @@ const themeFiles: { [key in VSCodeTheme]: string } = {
       .default,
 };
 
-export const withTheme = (
-  StoryFn: StoryFunction<AnyFramework>,
-  context: StoryContext<AnyFramework>,
-) => {
-  const [{ vscodeTheme }] = useGlobals();
+export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {
+  const { vscodeTheme } = context.globals;
 
   useEffect(() => {
     const styleSelectorId =

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,42 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.8.9 - 3 August 2023
+
+- Remove "last updated" information and sorting from variant analysis results view. [#2637](https://github.com/github/vscode-codeql/pull/2637)
+- Links to code on GitHub now include column numbers as well as line numbers. [#2406](https://github.com/github/vscode-codeql/pull/2406)
+- No longer highlight trailing commas for jump to definition.  [#2615](https://github.com/github/vscode-codeql/pull/2615)
+
+## 1.8.8 - 17 July 2023
+
+- Remove support for CodeQL CLI versions older than 2.9.4. [#2610](https://github.com/github/vscode-codeql/pull/2610)
+- Implement syntax highlighting for the `additional` and `default` keywords. [#2609](https://github.com/github/vscode-codeql/pull/2609)
+
+## 1.8.7 - 29 June 2023
+
+- Show a run button on the file tab for query files, that will start a local query. This button will only show when a local database is selected in the extension. [#2544](https://github.com/github/vscode-codeql/pull/2544)
+- Add a `CodeQL: Quick Evaluation Count` command to generate the count summary statistics of the results set
+  without spending the time to compute locations and strings. [#2475](https://github.com/github/vscode-codeql/pull/2475)
+
+## 1.8.6 - 14 June 2023
+
+- Add repositories to a variant analysis list with GitHub Code Search. [#2439](https://github.com/github/vscode-codeql/pull/2439) and [#2476](https://github.com/github/vscode-codeql/pull/2476)
+
+## 1.8.5 - 6 June 2023
+
+- Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Fix bug to ensure error messages have complete stack trace in message logs. [#2425](https://github.com/github/vscode-codeql/pull/2425)
+- Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
+- Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
+- Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
+- Retry variant analysis results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
+
+## 1.8.4 - 3 May 2023
+
+- Avoid repeated error messages when unable to monitor a variant analysis. [#2396](https://github.com/github/vscode-codeql/pull/2396)
+- Fix bug where a variant analysis didn't display the `#select` results set correctly when the [query metadata](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#query-metadata) didn't exactly match the query results. [#2395](https://github.com/github/vscode-codeql/pull/2395)
+- On the variant analysis results page, show the count of successful analyses instead of completed analyses, and indicate the reason why analyses were not successful. [#2349](https://github.com/github/vscode-codeql/pull/2349)
+- Fix bug where the "CodeQL: Set Current Database" command didn't always select the database. [#2384](https://github.com/github/vscode-codeql/pull/2384)
+
 ## 1.8.3 - 26 April 2023
 
 - Added ability to filter repositories for a variant analysis to only those that have results [#2343](https://github.com/github/vscode-codeql/pull/2343)

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -15,9 +15,6 @@ export const config: webpack.Configuration = {
   devtool: isDevBuild ? "inline-source-map" : "source-map",
   resolve: {
     extensions: [".js", ".ts", ".tsx", ".json"],
-    fallback: {
-      path: require.resolve("path-browserify"),
-    },
   },
   module: {
     rules: [
@@ -62,18 +59,10 @@ export const config: webpack.Configuration = {
       },
       {
         test: /\.(woff(2)?|ttf|eot)$/,
-        use: [
-          {
-            loader: "file-loader",
-            options: {
-              name: "[name].[ext]",
-              outputPath: "fonts/",
-              // We need this to make Webpack use the correct path for the fonts.
-              // Without this, the CSS file will use `url([object Module])`
-              esModule: false,
-            },
-          },
-        ],
+        type: "asset/resource",
+        generator: {
+          filename: "fonts/[hash][ext][query]",
+        },
       },
     ],
   },

extensions/ql-vscode/language-configuration.json

@@ -30,5 +30,5 @@
       "end": "^\\s*//\\s*#?endregion\\b"
     }
   },
-  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s\\,]+)"
 }

extensions/ql-vscode/media/dark/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#C5C5C5"/></svg>

extensions/ql-vscode/media/light/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#424242"/></svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.8.3",
+  "version": "1.8.9",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -36,6 +36,7 @@
   "activationEvents": [
     "onLanguage:ql",
     "onLanguage:ql-summary",
+    "onView:codeQLQueries",
     "onView:codeQLDatabases",
     "onView:codeQLVariantAnalysisRepositories",
     "onView:codeQLQueryHistory",
@@ -70,6 +71,7 @@
   "contributes": {
     "configurationDefaults": {
       "[ql]": {
+        "debug.saveBeforeStart": "nonUntitledEditorsInActiveGroup",
         "editor.wordBasedSuggestions": false
       },
       "[dbscheme]": {
@@ -188,7 +190,7 @@
           "scope": "machine-overridable",
           "type": "string",
           "default": "",
-          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
+          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -245,8 +247,8 @@
         },
         "codeQL.runningQueries.autoSave": {
           "type": "boolean",
-          "default": false,
-          "description": "Enable automatically saving a modified query file when running a query."
+          "description": "Enable automatically saving a modified query file when running a query.",
+          "markdownDeprecationMessage": "This property is deprecated and no longer has any effect. To control automatic saving of documents before running queries, use the `debug.saveBeforeStart` setting."
         },
         "codeQL.runningQueries.maxQueries": {
           "type": "integer",
@@ -329,6 +331,34 @@
           "patternErrorMessage": "Please enter a valid GitHub repository",
           "markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
         },
+        "codeQL.variantAnalysis.defaultResultsFilter": {
+          "type": "string",
+          "default": "all",
+          "enum": [
+            "all",
+            "withResults"
+          ],
+          "enumDescriptions": [
+            "Show all repositories in the results view.",
+            "Show only repositories with results in the results view."
+          ],
+          "description": "The default filter to apply to the variant analysis results view."
+        },
+        "codeQL.variantAnalysis.defaultResultsSort": {
+          "type": "string",
+          "default": "numberOfResults",
+          "enum": [
+            "alphabetically",
+            "popularity",
+            "numberOfResults"
+          ],
+          "enumDescriptions": [
+            "Sort repositories alphabetically in the results view.",
+            "Sort repositories by popularity in the results view.",
+            "Sort repositories by number of results in the results view."
+          ],
+          "description": "The default sorting order for repositories in the variant analysis results view."
+        },
         "codeQL.logInsights.joinOrderWarningThreshold": {
           "type": "number",
           "default": 50,
@@ -341,11 +371,23 @@
           "default": false,
           "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
         },
-        "codeQL.createQuery.folder": {
+        "codeQL.createQuery.qlPackLocation": {
           "type": "string",
-          "default": "",
           "patternErrorMessage": "Please enter a valid folder",
-          "markdownDescription": "The name of the folder where we want to create queries and query packs via the \"CodeQL: Create Query\" command. The folder should exist."
+          "markdownDescription": "The name of the folder where we want to create queries and QL packs via the \"CodeQL: Create Query\" command. The folder should exist."
+        },
+        "codeQL.createQuery.autogenerateQlPacks": {
+          "type": "string",
+          "default": "ask",
+          "enum": [
+            "ask",
+            "never"
+          ],
+          "enumDescriptions": [
+            "Ask to create a QL pack when a new CodeQL database is added.",
+            "Never create a QL pack when a new CodeQL database is added."
+          ],
+          "description": "Ask the user to generate a QL pack when a new CodeQL database is downloaded."
         }
       }
     },
@@ -414,6 +456,10 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.quickEvalCount",
+        "title": "CodeQL: Quick Evaluation Count"
+      },
       {
         "command": "codeQL.quickEvalContextEditor",
         "title": "CodeQL: Quick Evaluation"
@@ -434,6 +480,14 @@
         "command": "codeQL.previewQueryHelp",
         "title": "CodeQL: Preview Query Help"
       },
+      {
+        "command": "codeQL.previewQueryHelpContextExplorer",
+        "title": "CodeQL: Preview Query Help"
+      },
+      {
+        "command": "codeQL.previewQueryHelpContextEditor",
+        "title": "CodeQL: Preview Query Help"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
@@ -450,6 +504,33 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+        "title": "Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesFromPanel",
+        "title": "Run local queries",
+        "icon": "$(run-all)"
+      },
+      {
+        "command": "codeQL.runLocalQueryFromFileTab",
+        "title": "CodeQL: Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueryContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runVariantAnalysisContextMenu",
+        "title": "Run against variant analysis repositories"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.openConfigFile",
         "title": "Open database configuration file",
@@ -465,6 +546,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+        "title": "Add repositories with GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -786,6 +871,11 @@
         "title": "CodeQL: Go to QL Code",
         "enablement": "codeql.hasQLSource"
       },
+      {
+        "command": "codeQL.gotoQLContextEditor",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
+      },
       {
         "command": "codeQL.openDataExtensionsEditor",
         "title": "CodeQL: Open Data Extensions Editor"
@@ -812,6 +902,13 @@
       }
     ],
     "menus": {
+      "editor/title": [
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "group": "navigation",
+          "when": "resourceExtname == .ql && codeQL.currentDatabaseItem"
+        }
+      ],
       "view/title": [
         {
           "command": "codeQLDatabases.sortByName",
@@ -905,6 +1002,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -973,17 +1075,17 @@
         {
           "command": "codeQLQueryHistory.showEvalLog",
           "group": "4_queryHistory@1",
-          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showEvalLogSummary",
           "group": "4_queryHistory@2",
-          "when": "codeql.supportsEvalLog && viewItem == rawResultsItem || codeql.supportsEvalLog && viewItem == interpretedResultsItem || codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem || viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showEvalLogViewer",
           "group": "4_queryHistory@3",
-          "when": "config.codeQL.canary && codeql.supportsEvalLog && viewItem == rawResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == interpretedResultsItem || config.codeQL.canary && codeql.supportsEvalLog && viewItem == cancelledResultsItem"
+          "when": "config.codeQL.canary && viewItem == rawResultsItem || config.codeQL.canary && viewItem == interpretedResultsItem || config.codeQL.canary && viewItem == cancelledResultsItem"
         },
         {
           "command": "codeQLQueryHistory.showQueryText",
@@ -1030,6 +1132,31 @@
           "group": "1_queryHistory@1",
           "when": "viewItem == remoteResultsItem"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -1075,7 +1202,7 @@
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
@@ -1089,6 +1216,18 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "when": "false"
+        },
         {
           "command": "codeQL.runQueryContextEditor",
           "when": "false"
@@ -1141,6 +1280,10 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.quickEvalCount",
+          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+        },
         {
           "command": "codeQL.quickEvalContextEditor",
           "when": "false"
@@ -1161,6 +1304,14 @@
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
+        {
+          "command": "codeQL.previewQueryHelpContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.previewQueryHelpContextExplorer",
+          "when": "false"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -1201,6 +1352,18 @@
           "command": "codeQL.openDataExtensionsEditor",
           "when": "config.codeQL.canary && config.codeQL.dataExtensions.editor"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "when": "false"
+        },
         {
           "command": "codeQLVariantAnalysisRepositories.openConfigFile",
           "when": "false"
@@ -1233,6 +1396,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -1424,6 +1591,10 @@
         {
           "command": "codeQLTests.acceptOutputContextTestItem",
           "when": "false"
+        },
+        {
+          "command": "codeQL.gotoQLContextEditor",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -1468,11 +1639,11 @@
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextEditor",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
         {
-          "command": "codeQL.gotoQL",
+          "command": "codeQL.gotoQLContextEditor",
           "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
@@ -1488,6 +1659,11 @@
     },
     "views": {
       "ql-container": [
+        {
+          "id": "codeQLQueries",
+          "name": "Queries",
+          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
+        },
         {
           "id": "codeQLDatabases",
           "name": "Databases"
@@ -1520,6 +1696,10 @@
         "view": "codeQLQueryHistory",
         "contents": "You have no query history items at the moment.\n\nSelect a database to run a CodeQL query and get your first results."
       },
+      {
+        "view": "codeQLQueries",
+        "contents": "Looking for queries..."
+      },
       {
         "view": "codeQLDatabases",
         "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)"
@@ -1545,21 +1725,23 @@
     "test:vscode-integration:activated-extension": "jest --projects test/vscode-tests/activated-extension",
     "test:vscode-integration:no-workspace": "jest --projects test/vscode-tests/no-workspace",
     "test:vscode-integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace",
-    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration",
+    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration --verbose",
+    "clean-test-dir": "find . -type d -name .vscode-test -exec rm -r {} +",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",
     "lint:markdown": "markdownlint-cli2 \"../../**/*.{md,mdx}\" \"!**/node_modules/**\" \"!**/.vscode-test/**\" \"!**/build/cli/v*/**\"",
+    "find-deadcode": "ts-node scripts/find-deadcode.ts",
     "format-staged": "lint-staged",
-    "storybook": "start-storybook -p 6006",
-    "build-storybook": "build-storybook",
+    "storybook": "storybook dev -p 6006",
+    "build-storybook": "storybook build",
     "lint:scenarios": "ts-node scripts/lint-scenarios.ts",
     "check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
     "postinstall": "patch-package",
     "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
-    "@octokit/plugin-retry": "^3.0.9",
+    "@octokit/plugin-retry": "^4.1.6",
     "@octokit/rest": "^19.0.4",
     "@vscode/codicons": "^0.0.31",
     "@vscode/debugadapter": "^1.59.0",
@@ -1568,58 +1750,63 @@
     "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
     "chokidar": "^3.5.3",
-    "classnames": "~2.2.6",
+    "classnames": "^2.2.6",
     "d3": "^7.6.1",
     "d3-graphviz": "^5.0.2",
     "fs-extra": "^11.1.1",
     "immutable": "^4.0.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^9.0.0",
-    "minimist": "~1.2.6",
+    "minimist": "^1.2.6",
     "msw": "^1.2.0",
     "nanoid": "^3.2.0",
-    "node-fetch": "~2.6.7",
+    "node-fetch": "^2.6.7",
     "p-queue": "^6.0.0",
-    "path-browserify": "^1.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "semver": "~7.3.2",
+    "semver": "^7.5.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
     "stream": "^0.0.2",
-    "stream-chain": "~2.2.4",
-    "stream-json": "~1.7.3",
+    "stream-chain": "^2.2.4",
+    "stream-json": "^1.7.3",
     "styled-components": "^5.3.3",
     "tmp": "^0.1.0",
-    "tmp-promise": "~3.0.2",
-    "tree-kill": "~1.2.2",
-    "unzipper": "~0.10.5",
+    "tmp-promise": "^3.0.2",
+    "tree-kill": "^1.2.2",
+    "unzipper": "^0.10.5",
     "vscode-extension-telemetry": "^0.1.6",
     "vscode-jsonrpc": "^8.0.2",
     "vscode-languageclient": "^8.0.2",
-    "vscode-test-adapter-api": "~1.7.0",
-    "vscode-test-adapter-util": "~0.7.0",
-    "zip-a-folder": "~1.1.3"
+    "vscode-test-adapter-api": "^1.7.0",
+    "vscode-test-adapter-util": "^0.7.0",
+    "zip-a-folder": "^2.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
-    "@faker-js/faker": "^7.5.0",
+    "@babel/preset-env": "^7.21.4",
+    "@babel/preset-react": "^7.18.6",
+    "@babel/preset-typescript": "^7.21.4",
+    "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.3.0",
     "@octokit/plugin-throttling": "^5.0.1",
-    "@storybook/addon-actions": "^6.5.17-alpha.0",
-    "@storybook/addon-essentials": "^6.5.17-alpha.0",
-    "@storybook/addon-interactions": "^6.5.17-alpha.0",
-    "@storybook/addon-links": "^6.5.17-alpha.0",
-    "@storybook/builder-webpack5": "^6.5.17-alpha.0",
-    "@storybook/manager-webpack5": "^6.5.17-alpha.0",
-    "@storybook/react": "^6.5.17-alpha.0",
-    "@storybook/testing-library": "^0.0.13",
+    "@storybook/addon-actions": "^7.1.0",
+    "@storybook/addon-essentials": "^7.1.0",
+    "@storybook/addon-interactions": "^7.1.0",
+    "@storybook/addon-links": "^7.1.0",
+    "@storybook/components": "^7.1.0",
+    "@storybook/csf": "^0.1.1",
+    "@storybook/manager-api": "^7.1.0",
+    "@storybook/react": "^7.1.0",
+    "@storybook/react-webpack5": "^7.1.0",
+    "@storybook/theming": "^7.1.0",
+    "@testing-library/dom": "^9.3.0",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",
     "@types/child-process-promise": "^2.2.1",
-    "@types/classnames": "~2.2.9",
+    "@types/classnames": "^2.2.9",
     "@types/d3": "^7.4.0",
     "@types/d3-graphviz": "^2.6.6",
     "@types/del": "^4.0.0",
@@ -1630,41 +1817,41 @@
     "@types/gulp-sourcemaps": "0.0.32",
     "@types/jest": "^29.0.2",
     "@types/js-yaml": "^3.12.5",
-    "@types/jszip": "~3.1.6",
+    "@types/jszip": "^3.1.6",
     "@types/nanoid": "^3.0.0",
     "@types/node": "^16.11.25",
-    "@types/node-fetch": "~2.5.2",
+    "@types/node-fetch": "^2.5.2",
     "@types/react": "^18.0.28",
     "@types/react-dom": "^18.0.11",
-    "@types/sarif": "~2.1.2",
-    "@types/semver": "~7.2.0",
-    "@types/stream-chain": "~2.0.1",
-    "@types/stream-json": "~1.7.1",
+    "@types/sarif": "^2.1.2",
+    "@types/semver": "^7.2.0",
+    "@types/stream-chain": "^2.0.1",
+    "@types/stream-json": "^1.7.1",
     "@types/styled-components": "^5.1.11",
     "@types/tar-stream": "^2.2.2",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
-    "@types/unzipper": "~0.10.1",
+    "@types/unzipper": "^0.10.1",
     "@types/vscode": "^1.67.0",
     "@types/webpack": "^5.28.0",
     "@types/webpack-env": "^1.18.0",
-    "@types/xml2js": "~0.4.4",
     "@typescript-eslint/eslint-plugin": "^5.38.0",
     "@typescript-eslint/parser": "^5.38.0",
     "@vscode/test-electron": "^2.2.0",
-    "@vscode/vsce": "^2.15.0",
+    "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
+    "cosmiconfig": "^7.1.0",
     "cross-env": "^7.0.3",
-    "css-loader": "~3.1.0",
+    "css-loader": "^6.8.1",
     "del": "^6.0.0",
     "esbuild": "^0.15.15",
     "eslint": "^8.23.1",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-etc": "^2.0.2",
     "eslint-plugin-github": "^4.4.1",
-    "eslint-plugin-jest-dom": "^4.0.2",
-    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-jest-dom": "^5.0.1",
+    "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-react": "^7.31.8",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-storybook": "^0.6.4",
@@ -1679,13 +1866,14 @@
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "~13.2.0",
+    "lint-staged": "^13.2.0",
     "markdownlint-cli2": "^0.6.0",
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "patch-package": "^6.5.0",
-    "prettier": "^2.7.1",
+    "patch-package": "^7.0.0",
+    "prettier": "^3.0.0",
+    "storybook": "^7.1.0",
     "tar-stream": "^3.0.0",
     "through2": "^4.0.2",
     "ts-jest": "^29.0.1",
@@ -1693,6 +1881,7 @@
     "ts-loader": "^9.4.2",
     "ts-node": "^10.7.0",
     "ts-protoc-gen": "^0.9.0",
+    "ts-unused-exports": "^9.0.5",
     "typescript": "^5.0.2",
     "webpack": "^5.76.0",
     "webpack-cli": "^5.0.1"

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -18,13 +18,16 @@ import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
 
 import { getFiles } from "./util/files";
-import type { GitHubApiRequest } from "../src/mocks/gh-api-request";
-import { isGetVariantAnalysisRequest } from "../src/mocks/gh-api-request";
+import type { GitHubApiRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
+import { isGetVariantAnalysisRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
 import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
 import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 
 // Make sure we don't run into rate limits by automatically waiting until we can
 // make another request.

extensions/ql-vscode/scripts/find-deadcode.ts

@@ -0,0 +1,47 @@
+import { basename, join, relative, resolve } from "path";
+import analyzeTsConfig from "ts-unused-exports";
+import { containsPath, pathsEqual } from "../src/common/files";
+import { exit } from "process";
+
+function ignoreFile(file: string): boolean {
+  return (
+    containsPath("gulpfile.ts", file) ||
+    containsPath(join("src", "stories"), file) ||
+    pathsEqual(
+      join("test", "vscode-tests", "jest-runner-installed-extensions.ts"),
+      file,
+    ) ||
+    basename(file) === "jest.config.ts" ||
+    basename(file) === "index.tsx" ||
+    basename(file) === "index.ts"
+  );
+}
+
+function main() {
+  const repositoryRoot = resolve(join(__dirname, ".."));
+
+  const result = analyzeTsConfig("tsconfig.deadcode.json");
+  let foundUnusedExports = false;
+
+  for (const [filepath, exportNameAndLocations] of Object.entries(result)) {
+    const relativeFilepath = relative(repositoryRoot, filepath);
+
+    if (ignoreFile(relativeFilepath)) {
+      continue;
+    }
+
+    foundUnusedExports = true;
+
+    console.log(relativeFilepath);
+    for (const exportNameAndLocation of exportNameAndLocations) {
+      console.log(`    ${exportNameAndLocation.exportName}`);
+    }
+    console.log();
+  }
+
+  if (foundUnusedExports) {
+    exit(1);
+  }
+}
+
+main();

extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts

@@ -20,7 +20,10 @@ if (process.argv.length !== 3) {
 const scenarioName = process.argv[2];
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 const scenarioDirectory = resolve(scenariosDirectory, scenarioName);
 
 async function fixScenarioFiles() {

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -8,13 +8,19 @@ import { getFiles } from "./util/files";
 
 const extensionDirectory = resolve(__dirname, "..");
 const rootDirectory = resolve(extensionDirectory, "../..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 
 const debug = process.env.RUNNER_DEBUG || process.argv.includes("--debug");
 
 async function lintScenarios() {
   const schema = createGenerator({
-    path: resolve(extensionDirectory, "src/mocks/gh-api-request.ts"),
+    path: resolve(
+      extensionDirectory,
+      "src/variant-analysis/gh-api/mocks/gh-api-request.ts",
+    ),
     tsconfig: resolve(extensionDirectory, "tsconfig.json"),
     type: "GitHubApiRequest",
     skipTypeCheck: true,

extensions/ql-vscode/src/code-tour/code-tour.ts

@@ -0,0 +1,55 @@
+import { AppCommandManager } from "../common/commands";
+import { Uri, workspace } from "vscode";
+import { join } from "path";
+import { pathExists } from "fs-extra";
+import { isCodespacesTemplate } from "../config";
+import { showBinaryChoiceDialog } from "../common/vscode/dialog";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Check if the current workspace is the CodeTour and open the workspace folder.
+ * Without this, we can't run the code tour correctly.
+ **/
+export async function prepareCodeTour(
+  commandManager: AppCommandManager,
+): Promise<void> {
+  if (workspace.workspaceFolders?.length) {
+    const currentFolder = workspace.workspaceFolders[0].uri.fsPath;
+
+    const tutorialWorkspacePath = join(
+      currentFolder,
+      "tutorial.code-workspace",
+    );
+    const toursFolderPath = join(currentFolder, ".tours");
+
+    /** We're opening the tutorial workspace, if we detect it.
+     * This will only happen if the following three conditions are met:
+     * - the .tours folder exists
+     * - the tutorial.code-workspace file exists
+     * - the CODESPACES_TEMPLATE setting doesn't exist (it's only set if the user has already opened
+     * the tutorial workspace so it's a good indicator that the user is in the folder but has ignored
+     * the prompt to open the workspace)
+     */
+    if (
+      (await pathExists(tutorialWorkspacePath)) &&
+      (await pathExists(toursFolderPath)) &&
+      !isCodespacesTemplate()
+    ) {
+      const answer = await showBinaryChoiceDialog(
+        "We've detected you're in the CodeQL Tour repo. We will need to open the workspace file to continue. Reload?",
+      );
+
+      if (!answer) {
+        return;
+      }
+
+      const tutorialWorkspaceUri = Uri.file(tutorialWorkspacePath);
+
+      void extLogger.log(
+        `In prepareCodeTour() method, going to open the tutorial workspace file: ${tutorialWorkspacePath}`,
+      );
+
+      await commandManager.execute("vscode.openFolder", tutorialWorkspaceUri);
+    }
+  }
+}

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,7 +1,7 @@
 import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { Logger } from "../common/logging";
+import { getErrorMessage } from "../common/helpers-pure";
 
 /**
  * Get the version of a CodeQL CLI.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -11,8 +11,8 @@ import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "./pure/bqrs-cli-types";
-import { allowCanaryQueryServer, CliConfig } from "./config";
+import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
   FindDistributionResultKind,
@@ -21,14 +21,15 @@ import {
   assertNever,
   getErrorMessage,
   getErrorStack,
-} from "./pure/helpers-pure";
-import { QueryMetadata, SortDirection } from "./pure/interface-types";
-import { BaseLogger, Logger, ProgressReporter } from "./common";
-import { CompilationMessage } from "./pure/legacy-messages";
-import { sarifParser } from "./sarif-parser";
-import { walkDirectory } from "./helpers";
-import { App } from "./common/app";
-import { QueryLanguage } from "./common/query-language";
+} from "../common/helpers-pure";
+import { walkDirectory } from "../common/files";
+import { QueryMetadata, SortDirection } from "../common/interface-types";
+import { BaseLogger, Logger } from "../common/logging";
+import { ProgressReporter } from "../common/logging/vscode";
+import { CompilationMessage } from "../query-server/legacy-messages";
+import { sarifParser } from "../common/sarif-parser";
+import { App } from "../common/app";
+import { QueryLanguage } from "../common/query-language";
 
 /**
  * The version of the SARIF format that we are using.
@@ -134,6 +135,11 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries`.
+ */
+export type ResolvedQueries = string[];
+
 /**
  * The expected output of `codeql resolve tests`.
  */
@@ -213,7 +219,7 @@ export class CodeQLCliServer implements Disposable {
     private readonly app: App,
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    public readonly logger: Logger,
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -325,6 +331,7 @@ export class CodeQLCliServer implements Disposable {
     commandArgs: string[],
     description: string,
     onLine?: OnLineCallback,
+    silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
@@ -344,7 +351,12 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(" ");
-      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      // If we are running silently, we don't want to print anything to the console.
+      if (!silent) {
+        void this.logger.log(
+          `${description} using CodeQL CLI: ${argsString}...`,
+        );
+      }
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -390,24 +402,30 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        void this.logger.log("CLI command succeeded.");
+        if (!silent) {
+          void this.logger.log("CLI command succeeded.");
+        }
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
+        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
         const newError =
           stderrBuffers.length === 0
-            ? new Error(`${description} failed: ${err}`)
+            ? new Error(
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
+              )
             : new Error(
-                `${description} failed: ${Buffer.concat(stderrBuffers).toString(
-                  "utf8",
-                )}`,
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
+                  stderrBuffers,
+                ).toString("utf8")}`,
               );
         newError.stack += getErrorStack(err);
         throw newError;
       } finally {
-        void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent) {
+          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
         process.stderr.removeAllListeners("data");
@@ -544,9 +562,11 @@ export class CodeQLCliServer implements Disposable {
     {
       progressReporter,
       onLine,
+      silent = false,
     }: {
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<string> {
     if (progressReporter) {
@@ -562,6 +582,7 @@ export class CodeQLCliServer implements Disposable {
             commandArgs,
             description,
             onLine,
+            silent,
           ).then(resolve, reject);
         } catch (err) {
           reject(err);
@@ -595,10 +616,12 @@ export class CodeQLCliServer implements Disposable {
       addFormat = true,
       progressReporter,
       onLine,
+      silent = false,
     }: {
       addFormat?: boolean;
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
@@ -609,6 +632,7 @@ export class CodeQLCliServer implements Disposable {
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
       onLine,
+      silent,
     });
     try {
       return JSON.parse(result) as OutputType;
@@ -695,6 +719,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(
     workspaces: string[],
     queryPath: string,
+    silent = false,
   ): Promise<QuerySetup> {
     const subcommandArgs = [
       "--query",
@@ -705,6 +730,7 @@ export class CodeQLCliServer implements Disposable {
       ["resolve", "library-path"],
       subcommandArgs,
       "Resolving library paths",
+      { silent },
     );
   }
 
@@ -731,6 +757,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Finds all available queries in a given directory.
+   * @param queryDir Root of directory tree to search for queries.
+   * @param silent If true, don't print logs to the CodeQL extension log.
+   * @returns The list of queries that were found.
+   */
+  public async resolveQueries(
+    queryDir: string,
+    silent?: boolean,
+  ): Promise<ResolvedQueries> {
+    const subcommandArgs = [queryDir];
+    return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
+      ["resolve", "queries"],
+      subcommandArgs,
+      "Resolving queries",
+      { silent },
+    );
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.
@@ -1031,6 +1076,7 @@ export class CodeQLCliServer implements Disposable {
     resultsPath: string,
     interpretedResultsPath: string,
     sourceInfo?: SourceInfo,
+    args?: string[],
   ): Promise<sarif.Log> {
     const additionalArgs = [
       // TODO: This flag means that we don't group interpreted results
@@ -1038,6 +1084,7 @@ export class CodeQLCliServer implements Disposable {
       // interpretation with and without this flag, or do some
       // grouping client-side.
       "--no-group-results",
+      ...(args ?? []),
     ];
 
     await this.runInterpretCommand(
@@ -1381,21 +1428,13 @@ export class CodeQLCliServer implements Disposable {
 
   async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
     const args = includeQueries ? [dir] : ["--no-include-queries", dir];
-    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
-    // previous versions return a list of packs.
-    const results: { paths: string[] } | string[] =
-      await this.runJsonCodeQlCliCommand(
-        ["pack", "packlist"],
-        args,
-        "Generating the pack list",
-      );
+    const results: { paths: string[] } = await this.runJsonCodeQlCliCommand(
+      ["pack", "packlist"],
+      args,
+      "Generating the pack list",
+    );
 
-    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
-    if ("paths" in results) {
-      return results.paths;
-    } else {
-      return results;
-    }
+    return results.paths;
   }
 
   async packResolveDependencies(
@@ -1431,9 +1470,9 @@ export class CodeQLCliServer implements Disposable {
         // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
         await this.app.commands.execute(
           "setContext",
-          "codeql.supportsEvalLog",
+          "codeql.supportsQuickEvalCount",
           newVersion.compare(
-            CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
           ) >= 0,
         );
       } catch (e) {
@@ -1525,10 +1564,23 @@ export function spawnServer(
     );
   }
 
+  let lastStdout: any = undefined;
+  child.stdout!.on("data", (data) => {
+    lastStdout = data;
+  });
   // Set up event listeners.
-  child.on("close", (code) =>
-    logger.log(`Child process exited with code ${code}`),
-  );
+  child.on("close", async (code, signal) => {
+    if (code !== null)
+      void logger.log(`Child process exited with code ${code}`);
+    if (signal)
+      void logger.log(
+        `Child process exited due to receipt of signal ${signal}`,
+      );
+    // If the process exited abnormally, log the last stdout message,
+    // It may be from the jvm.
+    if (code !== 0 && lastStdout !== undefined)
+      void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+  });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
     child.stdout!.on("data", stdoutListener);
@@ -1699,31 +1751,32 @@ async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
   }
 }
 
-export function shouldDebugIdeServer() {
+function isEnvTrue(name: string): boolean {
   return (
-    "IDE_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.IDE_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
+    name in process.env &&
+    process.env[name] !== "0" &&
+    // Use en-US since we expect the value to be either "false" or "FALSE", not a localized version.
+    process.env[name]?.toLocaleLowerCase("en-US") !== "false"
   );
 }
 
+export function shouldDebugIdeServer() {
+  return isEnvTrue("IDE_SERVER_JAVA_DEBUG");
+}
+
 export function shouldDebugQueryServer() {
-  return (
-    "QUERY_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.QUERY_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("QUERY_SERVER_JAVA_DEBUG");
 }
 
 export function shouldDebugCliServer() {
-  return (
-    "CLI_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.CLI_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("CLI_SERVER_JAVA_DEBUG");
 }
 
 export class CliVersionConstraint {
+  // The oldest version of the CLI that we support. This is used to determine
+  // whether to show a warning about the CLI being too old on startup.
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.9.4");
+
   /**
    * CLI version where building QLX packs for remote queries is supported.
    * (The options were _accepted_ by a few earlier versions, but only from
@@ -1739,21 +1792,9 @@ export class CliVersionConstraint {
   );
 
   /**
-   * CLI version where the `--evaluator-log` and related options to the query server were introduced,
-   * on a per-query server basis.
+   * CLI version where the `resolve extensions` subcommand exists.
    */
-  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer("2.8.2");
-
-  /**
-   * CLI version that supports rotating structured logs to produce one per query.
-   *
-   * Note that 2.8.4 supports generating the evaluation logs and summaries,
-   * but 2.9.0 includes a new option to produce the end-of-query summary logs to
-   * the query server console. For simplicity we gate all features behind 2.9.0,
-   * but if a user is tied to the 2.8 release, we can enable evaluator logs
-   * and summaries for them.
-   */
-  public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer("2.9.0");
+  public static CLI_VERSION_WITH_RESOLVE_EXTENSIONS = new SemVer("2.10.2");
 
   /**
    * CLI version that supports the `--sourcemap` option for log generation.
@@ -1782,6 +1823,20 @@ export class CliVersionConstraint {
     "2.12.4",
   );
 
+  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
+
+  /**
+   * CLI version where the query server supports quick-eval count mode.
+   */
+  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
+
+  /**
+   * CLI version where the langauge server supports visisbility change notifications.
+   */
+  public static CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS = new SemVer(
+    "2.14.0",
+  );
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1800,15 +1855,9 @@ export class CliVersionConstraint {
     );
   }
 
-  async supportsStructuredEvalLog() {
+  async supportsResolveExtensions() {
     return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG,
-    );
-  }
-
-  async supportsPerQueryEvalLog() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+      CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS,
     );
   }
 
@@ -1851,4 +1900,20 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
     );
   }
+
+  async usesGlobalCompilationCache() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
+  }
+
+  async supportsVisibilityNotifications() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
+    );
+  }
+
+  async supportsQuickEvalCount() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -3,24 +3,29 @@ import { pathExists, mkdtemp, createWriteStream, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
 import * as semver from "semver";
-import { parse } from "url";
+import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
-import { DistributionConfig } from "./config";
-import {
-  InvocationRateLimiter,
-  InvocationRateLimiterResultKind,
-  showAndLogErrorMessage,
-  showAndLogWarningMessage,
-} from "./helpers";
-import { extLogger } from "./common";
+import { DistributionConfig } from "../config";
+import { extLogger } from "../common/logging/vscode";
 import { getCodeQlCliVersion } from "./cli-version";
-import { ProgressCallback, reportStreamProgress } from "./progress";
+import {
+  ProgressCallback,
+  reportStreamProgress,
+} from "../common/vscode/progress";
 import {
   codeQlLauncherName,
   deprecatedCodeQlLauncherName,
   extractZipArchive,
   getRequiredAssetName,
-} from "./pure/distribution";
+} from "../common/distribution";
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+} from "../common/invocation-rate-limiter";
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+} from "../common/logging";
 
 /**
  * distribution.ts
@@ -73,7 +78,7 @@ export class DistributionManager implements DistributionProvider {
         extensionContext,
       );
     this.updateCheckRateLimiter = new InvocationRateLimiter(
-      extensionContext,
+      extensionContext.globalState,
       "extensionSpecificDistributionUpdateCheck",
       () =>
         this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),
@@ -155,6 +160,7 @@ export class DistributionManager implements DistributionProvider {
     if (this.config.customCodeQlPath) {
       if (!(await pathExists(this.config.customCodeQlPath))) {
         void showAndLogErrorMessage(
+          extLogger,
           `The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
             "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
             "that a CodeQL executable exists at the specified path or remove the setting.",
@@ -502,7 +508,7 @@ class ExtensionSpecificDistributionManager {
         0,
       ) || "";
     return join(
-      this.extensionContext.globalStoragePath,
+      this.extensionContext.globalStorageUri.fsPath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
         distributionFolderIndex,
     );
@@ -667,7 +673,7 @@ export class ReleasesApiConsumer {
       redirectUrl &&
       redirectCount < ReleasesApiConsumer._maxRedirects
     ) {
-      const parsedRedirectUrl = parse(redirectUrl);
+      const parsedRedirectUrl = new URL(redirectUrl);
       if (parsedRedirectUrl.protocol !== "https:") {
         throw new Error("Encountered a non-https redirect, rejecting");
       }
@@ -712,7 +718,7 @@ export enum DistributionKind {
   PathEnvironmentVariable,
 }
 
-export interface Distribution {
+interface Distribution {
   codeQlPath: string;
   kind: DistributionKind;
 }
@@ -770,22 +776,22 @@ type DistributionUpdateCheckResult =
   | InvalidLocationResult
   | UpdateAvailableResult;
 
-export interface AlreadyCheckedRecentlyResult {
+interface AlreadyCheckedRecentlyResult {
   kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
 }
 
-export interface AlreadyUpToDateResult {
+interface AlreadyUpToDateResult {
   kind: DistributionUpdateCheckResultKind.AlreadyUpToDate;
 }
 
 /**
  * The distribution could not be installed or updated because it is not managed by the extension.
  */
-export interface InvalidLocationResult {
+interface InvalidLocationResult {
   kind: DistributionUpdateCheckResultKind.InvalidLocation;
 }
 
-export interface UpdateAvailableResult {
+interface UpdateAvailableResult {
   kind: DistributionUpdateCheckResultKind.UpdateAvailable;
   updatedRelease: Release;
 }
@@ -847,6 +853,7 @@ export async function getExecutableFromDirectory(
 
 function warnDeprecatedLauncher() {
   void showAndLogWarningMessage(
+    extLogger,
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
       `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`,
   );
@@ -855,7 +862,7 @@ function warnDeprecatedLauncher() {
 /**
  * A release on GitHub.
  */
-export interface Release {
+interface Release {
   assets: ReleaseAsset[];
 
   /**
@@ -877,7 +884,7 @@ export interface Release {
 /**
  * An asset corresponding to a release on GitHub.
  */
-export interface ReleaseAsset {
+interface ReleaseAsset {
   /**
    * The id associated with the asset on GitHub.
    */
@@ -947,7 +954,10 @@ export interface GithubReleaseAsset {
 }
 
 export class GithubApiError extends Error {
-  constructor(public status: number, public body: string) {
+  constructor(
+    public status: number,
+    public body: string,
+  ) {
     super(`API call failed with status code ${status}, body: ${body}`);
   }
 }

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -0,0 +1,80 @@
+import { CodeQLCliServer } from "./cli";
+import { Uri, window } from "vscode";
+import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { UserCancellationException } from "../common/vscode/progress";
+import { showAndLogErrorMessage } from "../common/logging";
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined,
+): Promise<QueryLanguage | undefined> {
+  const uri = queryUri || window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(
+        getOnDiskWorkspaceFolders(),
+        uri,
+      );
+      const language = Object.keys(queryInfo.byLanguage)[0];
+      void extLogger.log(`Detected query language: ${language}`);
+
+      if (isQueryLanguage(language)) {
+        return language;
+      }
+
+      void extLogger.log(
+        "Query language is unsupported. Select language manually.",
+      );
+    } catch (e) {
+      void extLogger.log(
+        "Could not autodetect query language. Select language manually.",
+      );
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(
+  cliServer: CodeQLCliServer,
+  throwOnEmpty = true,
+): Promise<QueryLanguage | undefined> {
+  const language = await window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    {
+      placeHolder: "Select target language for your query",
+      ignoreFocusOut: true,
+    },
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException("Cancelled.");
+    } else {
+      void showAndLogErrorMessage(
+        extLogger,
+        "Language not found. Language must be specified manually.",
+      );
+    }
+    return undefined;
+  }
+
+  if (!isQueryLanguage(language)) {
+    void showAndLogErrorMessage(
+      extLogger,
+      `Language '${language}' is not supported. Only languages ${Object.values(
+        QueryLanguage,
+      ).join(", ")} are supported.`,
+    );
+    return undefined;
+  }
+
+  return language;
+}

extensions/ql-vscode/src/codeql-cli/query-metadata.ts

@@ -0,0 +1,22 @@
+import { CodeQLCliServer } from "./cli";
+import { QueryMetadata } from "../common/interface-types";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Gets metadata for a query, if it exists.
+ * @param cliServer The CLI server.
+ * @param queryPath The path to the query.
+ * @returns A promise that resolves to the query metadata, if available.
+ */
+export async function tryGetQueryMetadata(
+  cliServer: CodeQLCliServer,
+  queryPath: string,
+): Promise<QueryMetadata | undefined> {
+  try {
+    return await cliServer.resolveMetadata(queryPath);
+  } catch (e) {
+    // Ignore errors and provide no metadata.
+    void extLogger.log(`Couldn't resolve metadata for ${queryPath}: ${e}`);
+    return;
+  }
+}

extensions/ql-vscode/src/common/app.ts

@@ -1,14 +1,16 @@
 import { Credentials } from "./authentication";
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 import { AppEventEmitter } from "./events";
-import { Logger } from "./logging";
+import { NotificationLogger } from "./logging";
 import { Memento } from "./memento";
 import { AppCommandManager } from "./commands";
+import { AppTelemetry } from "./telemetry";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
   readonly mode: AppMode;
-  readonly logger: Logger;
+  readonly logger: NotificationLogger;
+  readonly telemetry?: AppTelemetry;
   readonly subscriptions: Disposable[];
   readonly extensionPath: string;
   readonly globalStoragePath: string;
@@ -16,6 +18,7 @@ export interface App {
   readonly workspaceState: Memento;
   readonly credentials: Credentials;
   readonly commands: AppCommandManager;
+  readonly environment: EnvironmentContext;
 }
 
 export enum AppMode {
@@ -23,3 +26,7 @@ export enum AppMode {
   Development = 2,
   Test = 3,
 }
+
+export interface EnvironmentContext {
+  language: string;
+}

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -1,5 +1,3 @@
-export const PAGE_SIZE = 1000;
-
 /**
  * The single-character codes used in the bqrs format for the the kind
  * of a result column. This namespace is intentionally not an enum, see
@@ -15,7 +13,7 @@ export namespace ColumnKindCode {
   export const ENTITY = "e";
 }
 
-export type ColumnKind =
+type ColumnKind =
   | typeof ColumnKindCode.FLOAT
   | typeof ColumnKindCode.INTEGER
   | typeof ColumnKindCode.STRING
@@ -46,7 +44,7 @@ export function getResultSetSchema(
   }
   return undefined;
 }
-export interface PaginationInfo {
+interface PaginationInfo {
   "step-size": number;
   offsets: number[];
 }
@@ -115,7 +113,7 @@ export type BqrsKind =
   | "Entity";
 
 interface BqrsColumn {
-  name: string;
+  name?: string;
   kind: BqrsKind;
 }
 export interface DecodedBqrsChunk {

extensions/ql-vscode/src/common/bqrs-utils.ts

@@ -4,7 +4,7 @@ import {
   LineColumnLocation,
   WholeFileLocation,
 } from "./bqrs-cli-types";
-import { createRemoteFileRef } from "./location-link-utils";
+import { createRemoteFileRef } from "../common/location-link-utils";
 
 /**
  * The CodeQL filesystem libraries use this pattern in `getURL()` predicates
@@ -142,5 +142,7 @@ export function tryGetRemoteLocation(
     fileLink,
     resolvableLocation.startLine,
     resolvableLocation.endLine,
+    resolvableLocation.startColumn,
+    resolvableLocation.endColumn,
   );
 }

extensions/ql-vscode/src/common/commands.ts

@@ -1,17 +1,17 @@
 import type { CommandManager } from "../packages/commands";
 import type { Uri, Range, TextDocumentShowOptions } from "vscode";
-import type { AstItem } from "../astViewer";
+import type { AstItem } from "../language-support";
 import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
-import type { DatabaseItem } from "../local-databases";
+import type { DatabaseItem } from "../databases/local-databases";
 import type { QueryHistoryInfo } from "../query-history/query-history-info";
-import type { RepositoriesFilterSortStateWithIds } from "../pure/variant-analysis-filter-sort";
-import type { TestTreeNode } from "../test-tree-node";
+import type { TestTreeNode } from "../query-testing/test-tree-node";
 import type {
   VariantAnalysis,
   VariantAnalysisScannedRepository,
   VariantAnalysisScannedRepositoryResult,
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
+import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -55,7 +55,7 @@ export type ExplorerSelectionCommandFunction<Item> = (
 
 // Builtin commands where the implementation is provided by VS Code and not by this extension.
 // See https://code.visualstudio.com/api/references/commands
-export type BuiltInVsCodeCommands = {
+type BuiltInVsCodeCommands = {
   // The codeQLDatabases.focus command is provided by VS Code because we've registered the custom view
   "codeQLDatabases.focus": () => Promise<void>;
   "markdown.showPreviewToSide": (uri: Uri) => Promise<void>;
@@ -115,6 +115,10 @@ export type QueryEditorCommands = {
     selectedQuery: Uri,
   ) => Promise<void>;
   "codeQL.previewQueryHelp": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextEditor": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
 };
 
 // Commands used for running local queries
@@ -125,8 +129,14 @@ export type LocalQueryCommands = {
   "codeQL.runQueryOnMultipleDatabasesContextEditor": (
     uri?: Uri,
   ) => Promise<void>;
+  "codeQLQueries.runLocalQueryFromQueriesPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueryContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesFromPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise<void>;
   "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
   "codeQL.quickEval": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalCount": (uri: Uri) => Promise<void>;
   "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
   "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
   "codeQL.quickQuery": () => Promise<void>;
@@ -233,10 +243,6 @@ export type VariantAnalysisCommands = {
     scannedRepo: VariantAnalysisScannedRepository,
     variantAnalysisSummary: VariantAnalysis,
   ) => Promise<void>;
-  "codeQL.copyVariantAnalysisRepoList": (
-    variantAnalysisId: number,
-    filterSort?: RepositoriesFilterSortStateWithIds,
-  ) => Promise<void>;
   "codeQL.loadVariantAnalysisRepoResults": (
     variantAnalysisId: number,
     repositoryFullName: string,
@@ -247,6 +253,9 @@ export type VariantAnalysisCommands = {
   "codeQL.monitorRehydratedVariantAnalysis": (
     variantAnalysis: VariantAnalysis,
   ) => Promise<void>;
+  "codeQL.monitorReauthenticatedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
   "codeQL.openVariantAnalysisLogs": (
     variantAnalysisId: number,
   ) => Promise<void>;
@@ -255,6 +264,7 @@ export type VariantAnalysisCommands = {
   ) => Promise<void>;
   "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
   "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQLQueries.runVariantAnalysisContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
 };
 
 export type DatabasePanelCommands = {
@@ -268,6 +278,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {
@@ -299,6 +310,7 @@ export type EvalLogViewerCommands = {
 
 export type SummaryLanguageSupportCommands = {
   "codeQL.gotoQL": () => Promise<void>;
+  "codeQL.gotoQLContextEditor": () => Promise<void>;
 };
 
 export type TestUICommands = {

extensions/ql-vscode/src/common/discovery.ts

@@ -1,25 +1,38 @@
-import { DisposableObject } from "./pure/disposable-object";
-import { extLogger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { DisposableObject } from "./disposable-object";
+import { getErrorMessage } from "./helpers-pure";
+import { Logger } from "./logging";
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
  * files. This class automatically prevents more than one discovery operation from running at the
  * same time.
  */
-export abstract class Discovery<T> extends DisposableObject {
-  private retry = false;
-  private discoveryInProgress = false;
+export abstract class Discovery extends DisposableObject {
+  private restartWhenFinished = false;
+  private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string) {
+  constructor(
+    protected readonly name: string,
+    private readonly logger: Logger,
+  ) {
     super();
   }
 
+  /**
+   * Returns the promise of the currently running refresh operation, if one is in progress.
+   * Otherwise returns a promise that resolves immediately.
+   */
+  public waitForCurrentRefresh(): Promise<void> {
+    return this.currentDiscoveryPromise ?? Promise.resolve();
+  }
+
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
+   *
+   * Returns a promise that resolves when the refresh is complete, including any retries.
    */
-  public refresh(): void {
+  public refresh(): Promise<void> {
     // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
     // got a storm of refresh requests due to, say, the copying or deletion of a large directory
     // tree, we could potentially spawn a separate simultaneous discovery operation for each
@@ -36,14 +49,16 @@ export abstract class Discovery<T> extends DisposableObject {
     // other change notifications that might be coming along. However, this would create more
     // latency in the common case, in order to save a bit of latency in the uncommon case.
 
-    if (this.discoveryInProgress) {
+    if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
-      this.retry = true;
+      this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
-      this.discoveryInProgress = true;
-      this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery().finally(() => {
+        this.currentDiscoveryPromise = undefined;
+      });
     }
+    return this.currentDiscoveryPromise;
   }
 
   /**
@@ -51,46 +66,28 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery operation completes, the `update` function will be invoked with the results of the
    * discovery.
    */
-  private launchDiscovery(): void {
-    const discoveryPromise = this.discover();
-    discoveryPromise
-      .then((results) => {
-        if (!this.retry) {
-          // Update any listeners with the results of the discovery.
-          this.discoveryInProgress = false;
-          this.update(results);
-        }
-      })
+  private async launchDiscovery(): Promise<void> {
+    try {
+      await this.discover();
+    } catch (err) {
+      void this.logger.log(
+        `${this.name} failed. Reason: ${getErrorMessage(err)}`,
+      );
+    }
 
-      .catch((err: unknown) => {
-        void extLogger.log(
-          `${this.name} failed. Reason: ${getErrorMessage(err)}`,
-        );
-      })
-
-      .finally(() => {
-        if (this.retry) {
-          // Another refresh request came in while we were still running a previous discovery
-          // operation. Since the discovery results we just computed are now stale, we'll launch
-          // another discovery operation instead of updating.
-          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-          // initial discovery operation failed.
-          this.retry = false;
-          this.launchDiscovery();
-        }
-      });
+    if (this.restartWhenFinished) {
+      // Another refresh request came in while we were still running a previous discovery
+      // operation. Since the discovery results we just computed are now stale, we'll launch
+      // another discovery operation instead of updating.
+      // We want to relaunch discovery regardless of if the initial discovery operation
+      // succeeded or failed.
+      this.restartWhenFinished = false;
+      await this.launchDiscovery();
+    }
   }
 
   /**
    * Overridden by the derived class to spawn the actual discovery operation, returning the results.
    */
-  protected abstract discover(): Promise<T>;
-
-  /**
-   * Overridden by the derived class to atomically update the `Discovery` object with the results of
-   * the discovery operation, and to notify any listeners that the discovery results may have
-   * changed.
-   * @param results The discovery results returned by the `discover` function.
-   */
-  protected abstract update(results: T): void;
+  protected abstract discover(): Promise<void>;
 }

extensions/ql-vscode/src/common/events.ts

@@ -1,10 +1,10 @@
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 
 export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;
 }
 
-export interface AppEventEmitter<T> {
+export interface AppEventEmitter<T> extends Disposable {
   event: AppEvent<T>;
   fire(data: T): void;
 }

extensions/ql-vscode/src/common/file-tree-nodes.ts

@@ -0,0 +1,122 @@
+import { basename, dirname, join } from "path";
+import { EnvironmentContext } from "./app";
+
+/**
+ * A node in the tree of files. This will be either a `FileTreeDirectory` or a `FileTreeLeaf`.
+ */
+export abstract class FileTreeNode<T = undefined> {
+  constructor(
+    private _path: string,
+    private _name: string,
+    private _data?: T,
+  ) {}
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public get data(): T | undefined {
+    return this._data;
+  }
+
+  public abstract get children(): ReadonlyArray<FileTreeNode<T>>;
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more files or other directories.
+ */
+export class FileTreeDirectory<T = undefined> extends FileTreeNode<T> {
+  constructor(
+    _path: string,
+    _name: string,
+    protected readonly env: EnvironmentContext,
+    private _children: Array<FileTreeNode<T>> = [],
+  ) {
+    super(_path, _name);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return this._children;
+  }
+
+  public addChild(child: FileTreeNode<T>): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): FileTreeDirectory<T> {
+    if (relativePath === ".") {
+      return this;
+    }
+    const dirName = dirname(relativePath);
+    if (dirName === ".") {
+      return this.createChildDirectory(relativePath);
+    } else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    // remove empty directories
+    this._children.filter(
+      (child) => child instanceof FileTreeLeaf || child.children.length > 0,
+    );
+    this._children.sort((a, b) =>
+      a.name.localeCompare(b.name, this.env.language),
+    );
+    this._children.forEach((child, i) => {
+      child.finish();
+      if (
+        child.children?.length === 1 &&
+        child.children[0] instanceof FileTreeDirectory
+      ) {
+        // collapse children
+        const replacement = new FileTreeDirectory<T>(
+          child.children[0].path,
+          `${child.name} / ${child.children[0].name}`,
+          this.env,
+          Array.from(child.children[0].children),
+        );
+        this._children[i] = replacement;
+      }
+    });
+  }
+
+  private createChildDirectory(name: string): FileTreeDirectory<T> {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return existingChild as FileTreeDirectory<T>;
+    } else {
+      const newChild = new FileTreeDirectory<T>(
+        join(this.path, name),
+        name,
+        this.env,
+      );
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single file.
+ */
+export class FileTreeLeaf<T = undefined> extends FileTreeNode<T> {
+  constructor(_path: string, _name: string, _data?: T) {
+    super(_path, _name, _data);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return [];
+  }
+
+  public finish(): void {
+    /**/
+  }
+}

extensions/ql-vscode/src/common/files.ts

@@ -0,0 +1,129 @@
+import { pathExists, stat, readdir, opendir } from "fs-extra";
+import { isAbsolute, join, relative, resolve } from "path";
+import { tmpdir as osTmpdir } from "os";
+
+/**
+ * Recursively finds all .ql files in this set of Uris.
+ *
+ * @param paths The list of Uris to search through
+ *
+ * @returns list of ql files and a boolean describing whether or not a directory was found/
+ */
+export async function gatherQlFiles(
+  paths: string[],
+): Promise<[string[], boolean]> {
+  const gatheredUris: Set<string> = new Set();
+  let dirFound = false;
+  for (const nextPath of paths) {
+    if ((await pathExists(nextPath)) && (await stat(nextPath)).isDirectory()) {
+      dirFound = true;
+      const subPaths = await readdir(nextPath);
+      const fullPaths = subPaths.map((p) => join(nextPath, p));
+      const nestedFiles = (await gatherQlFiles(fullPaths))[0];
+      nestedFiles.forEach((nested) => gatheredUris.add(nested));
+    } else if (nextPath.endsWith(".ql")) {
+      gatheredUris.add(nextPath);
+    }
+  }
+  return [Array.from(gatheredUris), dirFound];
+}
+
+/**
+ * Lists the names of directories inside the given path.
+ * @param path The path to the directory to read.
+ * @returns the names of the directories inside the given path.
+ */
+export async function getDirectoryNamesInsidePath(
+  path: string,
+): Promise<string[]> {
+  if (!(await pathExists(path))) {
+    throw Error(`Path does not exist: ${path}`);
+  }
+  if (!(await stat(path)).isDirectory()) {
+    throw Error(`Path is not a directory: ${path}`);
+  }
+
+  const dirItems = await readdir(path, { withFileTypes: true });
+
+  const dirNames = dirItems
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => dirent.name);
+
+  return dirNames;
+}
+
+export function normalizePath(path: string): string {
+  // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
+  // to normalize the paths to ensure they all get resolved to the
+  // same format. On Windows, we also need to do the comparison
+  // case-insensitively.
+  path = resolve(path);
+  if (process.platform === "win32") {
+    path = path.toLowerCase();
+  }
+  return path;
+}
+
+export function pathsEqual(path1: string, path2: string): boolean {
+  return normalizePath(path1) === normalizePath(path2);
+}
+
+/**
+ * Returns true if `parent` contains `child`, or if they are equal.
+ */
+export function containsPath(parent: string, child: string): boolean {
+  const relativePath = relative(parent, child);
+  return (
+    !relativePath.startsWith("..") &&
+    // On windows, if the two paths are in different drives, then the
+    // relative path will be an absolute path to the other drive.
+    !isAbsolute(relativePath)
+  );
+}
+
+export async function readDirFullPaths(path: string): Promise<string[]> {
+  const baseNames = await readdir(path);
+  return baseNames.map((baseName) => join(path, baseName));
+}
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(
+  dir: string,
+): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await opendir(dir)) {
+    const entry = join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}
+
+/**
+ * Error thrown from methods from the `fs` module.
+ *
+ * In practice, any error matching this is likely an instance of `NodeJS.ErrnoException`.
+ * If desired in the future, we could model more fields or use `NodeJS.ErrnoException` directly.
+ */
+export interface IOError {
+  readonly code: string;
+}
+
+export function isIOError(e: any): e is IOError {
+  return e.code !== undefined && typeof e.code === "string";
+}
+
+// This function is a wrapper around `os.tmpdir()` to make it easier to mock in tests.
+export function tmpdir(): string {
+  return osTmpdir();
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -1,4 +1,4 @@
-import { OWNER_REGEX, REPO_REGEX } from "../pure/helpers-pure";
+import { OWNER_REGEX, REPO_REGEX } from "./helpers-pure";
 
 /**
  * Checks if a string is a valid GitHub NWO.

extensions/ql-vscode/src/common/index.ts

@@ -1 +0,0 @@
-export * from "./logging";

extensions/ql-vscode/src/common/interface-types.ts

@@ -5,18 +5,22 @@ import {
   ResultSetSchema,
   Column,
   ResolvableLocationValue,
-} from "./bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import {
   VariantAnalysis,
   VariantAnalysisScannedRepositoryResult,
   VariantAnalysisScannedRepositoryState,
 } from "../variant-analysis/shared/variant-analysis";
-import { RepositoriesFilterSortStateWithIds } from "./variant-analysis-filter-sort";
-import { ErrorLike } from "./errors";
+import {
+  RepositoriesFilterSortState,
+  RepositoriesFilterSortStateWithIds,
+} from "../variant-analysis/shared/variant-analysis-filter-sort";
+import { ErrorLike } from "../common/errors";
 import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
 import { ExternalApiUsage } from "../data-extensions-editor/external-api-usage";
 import { ModeledMethod } from "../data-extensions-editor/modeled-method";
 import { DataExtensionEditorViewState } from "../data-extensions-editor/shared/view-state";
+import { Mode } from "../data-extensions-editor/shared/mode";
 
 /**
  * This module contains types and code that are shared between
@@ -72,11 +76,9 @@ export type GraphInterpretationData = {
   dot: string[];
 };
 
-export type InterpretationData =
-  | SarifInterpretationData
-  | GraphInterpretationData;
+type InterpretationData = SarifInterpretationData | GraphInterpretationData;
 
-export interface InterpretationT<T> {
+interface InterpretationT<T> {
   sourceLocationPrefix: string;
   numTruncatedResults: number;
   numTotalResults: number;
@@ -102,7 +104,7 @@ export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
  *
  * As a result of receiving this message, listeners might want to display a loading indicator.
  */
-export interface ResultsUpdatingMsg {
+interface ResultsUpdatingMsg {
   t: "resultsUpdating";
 }
 
@@ -110,7 +112,7 @@ export interface ResultsUpdatingMsg {
  * Message to set the initial state of the results view with a new
  * query.
  */
-export interface SetStateMsg {
+interface SetStateMsg {
   t: "setState";
   resultsPath: string;
   origResultsPaths: ResultsPaths;
@@ -139,7 +141,7 @@ export interface SetStateMsg {
  * Message indicating that the results view should display interpreted
  * results.
  */
-export interface ShowInterpretedPageMsg {
+interface ShowInterpretedPageMsg {
   t: "showInterpretedPage";
   interpretation: Interpretation;
   database: DatabaseInfo;
@@ -169,7 +171,7 @@ export interface NavigateMsg {
  * A message indicating that the results view should untoggle the
  * "Show results in Problems view" checkbox.
  */
-export interface UntoggleShowProblemsMsg {
+interface UntoggleShowProblemsMsg {
   t: "untoggleShowProblems";
 }
 
@@ -199,7 +201,7 @@ export type FromResultsViewMsg =
  * Message from the results view to open a database source
  * file at the provided location.
  */
-export interface ViewSourceFileMsg {
+interface ViewSourceFileMsg {
   t: "viewSourceFile";
   loc: ResolvableLocationValue;
   databaseUri: string;
@@ -208,7 +210,7 @@ export interface ViewSourceFileMsg {
 /**
  * Message from the results view to open a file in an editor.
  */
-export interface OpenFileMsg {
+interface OpenFileMsg {
   t: "openFile";
   /* Full path to the file to open. */
   filePath: string;
@@ -270,7 +272,7 @@ export interface RawResultsSortState {
   sortDirection: SortDirection;
 }
 
-export type InterpretedResultsSortColumn = "alert-message";
+type InterpretedResultsSortColumn = "alert-message";
 
 export interface InterpretedResultsSortState {
   sortBy: InterpretedResultsSortColumn;
@@ -314,7 +316,7 @@ export type FromCompareViewMessage =
 /**
  * Message from the compare view to request opening a query.
  */
-export interface OpenQueryMessage {
+interface OpenQueryMessage {
   readonly t: "openQuery";
   readonly kind: "from" | "to";
 }
@@ -402,63 +404,69 @@ export interface ParsedResultSets {
   resultSet: ResultSet;
 }
 
-export interface SetVariantAnalysisMessage {
+interface SetVariantAnalysisMessage {
   t: "setVariantAnalysis";
   variantAnalysis: VariantAnalysis;
 }
 
+interface SetFilterSortStateMessage {
+  t: "setFilterSortState";
+  filterSortState: RepositoriesFilterSortState;
+}
+
 export type VariantAnalysisState = {
   variantAnalysisId: number;
 };
 
-export interface SetRepoResultsMessage {
+interface SetRepoResultsMessage {
   t: "setRepoResults";
   repoResults: VariantAnalysisScannedRepositoryResult[];
 }
 
-export interface SetRepoStatesMessage {
+interface SetRepoStatesMessage {
   t: "setRepoStates";
   repoStates: VariantAnalysisScannedRepositoryState[];
 }
 
-export interface RequestRepositoryResultsMessage {
+interface RequestRepositoryResultsMessage {
   t: "requestRepositoryResults";
   repositoryFullName: string;
 }
 
-export interface OpenQueryFileMessage {
+interface OpenQueryFileMessage {
   t: "openQueryFile";
 }
 
-export interface OpenQueryTextMessage {
+interface OpenQueryTextMessage {
   t: "openQueryText";
 }
 
-export interface CopyRepositoryListMessage {
+interface CopyRepositoryListMessage {
   t: "copyRepositoryList";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface ExportResultsMessage {
+interface ExportResultsMessage {
   t: "exportResults";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface OpenLogsMessage {
+interface OpenLogsMessage {
   t: "openLogs";
 }
 
-export interface CancelVariantAnalysisMessage {
+interface CancelVariantAnalysisMessage {
   t: "cancelVariantAnalysis";
 }
 
-export interface ShowDataFlowPathsMessage {
+interface ShowDataFlowPathsMessage {
   t: "showDataFlowPaths";
   dataFlowPaths: DataFlowPaths;
 }
 
 export type ToVariantAnalysisMessage =
   | SetVariantAnalysisMessage
+  | SetFilterSortStateMessage
   | SetRepoResultsMessage
   | SetRepoStatesMessage;
 
@@ -473,7 +481,7 @@ export type FromVariantAnalysisMessage =
   | CancelVariantAnalysisMessage
   | ShowDataFlowPathsMessage;
 
-export interface SetDataFlowPathsMessage {
+interface SetDataFlowPathsMessage {
   t: "setDataFlowPaths";
   dataFlowPaths: DataFlowPaths;
 }
@@ -482,69 +490,82 @@ export type ToDataFlowPathsMessage = SetDataFlowPathsMessage;
 
 export type FromDataFlowPathsMessage = CommonFromViewMessages;
 
-export interface SetExtensionPackStateMessage {
+interface SetExtensionPackStateMessage {
   t: "setDataExtensionEditorViewState";
   viewState: DataExtensionEditorViewState;
 }
 
-export interface SetExternalApiUsagesMessage {
+interface SetExternalApiUsagesMessage {
   t: "setExternalApiUsages";
   externalApiUsages: ExternalApiUsage[];
 }
 
-export interface ShowProgressMessage {
-  t: "showProgress";
-  step: number;
-  maxStep: number;
-  message: string;
+interface LoadModeledMethodsMessage {
+  t: "loadModeledMethods";
+  modeledMethods: Record<string, ModeledMethod>;
 }
 
-export interface AddModeledMethodsMessage {
+interface AddModeledMethodsMessage {
   t: "addModeledMethods";
   modeledMethods: Record<string, ModeledMethod>;
-
-  /**
-   * If true, then any existing modeled methods set to "none" will be
-   * overwritten by the new modeled methods. Otherwise, the "none" modeled
-   * methods will not be overwritten, even if the new modeled methods
-   * contain a better model.
-   */
-  overrideNone?: boolean;
 }
 
-export interface JumpToUsageMessage {
+interface SwitchModeMessage {
+  t: "switchMode";
+  mode: Mode;
+}
+
+interface JumpToUsageMessage {
   t: "jumpToUsage";
   location: ResolvableLocationValue;
 }
 
-export interface OpenExtensionPackMessage {
+interface OpenDatabaseMessage {
+  t: "openDatabase";
+}
+
+interface OpenExtensionPackMessage {
   t: "openExtensionPack";
 }
 
-export interface OpenModelFileMessage {
-  t: "openModelFile";
+interface RefreshExternalApiUsages {
+  t: "refreshExternalApiUsages";
 }
 
-export interface SaveModeledMethods {
+interface SaveModeledMethods {
   t: "saveModeledMethods";
   externalApiUsages: ExternalApiUsage[];
   modeledMethods: Record<string, ModeledMethod>;
 }
 
-export interface GenerateExternalApiMessage {
+interface GenerateExternalApiMessage {
   t: "generateExternalApi";
 }
 
+interface GenerateExternalApiFromLlmMessage {
+  t: "generateExternalApiFromLlm";
+  externalApiUsages: ExternalApiUsage[];
+  modeledMethods: Record<string, ModeledMethod>;
+}
+
+interface ModelDependencyMessage {
+  t: "modelDependency";
+}
+
 export type ToDataExtensionsEditorMessage =
   | SetExtensionPackStateMessage
   | SetExternalApiUsagesMessage
-  | ShowProgressMessage
+  | LoadModeledMethodsMessage
   | AddModeledMethodsMessage;
 
 export type FromDataExtensionsEditorMessage =
   | ViewLoadedMsg
-  | OpenModelFileMessage
+  | SwitchModeMessage
+  | RefreshExternalApiUsages
+  | OpenDatabaseMessage
   | OpenExtensionPackMessage
   | JumpToUsageMessage
   | SaveModeledMethods
-  | GenerateExternalApiMessage;
+  | GenerateExternalApiMessage
+  | GenerateExternalApiFromLlmMessage
+  | ModelDependencyMessage;

extensions/ql-vscode/src/common/invocation-rate-limiter.ts

@@ -0,0 +1,89 @@
+import { Memento } from "./memento";
+
+/**
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
+ */
+export class InvocationRateLimiter<T> {
+  constructor(
+    private readonly globalState: Memento,
+    private readonly funcIdentifier: string,
+    private readonly func: () => Promise<T>,
+    private readonly createDate: (dateString?: string) => Date = (s) =>
+      s ? new Date(s) : new Date(),
+  ) {}
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(
+    minSecondsSinceLastInvocation: number,
+  ): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this.createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
+        updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this.func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined = this.globalState.get(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+    );
+    return maybeDateString ? this.createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this.globalState.update(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+      date,
+    );
+  }
+
+  private static readonly _invocationRateLimiterPrefix =
+    "invocationRateLimiter_lastInvocationDate_";
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited,
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result,
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited,
+  };
+}

extensions/ql-vscode/src/common/location-link-utils.ts

@@ -0,0 +1,27 @@
+import { FileLink } from "../variant-analysis/shared/analysis-result";
+
+export function createRemoteFileRef(
+  fileLink: FileLink,
+  startLine?: number,
+  endLine?: number,
+  startColumn?: number,
+  endColumn?: number,
+): string {
+  if (
+    startColumn &&
+    endColumn &&
+    startLine &&
+    endLine &&
+    // Verify that location information is valid; otherwise highlighting might be broken
+    ((startLine === endLine && startColumn < endColumn) || startLine < endLine)
+  ) {
+    // This relies on column highlighting of new code view on GitHub
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}C${startColumn}-L${endLine}C${endColumn}`;
+  } else if (startLine && endLine && startLine < endLine) {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}-L${endLine}`;
+  } else if (startLine) {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}`;
+  } else {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}`;
+  }
+}

extensions/ql-vscode/src/common/logging/index.ts

@@ -1,4 +1,4 @@
 export * from "./logger";
+export * from "./notification-logger";
+export * from "./notifications";
 export * from "./tee-logger";
-export * from "./vscode/loggers";
-export * from "./vscode/output-channel-logger";

extensions/ql-vscode/src/common/logging/notification-logger.ts

@@ -0,0 +1,7 @@
+import { Logger } from "./logger";
+
+export interface NotificationLogger extends Logger {
+  showErrorMessage(message: string): Promise<void>;
+  showWarningMessage(message: string): Promise<void>;
+  showInformationMessage(message: string): Promise<void>;
+}

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -0,0 +1,116 @@
+import { NotificationLogger } from "./notification-logger";
+import { AppTelemetry } from "../telemetry";
+import { RedactableError } from "../errors";
+
+interface ShowAndLogOptions {
+  /**
+   * An alternate message that is added to the log, but not displayed in the popup.
+   * This is useful for adding extra detail to the logs that would be too noisy for the popup.
+   */
+  fullMessage?: string;
+}
+
+/**
+ * Show an error message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogErrorMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    dropLinesExceptInitial(message),
+    logger.showErrorMessage,
+    { fullMessage: message, ...options },
+  );
+}
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join("\n");
+}
+
+/**
+ * Show a warning message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogWarningMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showWarningMessage,
+    options,
+  );
+}
+
+/**
+ * Show an information message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogInformationMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showInformationMessage,
+    options,
+  );
+}
+
+async function internalShowAndLog(
+  logger: NotificationLogger,
+  message: string,
+  fn: (message: string) => Promise<void>,
+  { fullMessage }: ShowAndLogOptions = {},
+): Promise<void> {
+  void logger.log(fullMessage || message);
+  await fn.bind(logger)(message);
+}
+
+interface ShowAndLogExceptionOptions extends ShowAndLogOptions {
+  /** Custom properties to include in the telemetry report. */
+  extraTelemetryProperties?: { [key: string]: string };
+}
+
+/**
+ * Show an error message, log it to the console, and emit redacted information as telemetry
+ *
+ * @param logger The logger that will receive the message.
+ * @param telemetry The telemetry instance to use for reporting.
+ * @param error The error to show. Only redacted information will be included in the telemetry.
+ * @param options See individual fields on `ShowAndLogExceptionOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogExceptionWithTelemetry(
+  logger: NotificationLogger,
+  telemetry: AppTelemetry | undefined,
+  error: RedactableError,
+  options: ShowAndLogExceptionOptions = {},
+): Promise<void> {
+  telemetry?.sendError(error, options.extraTelemetryProperties);
+  return showAndLogErrorMessage(logger, error.fullMessage, options);
+}

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -1,6 +1,6 @@
 import { appendFile, ensureFile } from "fs-extra";
 import { isAbsolute } from "path";
-import { getErrorMessage } from "../../pure/helpers-pure";
+import { getErrorMessage } from "../helpers-pure";
 import { Logger, LogOptions } from "./logger";
 
 /**

extensions/ql-vscode/src/common/logging/vscode/index.ts

@@ -0,0 +1,2 @@
+export * from "./loggers";
+export * from "./output-channel-logger";

extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts

@@ -1,11 +1,15 @@
 import { window as Window, OutputChannel, Progress } from "vscode";
 import { Logger, LogOptions } from "../logger";
-import { DisposableObject } from "../../../pure/disposable-object";
+import { DisposableObject } from "../../disposable-object";
+import { NotificationLogger } from "../notification-logger";
 
 /**
  * A logger that writes messages to an output channel in the VS Code Output tab.
  */
-export class OutputChannelLogger extends DisposableObject implements Logger {
+export class OutputChannelLogger
+  extends DisposableObject
+  implements Logger, NotificationLogger
+{
   public readonly outputChannel: OutputChannel;
   isCustomLogDirectory: boolean;
 
@@ -42,6 +46,30 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   show(preserveFocus?: boolean): void {
     this.outputChannel.show(preserveFocus);
   }
+
+  async showErrorMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showErrorMessage);
+  }
+
+  async showInformationMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showInformationMessage);
+  }
+
+  async showWarningMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showWarningMessage);
+  }
+
+  private async showMessage(
+    message: string,
+    show: (message: string, ...items: string[]) => Thenable<string | undefined>,
+  ): Promise<void> {
+    const label = "Show Log";
+    const result = await show(message, label);
+
+    if (result === label) {
+      this.show();
+    }
+  }
 }
 
 export type ProgressReporter = Progress<{ message: string }>;

extensions/ql-vscode/src/common/path.ts

@@ -19,3 +19,11 @@ export const basename = (path: string): string => {
   const index = path.lastIndexOf("\\");
   return index === -1 ? path : path.slice(index + 1);
 };
+
+// Returns the extension of a path, including the leading dot.
+export const extname = (path: string): string => {
+  const name = basename(path);
+
+  const index = name.lastIndexOf(".");
+  return index === -1 ? "" : name.slice(index);
+};

extensions/ql-vscode/src/common/query-language.ts

@@ -9,6 +9,29 @@ export enum QueryLanguage {
   Swift = "swift",
 }
 
+export function getLanguageDisplayName(language: string): string {
+  switch (language) {
+    case QueryLanguage.CSharp:
+      return "C#";
+    case QueryLanguage.Cpp:
+      return "C / C++";
+    case QueryLanguage.Go:
+      return "Go";
+    case QueryLanguage.Java:
+      return "Java";
+    case QueryLanguage.Javascript:
+      return "JavaScript";
+    case QueryLanguage.Python:
+      return "Python";
+    case QueryLanguage.Ruby:
+      return "Ruby";
+    case QueryLanguage.Swift:
+      return "Swift";
+    default:
+      return language;
+  }
+}
+
 export const PACKS_BY_QUERY_LANGUAGE = {
   [QueryLanguage.Cpp]: ["codeql/cpp-queries"],
   [QueryLanguage.CSharp]: [
@@ -25,13 +48,17 @@ export const PACKS_BY_QUERY_LANGUAGE = {
   [QueryLanguage.Ruby]: ["codeql/ruby-queries"],
 };
 
-export const dbSchemeToLanguage = {
-  "semmlecode.javascript.dbscheme": "javascript",
-  "semmlecode.cpp.dbscheme": "cpp",
-  "semmlecode.dbscheme": "java",
-  "semmlecode.python.dbscheme": "python",
-  "semmlecode.csharp.dbscheme": "csharp",
-  "go.dbscheme": "go",
-  "ruby.dbscheme": "ruby",
-  "swift.dbscheme": "swift",
+export const dbSchemeToLanguage: Record<string, QueryLanguage> = {
+  "semmlecode.javascript.dbscheme": QueryLanguage.Javascript,
+  "semmlecode.cpp.dbscheme": QueryLanguage.Cpp,
+  "semmlecode.dbscheme": QueryLanguage.Java,
+  "semmlecode.python.dbscheme": QueryLanguage.Python,
+  "semmlecode.csharp.dbscheme": QueryLanguage.CSharp,
+  "go.dbscheme": QueryLanguage.Go,
+  "ruby.dbscheme": QueryLanguage.Ruby,
+  "swift.dbscheme": QueryLanguage.Swift,
 };
+
+export function isQueryLanguage(language: string): language is QueryLanguage {
+  return Object.values(QueryLanguage).includes(language as QueryLanguage);
+}

extensions/ql-vscode/src/common/sarif-parser.ts

@@ -1,7 +1,7 @@
 import * as Sarif from "sarif";
 import { createReadStream } from "fs-extra";
 import { connectTo } from "stream-json/Assembler";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { getErrorMessage } from "./helpers-pure";
 import { withParser } from "stream-json/filters/Pick";
 
 const DUMMY_TOOL: Sarif.Tool = { driver: { name: "" } };

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -1,8 +1,8 @@
 import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
-import { ResolvableLocationValue } from "./bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 
-export interface SarifLink {
+interface SarifLink {
   dest: number;
   text: string;
 }
@@ -24,7 +24,7 @@ type ParsedSarifLocation =
   // that, and is appropriate for display in the UI.
   | NoLocation;
 
-export type SarifMessageComponent = string | SarifLink;
+type SarifMessageComponent = string | SarifLink;
 
 /**
  * Unescape "[", "]" and "\\" like in sarif plain text messages
@@ -203,7 +203,7 @@ export function shouldHighlightLine(
  * A line of code split into: plain text before the highlighted section, the highlighted
  * text itself, and plain text after the highlighted section.
  */
-export interface PartiallyHighlightedLine {
+interface PartiallyHighlightedLine {
   plainSection1: string;
   highlightedSection: string;
   plainSection2: string;

extensions/ql-vscode/src/common/telemetry.ts

@@ -0,0 +1,10 @@
+import { RedactableError } from "./errors";
+
+export interface AppTelemetry {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void;
+  sendUIInteraction(name: string): void;
+  sendError(
+    error: RedactableError,
+    extraProperties?: { [key: string]: string },
+  ): void;
+}

extensions/ql-vscode/src/common/time.ts

@@ -2,16 +2,16 @@
  * Contains an assortment of helper constants and functions for working with time, dates, and durations.
  */
 
-export const ONE_SECOND_IN_MS = 1000;
-export const ONE_MINUTE_IN_MS = ONE_SECOND_IN_MS * 60;
+const ONE_SECOND_IN_MS = 1000;
+const ONE_MINUTE_IN_MS = ONE_SECOND_IN_MS * 60;
 export const ONE_HOUR_IN_MS = ONE_MINUTE_IN_MS * 60;
 export const TWO_HOURS_IN_MS = ONE_HOUR_IN_MS * 2;
 export const THREE_HOURS_IN_MS = ONE_HOUR_IN_MS * 3;
 export const ONE_DAY_IN_MS = ONE_HOUR_IN_MS * 24;
 
 // These are approximations
-export const ONE_MONTH_IN_MS = ONE_DAY_IN_MS * 30;
-export const ONE_YEAR_IN_MS = ONE_DAY_IN_MS * 365;
+const ONE_MONTH_IN_MS = ONE_DAY_IN_MS * 30;
+const ONE_YEAR_IN_MS = ONE_DAY_IN_MS * 365;
 
 const durationFormatter = new Intl.RelativeTimeFormat("en", {
   numeric: "auto",

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -9,13 +9,9 @@ import {
 } from "vscode";
 import { join } from "path";
 
-import { DisposableObject, DisposeHandler } from "./pure/disposable-object";
-import { tmpDir } from "./helpers";
-import {
-  getHtmlForWebview,
-  WebviewMessage,
-  WebviewView,
-} from "./interface-utils";
+import { DisposableObject, DisposeHandler } from "../disposable-object";
+import { tmpDir } from "../../tmp-dir";
+import { getHtmlForWebview, WebviewMessage, WebviewView } from "./webview-html";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -23,6 +19,7 @@ export type WebviewPanelConfig = {
   viewColumn: ViewColumn;
   view: WebviewView;
   preserveFocus?: boolean;
+  iconPath?: Uri | { dark: Uri; light: Uri };
   additionalOptions?: WebviewPanelOptions & WebviewOptions;
   allowWasmEval?: boolean;
 };
@@ -90,6 +87,8 @@ export abstract class AbstractWebview<
       );
       this.panel = panel;
 
+      this.panel.iconPath = config.iconPath;
+
       this.setupPanel(panel, config);
 
       this.panelResolves.forEach((resolve) => resolve(panel));

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -1,20 +1,23 @@
 import { pathExists } from "fs-extra";
 import * as unzipper from "unzipper";
 import * as vscode from "vscode";
-import { extLogger } from "./common";
+import { extLogger } from "../logging/vscode";
 
 // All path operations in this file must be on paths *within* the zip
 // archive.
 import { posix } from "path";
 const path = posix;
 
-export class File implements vscode.FileStat {
+class File implements vscode.FileStat {
   type: vscode.FileType;
   ctime: number;
   mtime: number;
   size: number;
 
-  constructor(public name: string, public data: Uint8Array) {
+  constructor(
+    public name: string,
+    public data: Uint8Array,
+  ) {
     this.type = vscode.FileType.File;
     this.ctime = Date.now();
     this.mtime = Date.now();
@@ -23,7 +26,7 @@ export class File implements vscode.FileStat {
   }
 }
 
-export class Directory implements vscode.FileStat {
+class Directory implements vscode.FileStat {
   type: vscode.FileType;
   ctime: number;
   mtime: number;
@@ -38,7 +41,7 @@ export class Directory implements vscode.FileStat {
   }
 }
 
-export type Entry = File | Directory;
+type Entry = File | Directory;
 
 /**
  * A map containing directory hierarchy information in a convenient form.
@@ -49,7 +52,7 @@ export type Entry = File | Directory;
  * dirMap['/foo'] = {'bar': vscode.FileType.Directory}
  * dirMap['/foo/bar'] = {'baz': vscode.FileType.File}
  */
-export type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
+type DirectoryHierarchyMap = Map<string, Map<string, vscode.FileType>>;
 
 export type ZipFileReference = {
   sourceArchiveZipPath: string;

extensions/ql-vscode/src/common/vscode/authentication.ts

@@ -1,9 +1,9 @@
 import * as vscode from "vscode";
 import * as Octokit from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
-import { Credentials } from "./common/authentication";
+import { Credentials } from "../authentication";
 
-const GITHUB_AUTH_PROVIDER_ID = "github";
+export const GITHUB_AUTH_PROVIDER_ID = "github";
 
 // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist,
 // and 'read:packages' for reading private CodeQL packages.

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -1,18 +1,20 @@
 import { commands, Disposable } from "vscode";
 import { CommandFunction, CommandManager } from "../../packages/commands";
-import { extLogger, OutputChannelLogger } from "../logging";
+import {
+  NotificationLogger,
+  showAndLogWarningMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../logging";
+import { extLogger } from "../logging/vscode";
 import {
   asError,
   getErrorMessage,
   getErrorStack,
-} from "../../pure/helpers-pure";
-import { redactableError } from "../../pure/errors";
-import { UserCancellationException } from "../../progress";
-import {
-  showAndLogExceptionWithTelemetry,
-  showAndLogWarningMessage,
-} from "../../helpers";
-import { telemetryListener } from "../../telemetry";
+} from "../../common/helpers-pure";
+import { redactableError } from "../../common/errors";
+import { UserCancellationException } from "./progress";
+import { telemetryListener } from "./telemetry";
+import { AppTelemetry } from "../telemetry";
 
 /**
  * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -20,9 +22,12 @@ import { telemetryListener } from "../../telemetry";
  */
 export function createVSCodeCommandManager<
   Commands extends Record<string, CommandFunction>,
->(outputLogger?: OutputChannelLogger): CommandManager<Commands> {
+>(
+  logger?: NotificationLogger,
+  telemetry?: AppTelemetry,
+): CommandManager<Commands> {
   return new CommandManager((commandId, task) => {
-    return registerCommandWithErrorHandling(commandId, task, outputLogger);
+    return registerCommandWithErrorHandling(commandId, task, logger, telemetry);
   }, wrapExecuteCommand);
 }
 
@@ -32,11 +37,14 @@ export function createVSCodeCommandManager<
  * @param commandId The ID of the command to register.
  * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
  * arguments to the command handler are passed on to the task.
+ * @param logger The logger to use for error reporting.
+ * @param telemetry The telemetry listener to use for error reporting.
  */
 export function registerCommandWithErrorHandling(
   commandId: string,
   task: (...args: any[]) => Promise<any>,
-  outputLogger = extLogger,
+  logger: NotificationLogger = extLogger,
+  telemetry: AppTelemetry | undefined = telemetryListener,
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     const startTime = Date.now();
@@ -49,23 +57,20 @@ export function registerCommandWithErrorHandling(
       const errorMessage = redactableError(error)`${
         getErrorMessage(e) || e
       } (${commandId})`;
-      const errorStack = getErrorStack(e);
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          void outputLogger.log(errorMessage.fullMessage);
+          void logger.log(errorMessage.fullMessage);
         } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage, {
-            outputLogger,
-          });
+          void showAndLogWarningMessage(logger, errorMessage.fullMessage);
         }
       } else {
         // Include the full stack in the error log only.
+        const errorStack = getErrorStack(e);
         const fullMessage = errorStack
           ? `${errorMessage.fullMessage}\n${errorStack}`
           : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          outputLogger,
+        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
           fullMessage,
           extraTelemetryProperties: {
             command: commandId,

extensions/ql-vscode/src/common/vscode/dialog.ts

@@ -0,0 +1,135 @@
+import { env, Uri, window } from "vscode";
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+): Promise<boolean | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: false };
+  const noItem = { title: noTitle, isCloseAffordance: true };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+  );
+  if (!chosenItem) {
+    return undefined;
+  }
+  return chosenItem?.title === yesItem.title;
+}
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(
+  message: string,
+  url: string,
+): Promise<boolean | undefined> {
+  const urlItem = { title: "More Information", isCloseAffordance: false };
+  const yesItem = { title: "Yes", isCloseAffordance: false };
+  const noItem = { title: "No", isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await window.showInformationMessage(
+      message,
+      { modal: true },
+      urlItem,
+      yesItem,
+      noItem,
+    );
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
+/**
+ * Show an information message with a customisable action.
+ * @param message The message to show.
+ * @param actionMessage The call to action message.
+ *
+ * @return `true` if the user clicks the action, `false` if the user cancels the dialog.
+ */
+export async function showInformationMessageWithAction(
+  message: string,
+  actionMessage: string,
+): Promise<boolean> {
+  const actionItem = { title: actionMessage, isCloseAffordance: false };
+  const chosenItem = await window.showInformationMessage(message, actionItem);
+  return chosenItem === actionItem;
+}
+
+/**
+ * Opens a modal dialog for the user to make a choice between yes/no/never be asked again.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ * @param neverTitle The text in the box indicating the opt out choice.
+ *
+ * @return
+ *  `Yes` if the user clicks 'Yes',
+ *  `No` if the user clicks 'No' or cancels the dialog,
+ *  `No, and never ask me again` if the user clicks 'No, and never ask me again',
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showNeverAskAgainDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+  neverAskAgainTitle = "No, and never ask me again",
+): Promise<string | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: true };
+  const noItem = { title: noTitle, isCloseAffordance: false };
+  const neverAskAgainItem = {
+    title: neverAskAgainTitle,
+    isCloseAffordance: false,
+  };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+    neverAskAgainItem,
+  );
+
+  return chosenItem?.title;
+}

extensions/ql-vscode/src/common/vscode/environment-context.ts

@@ -0,0 +1,8 @@
+import { env } from "vscode";
+import { EnvironmentContext } from "../app";
+
+export class AppEnvironmentContext implements EnvironmentContext {
+  public get language(): string {
+    return env.language;
+  }
+}

extensions/ql-vscode/src/common/vscode/external-files.ts

@@ -1,11 +1,15 @@
 import { Uri, window } from "vscode";
-import { AppCommandManager } from "../common/commands";
+import { AppCommandManager } from "../commands";
+import { showBinaryChoiceDialog } from "./dialog";
+import { redactableError } from "../../common/errors";
 import {
-  showAndLogExceptionWithTelemetry,
-  showBinaryChoiceDialog,
-} from "../helpers";
-import { redactableError } from "../pure/errors";
-import { asError, getErrorMessage, getErrorStack } from "../pure/helpers-pure";
+  asError,
+  getErrorMessage,
+  getErrorStack,
+} from "../../common/helpers-pure";
+import { showAndLogExceptionWithTelemetry } from "../logging";
+import { extLogger } from "../logging/vscode";
+import { telemetryListener } from "./telemetry";
 
 export async function tryOpenExternalFile(
   commandManager: AppCommandManager,
@@ -32,6 +36,8 @@ the file in the file explorer and dragging it into the workspace.`,
           await commandManager.execute("revealFileInOS", uri);
         } catch (e) {
           void showAndLogExceptionWithTelemetry(
+            extLogger,
+            telemetryListener,
             redactableError(
               asError(e),
             )`Failed to reveal file in OS: ${getErrorMessage(e)}`,
@@ -40,6 +46,8 @@ the file in the file explorer and dragging it into the workspace.`,
       }
     } else {
       void showAndLogExceptionWithTelemetry(
+        extLogger,
+        telemetryListener,
         redactableError(asError(e))`Could not open file ${fileLocation}`,
         {
           fullMessage: `${getErrorMessage(e)}\n${getErrorStack(e)}`,

extensions/ql-vscode/src/common/vscode/file-path-discovery.ts

@@ -0,0 +1,272 @@
+import { Discovery } from "../discovery";
+import {
+  Event,
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFoldersChangeEvent,
+  workspace,
+} from "vscode";
+import { MultiFileSystemWatcher } from "./multi-file-system-watcher";
+import { AppEventEmitter } from "../events";
+import { extLogger } from "../logging/vscode";
+import { lstat } from "fs-extra";
+import { containsPath, isIOError } from "../files";
+import {
+  getOnDiskWorkspaceFolders,
+  getOnDiskWorkspaceFoldersObjects,
+} from "./workspace-folders";
+import { getErrorMessage } from "../../common/helpers-pure";
+
+interface PathData {
+  path: string;
+}
+
+/**
+ * Discovers and watches for changes to all files matching a given filter
+ * contained in the workspace. Also allows computing extra data about each
+ * file path, and only recomputing the data when the file changes.
+ *
+ * Scans the whole workspace on startup, and then watches for changes to files
+ * to do the minimum work to keep up with changes.
+ *
+ * Can configure which changes it watches for, which files are considered
+ * relevant, and what extra data to compute for each file.
+ */
+export abstract class FilePathDiscovery<T extends PathData> extends Discovery {
+  /**
+   * Has `discover` been called. This allows distinguishing between
+   * "no paths found" and not having scanned yet.
+   */
+  private discoverHasCompletedOnce = false;
+
+  /** The set of known paths and associated data that we are tracking */
+  private pathData: T[] = [];
+
+  /** Event that fires whenever the contents of `pathData` changes */
+  private readonly onDidChangePathDataEmitter: AppEventEmitter<void>;
+
+  /**
+   * The set of file paths that may have changed on disk since the last time
+   * refresh was run. Whenever a watcher reports some change to a file we add
+   * it to this set, and then during the next refresh we will process all
+   * file paths from this set and update our internal state to match whatever
+   * we find on disk (i.e. the file exists, doesn't exist, computed data has
+   * changed).
+   */
+  private readonly changedFilePaths = new Set<string>();
+
+  /**
+   * Watches for changes to files and directories in all workspace folders.
+   */
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  /**
+   * @param name Name of the discovery operation, for logging purposes.
+   * @param fileWatchPattern Passed to `vscode.RelativePattern` to determine the files to watch for changes to.
+   */
+  constructor(
+    name: string,
+    private readonly fileWatchPattern: string,
+  ) {
+    super(name, extLogger);
+
+    this.onDidChangePathDataEmitter = this.push(new EventEmitter<void>());
+    this.push(
+      workspace.onDidChangeWorkspaceFolders(
+        this.workspaceFoldersChanged.bind(this),
+      ),
+    );
+    this.push(this.watcher.onDidChange(this.fileChanged.bind(this)));
+  }
+
+  protected getPathData(): ReadonlyArray<Readonly<T>> | undefined {
+    if (!this.discoverHasCompletedOnce) {
+      return undefined;
+    }
+    return this.pathData;
+  }
+
+  protected get onDidChangePathData(): Event<void> {
+    return this.onDidChangePathDataEmitter.event;
+  }
+
+  /**
+   * Compute any extra data to be stored regarding the given path.
+   */
+  protected abstract getDataForPath(path: string): Promise<T>;
+
+  /**
+   * Is the given path relevant to this discovery operation?
+   */
+  protected abstract pathIsRelevant(path: string): boolean;
+
+  /**
+   * Should the given new data overwrite the existing data we have stored?
+   */
+  protected abstract shouldOverwriteExistingData(
+    newData: T,
+    existingData: T,
+  ): boolean;
+
+  /**
+   * Update the data for every path by calling `getDataForPath`.
+   */
+  protected async recomputeAllData() {
+    this.pathData = await Promise.all(
+      this.pathData.map((p) => this.getDataForPath(p.path)),
+    );
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  /**
+   * Do the initial scan of the entire workspace and set up watchers for future changes.
+   */
+  public async initialRefresh() {
+    getOnDiskWorkspaceFolders().forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder);
+    });
+
+    this.updateWatchers();
+    await this.refresh();
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  private workspaceFoldersChanged(event: WorkspaceFoldersChangeEvent) {
+    event.added.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+    event.removed.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+
+    this.updateWatchers();
+    void this.refresh();
+  }
+
+  private updateWatchers() {
+    this.watcher.clear();
+    for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
+      // Watch for changes to individual files
+      this.watcher.addWatch(
+        new RelativePattern(workspaceFolder, this.fileWatchPattern),
+      );
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(workspaceFolder, "**/"));
+    }
+  }
+
+  private fileChanged(uri: Uri) {
+    this.changedFilePaths.add(uri.fsPath);
+    void this.refresh();
+  }
+
+  protected async discover() {
+    let pathsUpdated = false;
+    for (const path of this.changedFilePaths) {
+      try {
+        this.changedFilePaths.delete(path);
+        if (await this.handleChangedPath(path)) {
+          pathsUpdated = true;
+        }
+      } catch (e) {
+        // If we get an error while processing a path, just log it and continue.
+        // There aren't any network operations happening here or anything else
+        // that's likely to succeed on a retry, so don't bother adding it back
+        // to the changedFilePaths set.
+        void extLogger.log(
+          `${
+            this.name
+          } failed while processing path "${path}": ${getErrorMessage(e)}`,
+        );
+      }
+    }
+
+    this.discoverHasCompletedOnce = true;
+    if (pathsUpdated) {
+      this.onDidChangePathDataEmitter.fire();
+    }
+  }
+
+  private async handleChangedPath(path: string): Promise<boolean> {
+    try {
+      // If the path is not in the workspace then we don't want to be
+      // tracking or displaying it, so treat it as if it doesn't exist.
+      if (!this.pathIsInWorkspace(path)) {
+        return this.handleRemovedPath(path);
+      }
+
+      if ((await lstat(path)).isDirectory()) {
+        return await this.handleChangedDirectory(path);
+      } else {
+        return this.handleChangedFile(path);
+      }
+    } catch (e) {
+      if (isIOError(e) && e.code === "ENOENT") {
+        return this.handleRemovedPath(path);
+      }
+      throw e;
+    }
+  }
+
+  private pathIsInWorkspace(path: string): boolean {
+    return getOnDiskWorkspaceFolders().some((workspaceFolder) =>
+      containsPath(workspaceFolder, path),
+    );
+  }
+
+  private handleRemovedPath(path: string): boolean {
+    const oldLength = this.pathData.length;
+    this.pathData = this.pathData.filter(
+      (existingPathData) => !containsPath(path, existingPathData.path),
+    );
+    return this.pathData.length !== oldLength;
+  }
+
+  private async handleChangedDirectory(path: string): Promise<boolean> {
+    const newPaths = await workspace.findFiles(
+      new RelativePattern(path, this.fileWatchPattern),
+    );
+
+    let pathsUpdated = false;
+    for (const path of newPaths) {
+      if (await this.addOrUpdatePath(path.fsPath)) {
+        pathsUpdated = true;
+      }
+    }
+    return pathsUpdated;
+  }
+
+  private async handleChangedFile(path: string): Promise<boolean> {
+    if (this.pathIsRelevant(path)) {
+      return await this.addOrUpdatePath(path);
+    } else {
+      return false;
+    }
+  }
+
+  private async addOrUpdatePath(path: string): Promise<boolean> {
+    const data = await this.getDataForPath(path);
+    const existingPathDataIndex = this.pathData.findIndex(
+      (existingPathData) => existingPathData.path === path,
+    );
+    if (existingPathDataIndex !== -1) {
+      if (
+        this.shouldOverwriteExistingData(
+          data,
+          this.pathData[existingPathDataIndex],
+        )
+      ) {
+        this.pathData.splice(existingPathDataIndex, 1, data);
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      this.pathData.push(data);
+      return true;
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../disposable-object";
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from "vscode";
 
 /**

extensions/ql-vscode/src/common/vscode/progress.ts

@@ -10,7 +10,10 @@ export class UserCancellationException extends Error {
    * @param message The error message
    * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
    */
-  constructor(message?: string, public readonly silent = false) {
+  constructor(
+    message?: string,
+    public readonly silent = false,
+  ) {
     super(message);
   }
 }
@@ -35,7 +38,7 @@ export type ProgressCallback = (p: ProgressUpdate) => void;
 // Make certain properties within a type optional
 type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
 
-export type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
+type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
 
 /**
  * A task that reports progress.

extensions/ql-vscode/src/common/vscode/selection-commands.ts

@@ -0,0 +1,55 @@
+import {
+  ExplorerSelectionCommandFunction,
+  TreeViewContextMultiSelectionCommandFunction,
+  TreeViewContextSingleSelectionCommandFunction,
+} from "../commands";
+import { showAndLogErrorMessage, NotificationLogger } from "../logging";
+
+// A hack to match types that are not an array, which is useful to help avoid
+// misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
+// as DatabaseItem[] instead of DatabaseItem.
+type NotArray = object & { length?: never };
+
+// A way to get the type system to help assert that one type is a supertype of another.
+type CreateSupertypeOf<Super, Sub extends Super> = Sub;
+
+// This asserts that SelectionCommand is assignable to all of the different types of
+// SelectionCommand defined in commands.ts. The intention is the output from the helpers
+// in this file can be used with any of the select command types and can handle any of
+// the inputs.
+type SelectionCommand<T extends NotArray> = CreateSupertypeOf<
+  TreeViewContextMultiSelectionCommandFunction<T> &
+    TreeViewContextSingleSelectionCommandFunction<T> &
+    ExplorerSelectionCommandFunction<T>,
+  (singleItem: T, multiSelect?: T[] | undefined) => Promise<void>
+>;
+
+export function createSingleSelectionCommand<T extends NotArray>(
+  logger: NotificationLogger,
+  f: (argument: T) => Promise<void>,
+  itemName: string,
+): SelectionCommand<T> {
+  return async (singleItem, multiSelect) => {
+    if (multiSelect === undefined || multiSelect.length === 1) {
+      return f(singleItem);
+    } else {
+      void showAndLogErrorMessage(
+        logger,
+        `Please select a single ${itemName}.`,
+      );
+      return;
+    }
+  };
+}
+
+export function createMultiSelectionCommand<T extends NotArray>(
+  f: (argument: T[]) => Promise<void>,
+): SelectionCommand<T> {
+  return async (singleItem, multiSelect) => {
+    if (multiSelect !== undefined && multiSelect.length > 0) {
+      return f(multiSelect);
+    } else {
+      return f([singleItem]);
+    }
+  };
+}

extensions/ql-vscode/src/common/vscode/telemetry.ts

@@ -13,18 +13,19 @@ import {
   LOG_TELEMETRY,
   isIntegrationTestMode,
   isCanary,
-} from "./config";
+} from "../../config";
 import * as appInsights from "applicationinsights";
-import { extLogger } from "./common";
+import { extLogger } from "../logging/vscode";
 import { UserCancellationException } from "./progress";
-import { showBinaryChoiceWithUrlDialog } from "./helpers";
-import { RedactableError } from "./pure/errors";
+import { showBinaryChoiceWithUrlDialog } from "./dialog";
+import { RedactableError } from "../errors";
 import { SemVer } from "semver";
+import { AppTelemetry } from "../telemetry";
 
 // Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
 const key = "REPLACE-APP-INSIGHTS-KEY";
 
-export enum CommandCompletion {
+enum CommandCompletion {
   Success = "Success",
   Failed = "Failed",
   Cancelled = "Cancelled",
@@ -54,7 +55,10 @@ const baseDataPropertiesToRemove = [
 
 const NOT_SET_CLI_VERSION = "not-set";
 
-export class TelemetryListener extends ConfigListener {
+export class ExtensionTelemetryListener
+  extends ConfigListener
+  implements AppTelemetry
+{
   static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
 
   private reporter?: TelemetryReporter;
@@ -152,7 +156,7 @@ export class TelemetryListener extends ConfigListener {
     void this.reporter?.dispose();
   }
 
-  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void {
     if (!this.reporter) {
       return;
     }
@@ -174,7 +178,7 @@ export class TelemetryListener extends ConfigListener {
     );
   }
 
-  sendUIInteraction(name: string) {
+  sendUIInteraction(name: string): void {
     if (!this.reporter) {
       return;
     }
@@ -193,7 +197,7 @@ export class TelemetryListener extends ConfigListener {
   sendError(
     error: RedactableError,
     extraProperties?: { [key: string]: string },
-  ) {
+  ): void {
     if (!this.reporter) {
       return;
     }
@@ -272,16 +276,16 @@ export class TelemetryListener extends ConfigListener {
 /**
  * The global Telemetry instance
  */
-export let telemetryListener: TelemetryListener | undefined;
+export let telemetryListener: ExtensionTelemetryListener | undefined;
 
 export async function initializeTelemetry(
   extension: Extension<any>,
   ctx: ExtensionContext,
-): Promise<TelemetryListener> {
+): Promise<ExtensionTelemetryListener> {
   if (telemetryListener !== undefined) {
     throw new Error("Telemetry is already initialized");
   }
-  telemetryListener = new TelemetryListener(
+  telemetryListener = new ExtensionTelemetryListener(
     extension.id,
     extension.packageJSON.version,
     key,

extensions/ql-vscode/src/common/vscode/vscode-app.ts

@@ -1,13 +1,17 @@
 import * as vscode from "vscode";
-import { VSCodeCredentials } from "../../authentication";
-import { Disposable } from "../../pure/disposable-object";
-import { App, AppMode } from "../app";
+import { VSCodeCredentials } from "./authentication";
+import { Disposable } from "../disposable-object";
+import { App, AppMode, EnvironmentContext } from "../app";
 import { AppEventEmitter } from "../events";
-import { extLogger, Logger, queryServerLogger } from "../logging";
+import { NotificationLogger } from "../logging";
+import { extLogger, queryServerLogger } from "../logging/vscode";
 import { Memento } from "../memento";
 import { VSCodeAppEventEmitter } from "./events";
 import { AppCommandManager, QueryServerCommandManager } from "../commands";
 import { createVSCodeCommandManager } from "./commands";
+import { AppEnvironmentContext } from "./environment-context";
+import { AppTelemetry } from "../telemetry";
+import { telemetryListener } from "./telemetry";
 
 export class ExtensionApp implements App {
   public readonly credentials: VSCodeCredentials;
@@ -54,11 +58,19 @@ export class ExtensionApp implements App {
     }
   }
 
-  public get logger(): Logger {
+  public get logger(): NotificationLogger {
     return extLogger;
   }
 
+  public get telemetry(): AppTelemetry | undefined {
+    return telemetryListener;
+  }
+
   public createEventEmitter<T>(): AppEventEmitter<T> {
     return new VSCodeAppEventEmitter<T>();
   }
+
+  public get environment(): EnvironmentContext {
+    return new AppEnvironmentContext();
+  }
 }

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -0,0 +1,101 @@
+import { ExtensionContext, Uri, Webview } from "vscode";
+import { randomBytes } from "crypto";
+import { EOL } from "os";
+
+export type WebviewView =
+  | "results"
+  | "compare"
+  | "variant-analysis"
+  | "data-flow-paths"
+  | "data-extensions-editor";
+
+export interface WebviewMessage {
+  t: string;
+}
+
+/**
+ * Returns HTML to populate the given webview.
+ * Uses a content security policy that only loads the given script.
+ */
+export function getHtmlForWebview(
+  ctx: ExtensionContext,
+  webview: Webview,
+  view: WebviewView,
+  {
+    allowInlineStyles,
+    allowWasmEval,
+  }: {
+    allowInlineStyles?: boolean;
+    allowWasmEval?: boolean;
+  } = {
+    allowInlineStyles: false,
+    allowWasmEval: false,
+  },
+): string {
+  const scriptUriOnDisk = Uri.file(ctx.asAbsolutePath("out/webview.js"));
+
+  const stylesheetUrisOnDisk = [
+    Uri.file(ctx.asAbsolutePath("out/webview.css")),
+  ];
+
+  // Convert the on-disk URIs into webview URIs.
+  const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
+  const stylesheetWebviewUris = stylesheetUrisOnDisk.map(
+    (stylesheetUriOnDisk) => webview.asWebviewUri(stylesheetUriOnDisk),
+  );
+
+  // Use a nonce in the content security policy to uniquely identify the above resources.
+  const nonce = getNonce();
+
+  const stylesheetsHtmlLines = allowInlineStyles
+    ? stylesheetWebviewUris.map((uri) => createStylesLinkWithoutNonce(uri))
+    : stylesheetWebviewUris.map((uri) => createStylesLinkWithNonce(nonce, uri));
+
+  const styleSrc = allowInlineStyles
+    ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
+    : `'nonce-${nonce}'`;
+
+  const fontSrc = webview.cspSource;
+
+  /*
+   * Content security policy:
+   * default-src: allow nothing by default.
+   * script-src:
+   *   - allow the given script, using the nonce.
+   *   - 'wasm-unsafe-eval: allow loading WebAssembly modules if necessary.
+   * style-src: allow only the given stylesheet, using the nonce.
+   * connect-src: only allow fetch calls to webview resource URIs
+   * (this is used to load BQRS result files).
+   */
+  return `
+<html>
+  <head>
+    <meta http-equiv="Content-Security-Policy"
+          content="default-src 'none'; script-src 'nonce-${nonce}'${
+            allowWasmEval ? " 'wasm-unsafe-eval'" : ""
+          }; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${
+            webview.cspSource
+          };">
+        ${stylesheetsHtmlLines.join(`    ${EOL}`)}
+  </head>
+  <body>
+    <div id=root data-view="${view}">
+    </div>
+      <script nonce="${nonce}" src="${scriptWebviewUri}">
+    </script>
+  </body>
+</html>`;
+}
+
+/** Gets a nonce string created with 128 bits of entropy. */
+function getNonce(): string {
+  return randomBytes(16).toString("base64");
+}
+
+function createStylesLinkWithNonce(nonce: string, uri: Uri): string {
+  return `<link nonce="${nonce}" rel="stylesheet" href="${uri}">`;
+}
+
+function createStylesLinkWithoutNonce(uri: Uri): string {
+  return `<link rel="stylesheet" href="${uri}">`;
+}

extensions/ql-vscode/src/common/vscode/workspace-folders.ts

@@ -0,0 +1,64 @@
+import { dirname, join } from "path";
+import { workspace, WorkspaceFolder } from "vscode";
+
+/** Returns true if the specified workspace folder is on the file system. */
+export function isWorkspaceFolderOnDisk(
+  workspaceFolder: WorkspaceFolder,
+): boolean {
+  return workspaceFolder.uri.scheme === "file";
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFoldersObjects() {
+  const workspaceFolders = workspace.workspaceFolders ?? [];
+  return workspaceFolders.filter(isWorkspaceFolderOnDisk);
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFolders() {
+  return getOnDiskWorkspaceFoldersObjects().map((folder) => folder.uri.fsPath);
+}
+
+/** Check if folder is already present in workspace */
+export function isFolderAlreadyInWorkspace(folderName: string) {
+  const workspaceFolders = workspace.workspaceFolders || [];
+
+  return !!workspaceFolders.find(
+    (workspaceFolder) => workspaceFolder.name === folderName,
+  );
+}
+
+/**
+ * Returns the path of the first folder in the workspace.
+ * This is used to decide where to create skeleton QL packs.
+ *
+ * If the first folder is a QL pack, then the parent folder is returned.
+ * This is because the vscode-codeql-starter repo contains a ql pack in
+ * the first folder.
+ *
+ * This is a temporary workaround until we can retire the
+ * vscode-codeql-starter repo.
+ */
+export function getFirstWorkspaceFolder() {
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+
+  if (!workspaceFolders || workspaceFolders.length === 0) {
+    throw new Error("No workspace folders found");
+  }
+
+  const firstFolderFsPath = workspaceFolders[0];
+
+  // For the vscode-codeql-starter repo, the first folder will be a ql pack
+  // so we need to get the parent folder
+  if (
+    firstFolderFsPath.includes(
+      join("vscode-codeql-starter", "codeql-custom-queries"),
+    )
+  ) {
+    // return the parent folder
+    return dirname(firstFolderFsPath);
+  } else {
+    // if the first folder is not a ql pack, then we are in a normal workspace
+    return firstFolderFsPath;
+  }
+}

extensions/ql-vscode/src/common/word.ts

@@ -7,8 +7,9 @@ export function pluralize(
   numItems: number | undefined,
   singular: string,
   plural: string,
+  numberFormatter: (value: number) => string = (value) => value.toString(),
 ): string {
   return numItems !== undefined
-    ? `${numItems} ${numItems === 1 ? singular : plural}`
+    ? `${numberFormatter(numItems)} ${numItems === 1 ? singular : plural}`
     : "";
 }

extensions/ql-vscode/src/common/zlib.ts

@@ -0,0 +1,12 @@
+import { promisify } from "util";
+import { gzip, gunzip } from "zlib";
+
+/**
+ * Promisified version of zlib.gzip
+ */
+export const gzipEncode = promisify(gzip);
+
+/**
+ * Promisified version of zlib.gunzip
+ */
+export const gzipDecode = promisify(gunzip);

extensions/ql-vscode/src/compare/compare-view.ts

@@ -4,24 +4,27 @@ import {
   FromCompareViewMessage,
   ToCompareViewMessage,
   QueryCompareResult,
-} from "../pure/interface-types";
-import { Logger } from "../common";
-import { CodeQLCliServer } from "../cli";
-import { DatabaseManager } from "../local-databases";
-import { jumpToLocation } from "../interface-utils";
+} from "../common/interface-types";
+import { Logger, showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { DatabaseManager } from "../databases/local-databases";
+import { jumpToLocation } from "../databases/local-databases/locations";
 import {
   transformBqrsResultSet,
   RawResultSet,
   BQRSInfo,
-} from "../pure/bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import resultsDiff from "./resultsDiff";
 import { CompletedLocalQueryInfo } from "../query-results";
-import { assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { assertNever, getErrorMessage } from "../common/helpers-pure";
 import { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
-import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
-import { telemetryListener } from "../telemetry";
-import { redactableError } from "../pure/errors";
-import { showAndLogExceptionWithTelemetry } from "../helpers";
+import {
+  AbstractWebview,
+  WebviewPanelConfig,
+} from "../common/vscode/abstract-webview";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
@@ -127,7 +130,12 @@ export class CompareView extends AbstractWebview<
         break;
 
       case "viewSourceFile":
-        await jumpToLocation(msg, this.databaseManager, this.logger);
+        await jumpToLocation(
+          msg.databaseUri,
+          msg.loc,
+          this.databaseManager,
+          this.logger,
+        );
         break;
 
       case "openQuery":
@@ -143,6 +151,8 @@ export class CompareView extends AbstractWebview<
 
       case "unhandledError":
         void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
           redactableError(
             msg.error,
           )`Unhandled error in result comparison view: ${msg.error.message}`,
@@ -172,21 +182,40 @@ export class CompareView extends AbstractWebview<
     const commonResultSetNames = fromSchemaNames.filter((name) =>
       toSchemaNames.includes(name),
     );
+
+    // Fall back on the default result set names if there are no common ones.
+    const defaultFromResultSetName = fromSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+    const defaultToResultSetName = toSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+
+    if (
+      commonResultSetNames.length === 0 &&
+      !(defaultFromResultSetName || defaultToResultSetName)
+    ) {
+      throw new Error(
+        "No common result sets found between the two queries. Please check that the queries are compatible.",
+      );
+    }
+
     const currentResultSetName =
       selectedResultSetName || commonResultSetNames[0];
     const fromResultSet = await this.getResultSet(
       fromSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultFromResultSetName!,
       from.completedQuery.query.resultsPaths.resultsPath,
     );
     const toResultSet = await this.getResultSet(
       toSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultToResultSetName!,
       to.completedQuery.query.resultsPaths.resultsPath,
     );
     return [
       commonResultSetNames,
-      currentResultSetName,
+      currentResultSetName ||
+        `${defaultFromResultSetName} <-> ${defaultToResultSetName}`,
       fromResultSet,
       toResultSet,
     ];

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -1,5 +1,5 @@
-import { RawResultSet } from "../pure/bqrs-cli-types";
-import { QueryCompareResult } from "../pure/interface-types";
+import { RawResultSet } from "../common/bqrs-cli-types";
+import { QueryCompareResult } from "../common/interface-types";
 
 /**
  * Compare the rows of two queries. Use deep equality to determine if

extensions/ql-vscode/src/config.ts

@@ -1,14 +1,20 @@
-import { DisposableObject } from "./pure/disposable-object";
+import { DisposableObject } from "./common/disposable-object";
 import {
   workspace,
   Event,
   EventEmitter,
   ConfigurationChangeEvent,
   ConfigurationTarget,
+  ConfigurationScope,
 } from "vscode";
-import { DistributionManager } from "./distribution";
-import { extLogger } from "./common";
-import { ONE_DAY_IN_MS } from "./pure/time";
+import { DistributionManager } from "./codeql-cli/distribution";
+import { extLogger } from "./common/logging/vscode";
+import { ONE_DAY_IN_MS } from "./common/time";
+import {
+  FilterKey,
+  SortKey,
+  defaultFilterSortState,
+} from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -39,12 +45,12 @@ export class Setting {
     }
   }
 
-  getValue<T>(): T {
+  getValue<T>(scope?: ConfigurationScope | null): T {
     if (this.parent === undefined) {
       throw new Error("Cannot get the value of a root setting.");
     }
     return workspace
-      .getConfiguration(this.parent.qualifiedName)
+      .getConfiguration(this.parent.qualifiedName, scope)
       .get<T>(this.name)!;
   }
 
@@ -58,17 +64,21 @@ export class Setting {
   }
 }
 
-export interface InspectionResult<T> {
-  globalValue?: T;
-  workspaceValue?: T;
-  workspaceFolderValue?: T;
-}
+const VSCODE_DEBUG_SETTING = new Setting("debug", undefined);
+export const VSCODE_SAVE_BEFORE_START_SETTING = new Setting(
+  "saveBeforeStart",
+  VSCODE_DEBUG_SETTING,
+);
 
 const ROOT_SETTING = new Setting("codeQL");
 
 // Global configuration
 const TELEMETRY_SETTING = new Setting("telemetry", ROOT_SETTING);
 const AST_VIEWER_SETTING = new Setting("astViewer", ROOT_SETTING);
+const CONTEXTUAL_QUERIES_SETTINGS = new Setting(
+  "contextualQueries",
+  ROOT_SETTING,
+);
 const GLOBAL_TELEMETRY_SETTING = new Setting("telemetry");
 const LOG_INSIGHTS_SETTING = new Setting("logInsights", ROOT_SETTING);
 
@@ -151,10 +161,6 @@ export const NUMBER_OF_TEST_THREADS_SETTING = new Setting(
   RUNNING_TESTS_SETTING,
 );
 export const MAX_QUERIES = new Setting("maxQueries", RUNNING_QUERIES_SETTING);
-export const AUTOSAVE_SETTING = new Setting(
-  "autoSave",
-  RUNNING_QUERIES_SETTING,
-);
 export const PAGE_SIZE = new Setting("pageSize", RESULTS_DISPLAY_SETTING);
 const CUSTOM_LOG_DIRECTORY_SETTING = new Setting(
   "customLogDirectory",
@@ -479,13 +485,21 @@ export function joinOrderWarningThreshold(): number {
 }
 
 /**
- * Avoids caching in the AST viewer if the user is also a canary user.
+ * Hidden setting: Avoids caching in the AST viewer if the user is also a canary user.
  */
 export const NO_CACHE_AST_VIEWER = new Setting(
   "disableCache",
   AST_VIEWER_SETTING,
 );
 
+/**
+ * Hidden setting: Avoids caching in jump to def and find refs contextual queries if the user is also a canary user.
+ */
+export const NO_CACHE_CONTEXTUAL_QUERIES = new Setting(
+  "disableCache",
+  CONTEXTUAL_QUERIES_SETTINGS,
+);
+
 // Settings for variant analysis
 const VARIANT_ANALYSIS_SETTING = new Setting("variantAnalysis", ROOT_SETTING);
 
@@ -529,6 +543,34 @@ export class VariantAnalysisConfigListener
   }
 }
 
+const VARIANT_ANALYSIS_FILTER_RESULTS = new Setting(
+  "defaultResultsFilter",
+  VARIANT_ANALYSIS_SETTING,
+);
+
+export function getVariantAnalysisDefaultResultsFilter(): FilterKey {
+  const value = VARIANT_ANALYSIS_FILTER_RESULTS.getValue<string>();
+  if (Object.values(FilterKey).includes(value as FilterKey)) {
+    return value as FilterKey;
+  } else {
+    return defaultFilterSortState.filterKey;
+  }
+}
+
+const VARIANT_ANALYSIS_SORT_RESULTS = new Setting(
+  "defaultResultsSort",
+  VARIANT_ANALYSIS_SETTING,
+);
+
+export function getVariantAnalysisDefaultResultsSort(): SortKey {
+  const value = VARIANT_ANALYSIS_SORT_RESULTS.getValue<string>();
+  if (Object.values(SortKey).includes(value as SortKey)) {
+    return value as SortKey;
+  } else {
+    return defaultFilterSortState.sortKey;
+  }
+}
+
 /**
  * The branch of "github/codeql-variant-analysis-action" to use with the "Run Variant Analysis" command.
  * Default value is "main".
@@ -544,10 +586,6 @@ export function isIntegrationTestMode() {
   return process.env.INTEGRATION_TEST_MODE === "true";
 }
 
-export function isVariantAnalysisLiveResultsEnabled(): boolean {
-  return true;
-}
-
 // Settings for mocking the GitHub API.
 const MOCK_GH_API_SERVER = new Setting("mockGitHubApiServer", ROOT_SETTING);
 
@@ -611,27 +649,90 @@ export function isCodespacesTemplate() {
 
 const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING);
 
-export const ALLOW_HTTP_SETTING = new Setting(
-  "allowHttp",
-  DATABASE_DOWNLOAD_SETTING,
-);
+const ALLOW_HTTP_SETTING = new Setting("allowHttp", DATABASE_DOWNLOAD_SETTING);
 
 export function allowHttp(): boolean {
   return ALLOW_HTTP_SETTING.getValue<boolean>() || false;
 }
 
 /**
- * The name of the folder where we want to create skeleton wizard QL packs.
+ * Parent setting for all settings related to the "Create Query" command.
+ */
+const CREATE_QUERY_COMMAND = new Setting("createQuery", ROOT_SETTING);
+
+/**
+ * The name of the folder where we want to create QL packs.
  **/
-const SKELETON_WIZARD_FOLDER = new Setting(
-  "folder",
-  new Setting("createQuery", ROOT_SETTING),
+const QL_PACK_LOCATION = new Setting("qlPackLocation", CREATE_QUERY_COMMAND);
+
+export function getQlPackLocation(): string | undefined {
+  return QL_PACK_LOCATION.getValue<string>() || undefined;
+}
+
+export async function setQlPackLocation(folder: string | undefined) {
+  await QL_PACK_LOCATION.updateValue(folder, ConfigurationTarget.Global);
+}
+
+/**
+ * Whether to ask the user to autogenerate a QL pack. The options are "ask" and "never".
+ **/
+const AUTOGENERATE_QL_PACKS = new Setting(
+  "autogenerateQlPacks",
+  CREATE_QUERY_COMMAND,
 );
 
-export function getSkeletonWizardFolder(): string | undefined {
-  return SKELETON_WIZARD_FOLDER.getValue<string>() || undefined;
+const AutogenerateQLPacksValues = ["ask", "never"] as const;
+type AutogenerateQLPacks = (typeof AutogenerateQLPacksValues)[number];
+
+export function getAutogenerateQlPacks(): AutogenerateQLPacks {
+  const value = AUTOGENERATE_QL_PACKS.getValue<AutogenerateQLPacks>();
+  return AutogenerateQLPacksValues.includes(value) ? value : "ask";
 }
 
-export async function setSkeletonWizardFolder(folder: string | undefined) {
-  await SKELETON_WIZARD_FOLDER.updateValue(folder, ConfigurationTarget.Global);
+export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
+  await AUTOGENERATE_QL_PACKS.updateValue(choice, ConfigurationTarget.Global);
+}
+
+/**
+ * A flag indicating whether to show the queries panel in the QL view container.
+ */
+const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
+
+export function showQueriesPanel(): boolean {
+  return !!QUERIES_PANEL.getValue<boolean>();
+}
+
+const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
+const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
+const LLM_GENERATION_V2 = new Setting("llmGenerationV2", DATA_EXTENSIONS);
+const FRAMEWORK_MODE = new Setting("frameworkMode", DATA_EXTENSIONS);
+const DISABLE_AUTO_NAME_EXTENSION_PACK = new Setting(
+  "disableAutoNameExtensionPack",
+  DATA_EXTENSIONS,
+);
+const EXTENSIONS_DIRECTORY = new Setting(
+  "extensionsDirectory",
+  DATA_EXTENSIONS,
+);
+
+export function showLlmGeneration(): boolean {
+  return !!LLM_GENERATION.getValue<boolean>();
+}
+
+export function useLlmGenerationV2(): boolean {
+  return !!LLM_GENERATION_V2.getValue<boolean>();
+}
+
+export function enableFrameworkMode(): boolean {
+  return !!FRAMEWORK_MODE.getValue<boolean>();
+}
+
+export function disableAutoNameExtensionPack(): boolean {
+  return !!DISABLE_AUTO_NAME_EXTENSION_PACK.getValue<boolean>();
+}
+
+export function getExtensionsDirectory(languageId: string): string | undefined {
+  return EXTENSIONS_DIRECTORY.getValue<string>({
+    languageId,
+  });
 }

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -1,198 +0,0 @@
-import { writeFile, promises } from "fs-extra";
-import { dump } from "js-yaml";
-import { file } from "tmp-promise";
-import { basename, dirname, resolve } from "path";
-
-import {
-  getPrimaryDbscheme,
-  getQlPackForDbscheme,
-  getOnDiskWorkspaceFolders,
-  QlPacksForLanguage,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { KeyType, kindOfKeyType, nameOfKeyType, tagOfKeyType } from "./keyType";
-import { CodeQLCliServer } from "../cli";
-import { DatabaseItem } from "../local-databases";
-import { extLogger, TeeLogger } from "../common";
-import { CancellationToken } from "vscode";
-import { ProgressCallback } from "../progress";
-import { CoreCompletedQuery, QueryRunner } from "../query-server";
-import { redactableError } from "../pure/errors";
-import { QLPACK_FILENAMES } from "../pure/ql";
-
-export async function qlpackOfDatabase(
-  cli: Pick<CodeQLCliServer, "resolveQlpacks">,
-  db: Pick<DatabaseItem, "contents">,
-): Promise<QlPacksForLanguage> {
-  if (db.contents === undefined) {
-    throw new Error("Database is invalid and cannot infer QLPack.");
-  }
-  const datasetPath = db.contents.datasetUri.fsPath;
-  const dbscheme = await getPrimaryDbscheme(datasetPath);
-  return await getQlPackForDbscheme(cli, dbscheme);
-}
-
-/**
- * Finds the contextual queries with the specified key in a list of CodeQL packs.
- *
- * @param cli The CLI instance to use.
- * @param qlpacks The list of packs to search.
- * @param keyType The contextual query key of the query to search for.
- * @returns The found queries from the first pack in which any matching queries were found.
- */
-async function resolveQueriesFromPacks(
-  cli: CodeQLCliServer,
-  qlpacks: string[],
-  keyType: KeyType,
-): Promise<string[]> {
-  const suiteFile = (
-    await file({
-      postfix: ".qls",
-    })
-  ).path;
-  const suiteYaml = [];
-  for (const qlpack of qlpacks) {
-    suiteYaml.push({
-      from: qlpack,
-      queries: ".",
-      include: {
-        kind: kindOfKeyType(keyType),
-        "tags contain": tagOfKeyType(keyType),
-      },
-    });
-  }
-  await writeFile(suiteFile, dump(suiteYaml), "utf8");
-
-  const queries = await cli.resolveQueriesInSuite(
-    suiteFile,
-    getOnDiskWorkspaceFolders(),
-  );
-  return queries;
-}
-
-export async function resolveQueries(
-  cli: CodeQLCliServer,
-  qlpacks: QlPacksForLanguage,
-  keyType: KeyType,
-): Promise<string[]> {
-  const packsToSearch: string[] = [];
-
-  // The CLI can handle both library packs and query packs, so search both packs in order.
-  packsToSearch.push(qlpacks.dbschemePack);
-  if (qlpacks.queryPack !== undefined) {
-    packsToSearch.push(qlpacks.queryPack);
-  }
-
-  const queries = await resolveQueriesFromPacks(cli, packsToSearch, keyType);
-  if (queries.length > 0) {
-    return queries;
-  }
-
-  // No queries found. Determine the correct error message for the various scenarios.
-  const keyTypeName = nameOfKeyType(keyType);
-  const keyTypeTag = tagOfKeyType(keyType);
-  const joinedPacksToSearch = packsToSearch.join(", ");
-  const error = redactableError`No ${keyTypeName} queries (tagged "${keyTypeTag}") could be found in the \
-current library path (tried searching the following packs: ${joinedPacksToSearch}). \
-Try upgrading the CodeQL libraries. If that doesn't work, then ${keyTypeName} queries are not yet available \
-for this language.`;
-
-  void showAndLogExceptionWithTelemetry(error);
-  throw error;
-}
-
-async function resolveContextualQuery(
-  cli: CodeQLCliServer,
-  query: string,
-): Promise<{ packPath: string; createdTempLockFile: boolean }> {
-  // Contextual queries now live within the standard library packs.
-  // This simplifies distribution (you don't need the standard query pack to use the AST viewer),
-  // but if the library pack doesn't have a lockfile, we won't be able to find
-  // other pack dependencies of the library pack.
-
-  // Work out the enclosing pack.
-  const packContents = await cli.packPacklist(query, false);
-  const packFilePath = packContents.find((p) =>
-    QLPACK_FILENAMES.includes(basename(p)),
-  );
-  if (packFilePath === undefined) {
-    // Should not happen; we already resolved this query.
-    throw new Error(
-      `Could not find a CodeQL pack file for the pack enclosing the contextual query ${query}`,
-    );
-  }
-  const packPath = dirname(packFilePath);
-  const lockFilePath = packContents.find((p) =>
-    ["codeql-pack.lock.yml", "qlpack.lock.yml"].includes(basename(p)),
-  );
-  let createdTempLockFile = false;
-  if (!lockFilePath) {
-    // No lock file, likely because this library pack is in the package cache.
-    // Create a lock file so that we can resolve dependencies and library path
-    // for the contextual query.
-    void extLogger.log(
-      `Library pack ${packPath} is missing a lock file; creating a temporary lock file`,
-    );
-    await cli.packResolveDependencies(packPath);
-    createdTempLockFile = true;
-    // Clear CLI server pack cache before installing dependencies,
-    // so that it picks up the new lock file, not the previously cached pack.
-    void extLogger.log("Clearing the CodeQL CLI server's pack cache");
-    await cli.clearCache();
-    // Install dependencies.
-    void extLogger.log(
-      `Installing package dependencies for library pack ${packPath}`,
-    );
-    await cli.packInstall(packPath);
-  }
-  return { packPath, createdTempLockFile };
-}
-
-async function removeTemporaryLockFile(packPath: string) {
-  const tempLockFilePath = resolve(packPath, "codeql-pack.lock.yml");
-  void extLogger.log(
-    `Deleting temporary package lock file at ${tempLockFilePath}`,
-  );
-  // It's fine if the file doesn't exist.
-  await promises.rm(resolve(packPath, "codeql-pack.lock.yml"), {
-    force: true,
-  });
-}
-
-export async function runContextualQuery(
-  query: string,
-  db: DatabaseItem,
-  queryStorageDir: string,
-  qs: QueryRunner,
-  cli: CodeQLCliServer,
-  progress: ProgressCallback,
-  token: CancellationToken,
-  templates: Record<string, string>,
-): Promise<CoreCompletedQuery> {
-  const { packPath, createdTempLockFile } = await resolveContextualQuery(
-    cli,
-    query,
-  );
-  const queryRun = qs.createQueryRun(
-    db.databaseUri.fsPath,
-    { queryPath: query, quickEvalPosition: undefined },
-    false,
-    getOnDiskWorkspaceFolders(),
-    undefined,
-    queryStorageDir,
-    undefined,
-    templates,
-  );
-  void extLogger.log(
-    `Running contextual query ${query}; results will be stored in ${queryRun.outputDir.querySaveDir}`,
-  );
-  const results = await queryRun.evaluate(
-    progress,
-    token,
-    new TeeLogger(qs.logger, queryRun.outputDir.logPath),
-  );
-  if (createdTempLockFile) {
-    await removeTemporaryLockFile(packPath);
-  }
-  return results;
-}

extensions/ql-vscode/src/data-extensions-editor/auto-model-api-v2.ts

@@ -0,0 +1,34 @@
+import { Credentials } from "../common/authentication";
+import { OctokitResponse } from "@octokit/types";
+
+export enum AutomodelMode {
+  Unspecified = "AUTOMODEL_MODE_UNSPECIFIED",
+  Framework = "AUTOMODEL_MODE_FRAMEWORK",
+  Application = "AUTOMODEL_MODE_APPLICATION",
+}
+
+export interface ModelRequest {
+  mode: AutomodelMode;
+  // Base64-encoded GZIP-compressed SARIF log
+  candidates: string;
+}
+
+export interface ModelResponse {
+  models: string;
+}
+
+export async function autoModelV2(
+  credentials: Credentials,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<ModelResponse> = await octokit.request(
+    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+    {
+      data: request,
+    },
+  );
+
+  return response.data;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts

@@ -0,0 +1,54 @@
+import { Credentials } from "../common/authentication";
+import { OctokitResponse } from "@octokit/types";
+
+export enum ClassificationType {
+  Unknown = "CLASSIFICATION_TYPE_UNKNOWN",
+  Neutral = "CLASSIFICATION_TYPE_NEUTRAL",
+  Source = "CLASSIFICATION_TYPE_SOURCE",
+  Sink = "CLASSIFICATION_TYPE_SINK",
+  Summary = "CLASSIFICATION_TYPE_SUMMARY",
+}
+
+export interface Classification {
+  type: ClassificationType;
+  kind: string;
+  explanation: string;
+}
+
+export interface Method {
+  package: string;
+  type: string;
+  name: string;
+  signature: string;
+  usages: string[];
+  classification?: Classification;
+  input?: string;
+  output?: string;
+}
+
+export interface ModelRequest {
+  language: string;
+  candidates: Method[];
+  samples: Method[];
+}
+
+export interface ModelResponse {
+  language: string;
+  predicted?: Method[];
+}
+
+export async function autoModel(
+  credentials: Credentials,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<ModelResponse> = await octokit.request(
+    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+    {
+      data: request,
+    },
+  );
+
+  return response.data;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts

@@ -0,0 +1,230 @@
+import { CodeQLCliServer, SourceInfo } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { ProgressCallback } from "../common/vscode/progress";
+import * as Sarif from "sarif";
+import { qlpackOfDatabase, resolveQueries } from "../local-queries";
+import { extLogger } from "../common/logging/vscode";
+import { Mode } from "./shared/mode";
+import { QlPacksForLanguage } from "../databases/qlpack";
+import { createLockFileForStandardQuery } from "../local-queries/standard-queries";
+import { CancellationToken, CancellationTokenSource } from "vscode";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { QueryResultType } from "../query-server/new-messages";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
+import { interpretResultsSarif } from "../query-results";
+import { join } from "path";
+import { assertNever } from "../common/helpers-pure";
+
+type AutoModelQueryOptions = {
+  queryTag: string;
+  mode: Mode;
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  qlpack: QlPacksForLanguage;
+  sourceInfo: SourceInfo | undefined;
+  extensionPacks: string[];
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+  token: CancellationToken;
+};
+
+function modeTag(mode: Mode): string {
+  switch (mode) {
+    case Mode.Application:
+      return "application-mode";
+    case Mode.Framework:
+      return "framework-mode";
+    default:
+      assertNever(mode);
+  }
+}
+
+async function runAutoModelQuery({
+  queryTag,
+  mode,
+  cliServer,
+  queryRunner,
+  databaseItem,
+  qlpack,
+  sourceInfo,
+  extensionPacks,
+  queryStorageDir,
+  progress,
+  token,
+}: AutoModelQueryOptions): Promise<Sarif.Log | undefined> {
+  // First, resolve the query that we want to run.
+  // All queries are tagged like this:
+  // internal extract automodel <mode> <queryTag>
+  // Example: internal extract automodel framework-mode candidates
+  const queries = await resolveQueries(
+    cliServer,
+    qlpack,
+    `Extract automodel ${queryTag}`,
+    {
+      kind: "problem",
+      "tags contain all": ["automodel", modeTag(mode), ...queryTag.split(" ")],
+    },
+  );
+  if (queries.length > 1) {
+    throw new Error(
+      `Found multiple auto model queries for ${mode} ${queryTag}. Can't continue`,
+    );
+  }
+  if (queries.length === 0) {
+    throw new Error(
+      `Did not found any auto model queries for ${mode} ${queryTag}. Can't continue`,
+    );
+  }
+
+  const queryPath = queries[0];
+  const { cleanup: cleanupLockFile } = await createLockFileForStandardQuery(
+    cliServer,
+    queryPath,
+  );
+
+  // Get metadata for the query. This is required to interpret the results. We already know the kind is problem
+  // (because of the constraint in resolveQueries), so we don't need any more checks on the metadata.
+  const metadata = await cliServer.resolveMetadata(queryPath);
+
+  const queryRun = queryRunner.createQueryRun(
+    databaseItem.databaseUri.fsPath,
+    {
+      queryPath,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
+    false,
+    getOnDiskWorkspaceFolders(),
+    extensionPacks,
+    queryStorageDir,
+    undefined,
+    undefined,
+  );
+
+  const completedQuery = await queryRun.evaluate(
+    progress,
+    token,
+    new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
+  );
+
+  await cleanupLockFile?.();
+
+  if (completedQuery.resultType !== QueryResultType.SUCCESS) {
+    void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
+      redactableError`Auto-model query ${queryTag} failed: ${
+        completedQuery.message ?? "No message"
+      }`,
+    );
+    return;
+  }
+
+  const interpretedResultsPath = join(
+    queryStorageDir,
+    `interpreted-results-${queryTag.replaceAll(" ", "-")}-${queryRun.id}.sarif`,
+  );
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars -- We only need the actual SARIF data, not the extra fields added by SarifInterpretationData
+  const { t, sortState, ...sarif } = await interpretResultsSarif(
+    cliServer,
+    metadata,
+    {
+      resultsPath: completedQuery.outputDir.bqrsPath,
+      interpretedResultsPath,
+    },
+    sourceInfo,
+    ["--sarif-add-snippets"],
+  );
+
+  return sarif;
+}
+
+type AutoModelQueriesOptions = {
+  mode: Mode;
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+};
+
+export type AutoModelQueriesResult = {
+  candidates: Sarif.Log;
+};
+
+export async function runAutoModelQueries({
+  mode,
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+}: AutoModelQueriesOptions): Promise<AutoModelQueriesResult | undefined> {
+  // maxStep for this part is 1500
+  const maxStep = 1500;
+
+  const cancellationTokenSource = new CancellationTokenSource();
+
+  const qlpack = await qlpackOfDatabase(cliServer, databaseItem);
+
+  // CodeQL needs to have access to the database to be able to retrieve the
+  // snippets from it. The source location prefix is used to determine the
+  // base path of the database.
+  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
+    cliServer,
+  );
+  const sourceArchiveUri = databaseItem.sourceArchive;
+  const sourceInfo =
+    sourceArchiveUri === undefined
+      ? undefined
+      : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix,
+        };
+
+  const additionalPacks = getOnDiskWorkspaceFolders();
+  const extensionPacks = Object.keys(
+    await cliServer.resolveQlpacks(additionalPacks, true),
+  );
+
+  progress({
+    step: 0,
+    maxStep,
+    message: "Finding candidates and examples",
+  });
+
+  const candidates = await runAutoModelQuery({
+    mode,
+    queryTag: "candidates",
+    cliServer,
+    queryRunner,
+    databaseItem,
+    qlpack,
+    sourceInfo,
+    extensionPacks,
+    queryStorageDir,
+    progress: (update) => {
+      progress({
+        step: update.step,
+        maxStep,
+        message: "Finding candidates and examples",
+      });
+    },
+    token: cancellationTokenSource.token,
+  });
+
+  if (!candidates) {
+    return undefined;
+  }
+
+  return {
+    candidates,
+  };
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

@@ -0,0 +1,140 @@
+import { CancellationTokenSource } from "vscode";
+import { join } from "path";
+import { runQuery } from "./external-api-usage-query";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { interpretResultsSarif } from "../query-results";
+import { ProgressCallback } from "../common/vscode/progress";
+import { Mode } from "./shared/mode";
+
+type Options = {
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  queryStorageDir: string;
+  queryDir: string;
+
+  progress: ProgressCallback;
+};
+
+export type UsageSnippetsBySignature = Record<string, string[]>;
+
+export async function getAutoModelUsages({
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  queryDir,
+  progress,
+}: Options): Promise<UsageSnippetsBySignature> {
+  const maxStep = 1500;
+
+  const cancellationTokenSource = new CancellationTokenSource();
+
+  // This will re-run the query that was already run when opening the data extensions editor. This
+  // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
+  // need to interpret the results.
+  const queryResult = await runQuery(Mode.Application, {
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    queryDir,
+    progress: (update) =>
+      progress({
+        maxStep,
+        step: update.step,
+        message: update.message,
+      }),
+    token: cancellationTokenSource.token,
+  });
+  if (!queryResult) {
+    throw new Error("Query failed");
+  }
+
+  progress({
+    maxStep,
+    step: 1100,
+    message: "Retrieving source location prefix",
+  });
+
+  // CodeQL needs to have access to the database to be able to retrieve the
+  // snippets from it. The source location prefix is used to determine the
+  // base path of the database.
+  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
+    cliServer,
+  );
+  const sourceArchiveUri = databaseItem.sourceArchive;
+  const sourceInfo =
+    sourceArchiveUri === undefined
+      ? undefined
+      : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix,
+        };
+
+  progress({
+    maxStep,
+    step: 1200,
+    message: "Interpreting results",
+  });
+
+  // Convert the results to SARIF so that Codeql will retrieve the snippets
+  // from the datababe. This means we don't need to do that in the extension
+  // and everything is handled by the CodeQL CLI.
+  const sarif = await interpretResultsSarif(
+    cliServer,
+    {
+      // To interpret the results we need to provide metadata about the query. We could do this using
+      // `resolveMetadata` but that would be an extra call to the CodeQL CLI server and would require
+      // us to know the path to the query on the filesystem. Since we know what the metadata should
+      // look like and the only metadata that the CodeQL CLI requires is an ID and the kind, we can
+      // simply use constants here.
+      kind: "problem",
+      id: "usage",
+    },
+    {
+      resultsPath: queryResult.outputDir.bqrsPath,
+      interpretedResultsPath: join(
+        queryStorageDir,
+        "interpreted-results.sarif",
+      ),
+    },
+    sourceInfo,
+    ["--sarif-add-snippets"],
+  );
+
+  progress({
+    maxStep,
+    step: 1400,
+    message: "Parsing results",
+  });
+
+  const snippets: UsageSnippetsBySignature = {};
+
+  const results = sarif.runs[0]?.results;
+  if (!results) {
+    throw new Error("No results");
+  }
+
+  // This will group the snippets by the method signature.
+  for (const result of results) {
+    const signature = result.message.text;
+
+    const snippet =
+      result.locations?.[0]?.physicalLocation?.contextRegion?.snippet?.text;
+
+    if (!signature || !snippet) {
+      continue;
+    }
+
+    if (!(signature in snippets)) {
+      snippets[signature] = [];
+    }
+
+    snippets[signature].push(snippet);
+  }
+
+  return snippets;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-v2.ts

@@ -0,0 +1,40 @@
+import { AutomodelMode, ModelRequest } from "./auto-model-api-v2";
+import { Mode } from "./shared/mode";
+import { AutoModelQueriesResult } from "./auto-model-codeml-queries";
+import { assertNever } from "../common/helpers-pure";
+import * as Sarif from "sarif";
+import { gzipEncode } from "../common/zlib";
+
+/**
+ * Encode a SARIF log to the format expected by the server: JSON, GZIP-compressed, base64-encoded
+ * @param log SARIF log to encode
+ * @returns base64-encoded GZIP-compressed SARIF log
+ */
+export async function encodeSarif(log: Sarif.Log): Promise<string> {
+  const json = JSON.stringify(log);
+  const buffer = Buffer.from(json, "utf-8");
+  const compressed = await gzipEncode(buffer);
+  return compressed.toString("base64");
+}
+
+export async function createAutoModelV2Request(
+  mode: Mode,
+  result: AutoModelQueriesResult,
+): Promise<ModelRequest> {
+  let requestMode: AutomodelMode;
+  switch (mode) {
+    case Mode.Application:
+      requestMode = AutomodelMode.Application;
+      break;
+    case Mode.Framework:
+      requestMode = AutomodelMode.Framework;
+      break;
+    default:
+      assertNever(mode);
+  }
+
+  return {
+    mode: requestMode,
+    candidates: await encodeSarif(result.candidates),
+  };
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model.ts

@@ -0,0 +1,262 @@
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+import {
+  Classification,
+  ClassificationType,
+  Method,
+  ModelRequest,
+} from "./auto-model-api";
+import type { UsageSnippetsBySignature } from "./auto-model-usages-query";
+import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
+import { Mode } from "./shared/mode";
+
+// Soft limit on the number of candidates to send to the model.
+// Note that the model may return fewer than this number of candidates.
+const candidateLimit = 20;
+// Soft limit on the number of samples to send to the model.
+const sampleLimit = 100;
+
+export function createAutoModelRequest(
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+  usages: UsageSnippetsBySignature,
+  mode: Mode,
+): ModelRequest {
+  const request: ModelRequest = {
+    language,
+    samples: [],
+    candidates: [],
+  };
+
+  // Sort the same way as the UI so we send the first ones listed in the UI first
+  const grouped = groupMethods(externalApiUsages, mode);
+  const sortedGroupNames = sortGroupNames(grouped);
+  const sortedExternalApiUsages = sortedGroupNames.flatMap((name) =>
+    sortMethods(grouped[name]),
+  );
+
+  for (const externalApiUsage of sortedExternalApiUsages) {
+    const modeledMethod: ModeledMethod = modeledMethods[
+      externalApiUsage.signature
+    ] ?? {
+      type: "none",
+    };
+
+    const usagesForMethod =
+      usages[externalApiUsage.signature] ??
+      externalApiUsage.usages.map((usage) => usage.label);
+
+    const numberOfArguments =
+      externalApiUsage.methodParameters === "()"
+        ? 0
+        : externalApiUsage.methodParameters.split(",").length;
+
+    const candidates: Method[] = [];
+    const samples: Method[] = [];
+    for (
+      let argumentIndex = -1; // Start at -1 which means `this` as in `this.method()`
+      argumentIndex < numberOfArguments;
+      argumentIndex++
+    ) {
+      const argumentInput: string =
+        argumentIndex === -1 ? "Argument[this]" : `Argument[${argumentIndex}]`;
+      const method: Method = {
+        package: externalApiUsage.packageName,
+        type: externalApiUsage.typeName,
+        name: externalApiUsage.methodName,
+        signature: externalApiUsage.methodParameters,
+        classification:
+          modeledMethod.type === "none"
+            ? undefined
+            : toMethodClassification(modeledMethod),
+        usages: usagesForMethod.slice(0, 6), // At most 6 usages per argument
+        input: argumentInput,
+      };
+
+      // A method that is supported is modeled outside of the model file, so it is not a candidate.
+      // We also do not want it as a sample because we do not know the classification.
+      if (modeledMethod.type === "none" && externalApiUsage.supported) {
+        continue;
+      }
+
+      // Candidates are methods that are not currently modeled
+      if (modeledMethod.type === "none") {
+        candidates.push(method);
+      } else {
+        samples.push(method);
+      }
+    }
+    // If there is room for at least one candidate, add all candidates.
+    // This ensures that we send all arguments for a method together.
+    // NOTE: this might go above the candidate limit, but that's okay.
+    if (request.candidates.length < candidateLimit) {
+      request.candidates.push(...candidates);
+    }
+    // Same for samples
+    if (request.samples.length < sampleLimit) {
+      request.samples.push(...samples);
+    }
+  }
+
+  return request;
+}
+
+/**
+ * For now, we have a simplified model that only models methods as sinks. It does not model methods as neutral,
+ * so we aren't actually able to correctly determine that a method is neutral; it could still be a source or summary.
+ * However, to keep this method simple and give output to the user, we will model any method for which none of its
+ * arguments are modeled as sinks as neutral.
+ *
+ * If there are multiple arguments which are modeled as sinks, we will only model the first one.
+ */
+export function parsePredictedClassifications(
+  predicted: Method[],
+): Record<string, ModeledMethod> {
+  const predictedBySignature: Record<string, Method[]> = {};
+  for (const method of predicted) {
+    const signature = toFullMethodSignature(method);
+
+    if (!(signature in predictedBySignature)) {
+      predictedBySignature[signature] = [];
+    }
+
+    predictedBySignature[signature].push(method);
+  }
+
+  const modeledMethods: Record<string, ModeledMethod> = {};
+
+  for (const signature in predictedBySignature) {
+    const predictedMethods = predictedBySignature[signature];
+
+    const sinks = predictedMethods.filter(
+      (method) => method.classification?.type === ClassificationType.Sink,
+    );
+    if (sinks.length === 0) {
+      // For now, model any method for which none of its arguments are modeled as sinks as neutral
+      modeledMethods[signature] = {
+        type: "neutral",
+        kind: "summary",
+        input: "",
+        output: "",
+        provenance: "ai-generated",
+        signature,
+        // predictedBySignature[signature] always has at least element
+        packageName: predictedMethods[0].package,
+        typeName: predictedMethods[0].type,
+        methodName: predictedMethods[0].name,
+        methodParameters: predictedMethods[0].signature,
+      };
+      continue;
+    }
+
+    // Order the sinks by the input alphabetically. This will ensure that the first argument is always
+    // first in the list of sinks, the second argument is always second, etc.
+    // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first
+    sinks.sort((a, b) => compareInputOutput(a.input ?? "", b.input ?? ""));
+
+    const sink = sinks[0];
+
+    modeledMethods[signature] = {
+      type: "sink",
+      kind: sink.classification?.kind ?? "",
+      input: sink.input ?? "",
+      output: sink.output ?? "",
+      provenance: "ai-generated",
+      signature,
+      packageName: sink.package,
+      typeName: sink.type,
+      methodName: sink.name,
+      methodParameters: sink.signature,
+    };
+  }
+
+  return modeledMethods;
+}
+
+function toMethodClassificationType(
+  type: ModeledMethodType,
+): ClassificationType {
+  switch (type) {
+    case "source":
+      return ClassificationType.Source;
+    case "sink":
+      return ClassificationType.Sink;
+    case "summary":
+      return ClassificationType.Summary;
+    case "neutral":
+      return ClassificationType.Neutral;
+    default:
+      return ClassificationType.Unknown;
+  }
+}
+
+function toMethodClassification(modeledMethod: ModeledMethod): Classification {
+  return {
+    type: toMethodClassificationType(modeledMethod.type),
+    kind: modeledMethod.kind,
+    explanation: "",
+  };
+}
+
+function toFullMethodSignature(method: Method): string {
+  return `${method.package}.${method.type}#${method.name}${method.signature}`;
+}
+
+const argumentRegex = /^Argument\[(\d+)]$/;
+
+// Argument[this] is before ReturnValue
+const nonNumericArgumentOrder = ["Argument[this]", "ReturnValue"];
+
+/**
+ * Compare two inputs or outputs matching `Argument[<number>]`, `Argument[this]`, or `ReturnValue`.
+ * If they are the same, return 0. If a is less than b, returns a negative number.
+ * If a is greater than b, returns a positive number.
+ */
+export function compareInputOutput(a: string, b: string): number {
+  if (a === b) {
+    return 0;
+  }
+
+  const aMatch = a.match(argumentRegex);
+  const bMatch = b.match(argumentRegex);
+
+  // Numeric arguments are always first
+  if (aMatch && !bMatch) {
+    return -1;
+  }
+  if (!aMatch && bMatch) {
+    return 1;
+  }
+
+  // Neither is an argument
+  if (!aMatch && !bMatch) {
+    const aIndex = nonNumericArgumentOrder.indexOf(a);
+    const bIndex = nonNumericArgumentOrder.indexOf(b);
+
+    // If either one is unknown, it is sorted last
+    if (aIndex === -1 && bIndex === -1) {
+      // Use en-US because these are well-known strings that are not localized
+      return a.localeCompare(b, "en-US");
+    }
+    if (aIndex === -1) {
+      return 1;
+    }
+    if (bIndex === -1) {
+      return -1;
+    }
+
+    return aIndex - bIndex;
+  }
+
+  // This case shouldn't happen, but makes TypeScript happy
+  if (!aMatch || !bMatch) {
+    return 0;
+  }
+
+  // Both are arguments
+  const aIndex = parseInt(aMatch[1]);
+  const bIndex = parseInt(bMatch[1]);
+
+  return aIndex - bIndex;
+}

extensions/ql-vscode/src/data-extensions-editor/bqrs.ts

@@ -1,5 +1,11 @@
-import { DecodedBqrsChunk } from "../pure/bqrs-cli-types";
-import { Call, ExternalApiUsage } from "./external-api-usage";
+import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import {
+  Call,
+  CallClassification,
+  ExternalApiUsage,
+} from "./external-api-usage";
+import { ModeledMethodType } from "./modeled-method";
+import { parseLibraryFilename } from "./library";
 
 export function decodeBqrsToExternalApiUsages(
   chunk: DecodedBqrsChunk,
@@ -7,9 +13,13 @@ export function decodeBqrsToExternalApiUsages(
   const methodsByApiName = new Map<string, ExternalApiUsage>();
 
   chunk?.tuples.forEach((tuple) => {
-    const signature = tuple[0] as string;
-    const supported = tuple[1] as boolean;
-    const usage = tuple[2] as Call;
+    const usage = tuple[0] as Call;
+    const signature = tuple[1] as string;
+    const supported = (tuple[2] as string) === "true";
+    let library = tuple[4] as string;
+    let libraryVersion: string | undefined = tuple[5] as string;
+    const type = tuple[6] as ModeledMethodType;
+    const classification = tuple[8] as CallClassification;
 
     const [packageWithType, methodDeclaration] = signature.split("#");
 
@@ -29,33 +39,42 @@ export function decodeBqrsToExternalApiUsages(
       methodDeclaration.indexOf("("),
     );
 
+    // For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
+    // properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
+    // for Java.
+    if (library.endsWith(".jar") || libraryVersion === "") {
+      const { name, version } = parseLibraryFilename(library);
+      library = name;
+      if (version) {
+        libraryVersion = version;
+      }
+    }
+
+    if (libraryVersion === "") {
+      libraryVersion = undefined;
+    }
+
     if (!methodsByApiName.has(signature)) {
       methodsByApiName.set(signature, {
+        library,
+        libraryVersion,
         signature,
         packageName,
         typeName,
         methodName,
         methodParameters,
         supported,
+        supportedType: type,
         usages: [],
       });
     }
 
     const method = methodsByApiName.get(signature)!;
-    method.usages.push(usage);
+    method.usages.push({
+      ...usage,
+      classification,
+    });
   });
 
-  const externalApiUsages = Array.from(methodsByApiName.values());
-  externalApiUsages.sort((a, b) => {
-    // Sort first by supported, putting unmodeled methods first.
-    if (a.supported && !b.supported) {
-      return 1;
-    }
-    if (!a.supported && b.supported) {
-      return -1;
-    }
-    // Then sort by number of usages descending
-    return b.usages.length - a.usages.length;
-  });
-  return externalApiUsages;
+  return Array.from(methodsByApiName.values());
 }