Merge pull request #2891 from github/koesie10/dispose-webview

Dispose tracked objects when panel is disposed

.github/codeql/queries/unique-command-use.ql

@@ -15,134 +15,145 @@
  * that should be changed to fix the alert.
  */
 
- import javascript
+import javascript
 
- /**
-  * The name of a VS Code command.
-  */
- class CommandName extends string {
-   CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
- 
-   /**
-    * In how many ways is this command used. Will always be at least 1.
-    */
-   int getNumberOfUsages() { result = count(this.getAUse()) }
+/**
+ * The name of a VS Code command.
+ */
+class CommandName extends string {
+  CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
 
-   /**
-    * Get a usage of this command.
-    */
-   CommandUsage getAUse() { result.getCommandName() = this }
- 
-   /**
-    * Get the canonical first usage of this command, to use for the location
-    * of the alert. The implementation of this ordering of usages is arbitrary
-    * and the usage given may not be the one that should be changed when fixing
-    * the alert.
-    */
-   CommandUsage getFirstUsage() {
-     result =
-       max(CommandUsage use |
-         use = this.getAUse()
-       |
-         use
-         order by
-           use.getFile().getRelativePath(), use.getLocation().getStartLine(),
-           use.getLocation().getStartColumn()
-       )
-   }
- }
- 
- /**
-  * Represents a single usage of a command, either from within code or
-  * from the command's definition in package.json
-  */
- abstract class CommandUsage extends Locatable {
-   abstract string getCommandName();
- }
- 
- /**
-  * A usage of a command from the typescript code, by calling `executeCommand`.
-  */
- class CommandUsageCallExpr extends CommandUsage, CallExpr {
-   CommandUsageCallExpr() {
-     this.getCalleeName() = "executeCommand" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
- 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
+  /**
+   * In how many ways is this command used. Will always be at least 1.
+   */
+  int getNumberOfUsages() { result = count(this.getAUse()) }
 
- /**
-  * A usage of a command from the typescript code, by calling `CommandManager.execute`.
-  */
- class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
-   CommandUsageCommandManagerMethodCallExpr() {
-     this.getCalleeName() = "execute" and
-     this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
+  /**
+   * Get a usage of this command.
+   */
+  CommandUsage getAUse() { result.getCommandName() = this }
 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
- 
- /**
-  * A usage of a command from any menu that isn't the command palette.
-  * This means a user could invoke the command by clicking on a button in
-  * something like a menu or a dropdown.
-  */
- class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonMenuItem() {
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson, string menuName |
-       packageJson
-           .getPropValue("contributes")
-           .getPropValue("menus")
-           .getPropValue(menuName)
-           .getElementValue(_) = this and
-       menuName != "commandPalette"
-     )
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- /**
-  * Is the given command disabled for use in the command palette by
-  * a block with a `"when": "false"` field.
-  */
- predicate isDisabledInCommandPalette(string commandName) {
-   exists(PackageJson packageJson, JsonObject commandPaletteObject |
-     packageJson
-         .getPropValue("contributes")
-         .getPropValue("menus")
-         .getPropValue("commandPalette")
-         .getElementValue(_) = commandPaletteObject and
-     commandPaletteObject.getPropValue("command").getStringValue() = commandName and
-     commandPaletteObject.getPropValue("when").getStringValue() = "false"
-   )
- }
- 
- /**
-  * Represents a command being usable from the command palette.
-  * This means that a user could choose to manually invoke the command.
-  */
- class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonCommandPalette() {
-     this.getFile().getBaseName() = "package.json" and
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson |
-       packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
-     ) and
-     not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- from CommandName c
- where c.getNumberOfUsages() > 1
- select c.getFirstUsage(),
-   "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"
- 
+  /**
+   * Get the canonical first usage of this command, to use for the location
+   * of the alert. The implementation of this ordering of usages is arbitrary
+   * and the usage given may not be the one that should be changed when fixing
+   * the alert.
+   */
+  CommandUsage getFirstUsage() {
+    result =
+      max(CommandUsage use |
+        use = this.getAUse()
+      |
+        use
+        order by
+          use.getFile().getRelativePath(), use.getLocation().getStartLine(),
+          use.getLocation().getStartColumn()
+      )
+  }
+}
+
+/**
+ * Matches one of the members of `BuiltInVsCodeCommands` from `extensions/ql-vscode/src/common/commands.ts`.
+ */
+class BuiltInVSCodeCommand extends string {
+  BuiltInVSCodeCommand() {
+    exists(TypeAliasDeclaration tad |
+      tad.getIdentifier().getName() = "BuiltInVsCodeCommands" and
+      tad.getDefinition().(InterfaceTypeExpr).getAMember().getName() = this
+    )
+  }
+}
+
+/**
+ * Represents a single usage of a command, either from within code or
+ * from the command's definition in package.json
+ */
+abstract class CommandUsage extends Locatable {
+  abstract string getCommandName();
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `executeCommand`.
+ */
+class CommandUsageCallExpr extends CommandUsage, CallExpr {
+  CommandUsageCallExpr() {
+    this.getCalleeName() = "executeCommand" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `CommandManager.execute`.
+ */
+class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
+  CommandUsageCommandManagerMethodCallExpr() {
+    this.getCalleeName() = "execute" and
+    this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from any menu that isn't the command palette.
+ * This means a user could invoke the command by clicking on a button in
+ * something like a menu or a dropdown.
+ */
+class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonMenuItem() {
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson, string menuName |
+      packageJson
+          .getPropValue("contributes")
+          .getPropValue("menus")
+          .getPropValue(menuName)
+          .getElementValue(_) = this and
+      menuName != "commandPalette"
+    )
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+/**
+ * Is the given command disabled for use in the command palette by
+ * a block with a `"when": "false"` field.
+ */
+predicate isDisabledInCommandPalette(string commandName) {
+  exists(PackageJson packageJson, JsonObject commandPaletteObject |
+    packageJson
+        .getPropValue("contributes")
+        .getPropValue("menus")
+        .getPropValue("commandPalette")
+        .getElementValue(_) = commandPaletteObject and
+    commandPaletteObject.getPropValue("command").getStringValue() = commandName and
+    commandPaletteObject.getPropValue("when").getStringValue() = "false"
+  )
+}
+
+/**
+ * Represents a command being usable from the command palette.
+ * This means that a user could choose to manually invoke the command.
+ */
+class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonCommandPalette() {
+    this.getFile().getBaseName() = "package.json" and
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson |
+      packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
+    ) and
+    not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+from CommandName c
+where c.getNumberOfUsages() > 1 and not c instanceof BuiltInVSCodeCommand
+select c.getFirstUsage(),
+  "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"

.github/workflows/bump-cli.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Bump CLI

.github/workflows/cli-test.yml

@@ -0,0 +1,111 @@
+name: Run CLI tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - .github/workflows/cli-test.yml
+      - extensions/ql-vscode/src/codeql-cli/**
+      - extensions/ql-vscode/src/language-support/**
+      - extensions/ql-vscode/src/query-server/**
+      - extensions/ql-vscode/supported_cli_versions.json
+
+jobs:
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
+
+  set-matrix:
+    name: Set Matrix for cli-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set the variables
+        id: set-variables
+        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+    outputs:
+      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
+      fail-fast: false
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.17.1'
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm ci
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Decide on ref of CodeQL repo
+        id: choose-ref
+        shell: bash
+        run: |
+          if [[ "${{ matrix.version }}" == "nightly" ]]
+          then
+            REF="codeql-cli/latest"
+          else
+            REF="codeql-cli/${{ matrix.version }}"
+          fi
+          echo "ref=$REF" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout QL
+        uses: actions/checkout@v4
+        with:
+          repository: github/codeql
+          ref: ${{ steps.choose-ref.outputs.ref }}
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          unset DBUS_SESSION_BUS_ADDRESS
+          /usr/bin/xvfb-run npm run test:cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run test:cli-integration

.github/workflows/codeql.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@main

.github/workflows/dependency-review.yml

@@ -11,6 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@v3

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -53,28 +53,12 @@ jobs:
           name: vscode-codeql-extension
           path: artifacts
 
-  find-nightly:
-    name: Find Nightly Release
-    runs-on: ubuntu-latest
-    outputs:
-      url: ${{ steps.get-url.outputs.nightly-url }}
-    steps:
-      - name: Get Nightly Release URL
-        id: get-url
-        env:
-          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
-        shell: bash
-        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
-        run: |
-          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
-          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
-
   lint:
     name: Lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -110,6 +94,47 @@ jobs:
         run: |
           npm run lint:scenarios
 
+      - name: Find deadcode
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run find-deadcode
+
+  generated:
+    name: Check generated code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.17.1'
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm ci
+        shell: bash
+
+      - name: Check that repo is clean
+        run: |
+          git diff --exit-code
+          git diff --exit-code --cached
+
+      - name: Generate code
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run generate
+
+      - name: Check for changes
+        run: |
+          git diff --exit-code
+          git diff --exit-code --cached
+
   unit-test:
     name: Unit Test
     runs-on: ${{ matrix.os }}
@@ -118,7 +143,7 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -152,7 +177,7 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -193,36 +218,36 @@ jobs:
         run: |
           npm run test:vscode-integration
 
-  set-matrix:
-    name: Set Matrix for cli-test
+  get-latest-cli-version:
+    name: Get latest CLI version
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Set the variables
-        id: set-variables
-        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+        uses: actions/checkout@v4
+      - name: Set the variable
+        id: set-variable
+        run: |
+          echo "cli-version=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc '.[0]')" >> $GITHUB_OUTPUT
+          echo "$cli-version"
     outputs:
-      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+      cli-version: ${{ steps.set-variable.outputs.cli-version }}
 
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}
-    needs: [find-nightly, set-matrix]
+    needs: [get-latest-cli-version]
     timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
       fail-fast: false
     env:
-      CLI_VERSION: ${{ matrix.version }}
-      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      CLI_VERSION: ${{ needs.get-latest-cli-version.outputs.cli-version }}
       TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/setup-node@v3
         with:
@@ -242,23 +267,11 @@ jobs:
           npm run build
         shell: bash
 
-      - name: Decide on ref of CodeQL repo
-        id: choose-ref
-        shell: bash
-        run: |
-          if [[ "${{ matrix.version }}" == "nightly" ]]
-          then
-            REF="codeql-cli/latest"
-          else
-            REF="codeql-cli/${{ matrix.version }}"
-          fi
-          echo "ref=$REF" >> "$GITHUB_OUTPUT"
-
       - name: Checkout QL
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: github/codeql
-          ref: ${{ steps.choose-ref.outputs.ref }}
+          ref: 'codeql-cli/${{ needs.get-latest-cli-version.outputs.cli-version }}'
           path: codeql
 
       - name: Run CLI tests (Linux)

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/setup-node@v3
         with:

CODEOWNERS

@@ -1,5 +1,5 @@
 **/* @github/codeql-vscode-reviewers
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
-**/data-extensions-editor/ @github/code-scanning-secexp-reviewers
+**/model-editor/ @github/code-scanning-secexp-reviewers
 **/queries-panel/ @github/code-scanning-secexp-reviewers

docs/node-version.md

@@ -0,0 +1,29 @@
+# Node version
+
+The CodeQL for VS Code extension defines the version of Node.js that it is intended to run with. This Node.js version is used when running most CI and unit tests.
+
+When running in production (i.e. as an extension for a VS Code application) it will use the Node.js version provided by VS Code. This can mean a different Node.js version is used by different users with different versions of VS Code.
+We should make sure the CodeQL for VS Code extension works with the Node.js version supplied by all versions of VS Code that we support.
+
+## Checking the version of Node.js supplied by VS Code
+
+You can find this info by seleting "About Visual Studio Code" from the top menu.
+
+![about-vscode](images/about-vscode.png)
+
+## Updating the Node.js version
+
+The following files will need to be updated:
+
+- `.github/workflows/cli-test.yml` - the "node-version: '[VERSION]'" setting
+- `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
+- `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting
+- `extensions/ql-vscode/.nvmrc` - this will enable nvm to automatically switch to the correct node version when you're in the project folder
+- `extensions/ql-vscode/package-lock.json` - the "engines.node: '[VERSION]'" setting
+- `extensions/ql-vscode/package.json` - the "engines.node: '[VERSION]'" setting
+
+## Node.js version used in tests
+
+Unit tests will use whatever version of Node.js is installed locally. In CI this will be the version specified in the workflow.
+
+Integration tests download a copy of VS Code and then will use whatever version of Node.js is provided by VS Code. Our integration tests are currently pinned to an older version of VS Code. See [VS Code version used in tests](./vscode-version.md#vs-code-version-used-in-tests) for more information.

docs/releasing.md

@@ -1,5 +1,6 @@
 # Releasing (write access required)
 
+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
@@ -10,10 +11,7 @@
       * New telemetry events are added.
       * Deprecation or removal of commands.
       * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. You can find this info by seleting "About Visual Studio Code" from the top menu. If it doesn't match, you will then need to update the node version in the following files:
-    * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
-    * `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
-    * `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting
+1. Double-check that the node version we're using matches the one used for VS Code. See the [Node.js version instructions](./node-version.md) for more information.
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
 1. Create a PR for this release:
     * This PR will contain any missing bits from steps 1, 2 and 3. Most of the time, this will just be updating `CHANGELOG.md` with today's date.

docs/test-plan.md

@@ -44,23 +44,23 @@ choose to go through some of the Optional Test Cases.
 
 #### Test case 2: Running a problem query and viewing results
 
-1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
 2. Select the `babel/babel` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
    - Check that the result messages are rendered
    - Check that alert locations can be clicked on
 
-#### Test case 3: Running a non-probem query and viewing results
+#### Test case 3: Running a non-problem query and viewing results
 
 1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
 2. Select the `google/brotli` database (or download it if you don't have one already)
 3. Run a local query.
 4. Once the query completes:
    - Check that the results table is rendered
-   - Check that alert locations can be clicked on
+   - Check that result locations can be clicked on
 
-#### Test case 3: Can use AST viewer
+#### Test case 4: Can use AST viewer
 
 1. Click on any code location from a previous query to open a source file from a database
 2. Open the AST viewing panel and click "View AST"
@@ -143,6 +143,41 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
+### CodeQL Model Editor
+
+#### Test Case 1: Opening the model editor
+
+1. Download the `sofastack/sofa-jraft` java database from GitHub.
+2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
+   - Check that the editor loads and shows methods to model.
+   - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
+   - Check that the "Open database" link works.
+
+#### Test Case 2: Model methods
+
+1. Expand one of the libraries.
+   - Change the model type and check that the other dropdowns change.
+2. Save the modeled methods.
+3. Click "Open extension pack"
+   - Check that the file explorer opens a directory with a "models" directory
+4. Open the ".model.yml" file corresponding to the library that was changed.
+   - Check that the file contrains the entries that was modeled.
+
+#### Test Case 3: Model with AI
+
+Note that this test requires the feature flag: `codeQL.model.llmGeneration`
+
+1. Click "Model with AI".
+   - Check that rows change to "Thinking".
+   - Check that results come back and rows get filled out.
+
+#### Test Case 4: Model as dependency
+
+1. Click "Model as dependency"
+   - Check that grouping are now per package (e.g. `com.alipay.sofa.rraft.option` or `com.google.protobuf`)
+2. Click "Generate".
+   - Check that rows are filled out.
+
 ### General
 
 #### Test case 1: Change to a different colour theme
@@ -318,7 +353,6 @@ This requires running a MRVA query and seeing the results view.
    1. Alphabetically
    2. By number of results
    3. By popularity
-   4. By most recent commit
 9. Can filter repos
 10. Shows correct statistics
     1. Total number of results

docs/testing.md

@@ -133,4 +133,4 @@ Once the scenario has been recorded, it's often useful to remove some of the req
 
 ### Scenario data location
 
-Pre-recorded scenarios are stored in `./src/variant-analysis/github-api/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.
+Pre-recorded scenarios are stored in `./src/common/mock-gh-api/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.

docs/vscode-version.md

@@ -0,0 +1,33 @@
+# VS Code version
+
+The CodeQL for VS Code extension specifies the versions of VS Code that it is compatible with. VS Code will only offer to install and upgrade the extension when this version range is satisfied.
+
+## Where is the VS Code version specified
+
+1. Hard limit in [`package.json`](https://github.com/github/vscode-codeql/blob/606bfd7f877d9fffe4ff83b78015ab15f8840b12/extensions/ql-vscode/package.json#L16)
+
+    This is the value that VS Code understands and respects. If a user does not meet this version requirement then VS Code will not offer to install the CodeQL for VS Code extension, and if the extension is already installed then it will silently refuse to upgrade the extension.
+
+1. Soft limit in [`extension.ts`](https://github.com/github/vscode-codeql/blob/606bfd7f877d9fffe4ff83b78015ab15f8840b12/extensions/ql-vscode/src/extension.ts#L307)
+
+    This value is used internally by the CodeQL for VS Code extension and is used to provide a warning to users without blocking them from installing or upgrading. If the extension detects that this version range is not met it will output a warning message to the user prompting them to upgrade their VS Code version to ge the latest features of CodeQL.
+
+## When to update the VS Code version
+
+Generally we should aim to support as wide a range of VS Code versions as we can, so unless there is a reason to do so we do not update the minimum VS Code version requirement.
+Reasons for updating the minimum VS Code version include:
+
+- A new feature is included in VS Code. We may want to ensure that it is available to use so we do not have to provide an alternative code path.
+- A breaking change has happened in VS Code, and it is not possible to support both new and old versions.
+
+Also consider what percentage of our users are using each VS Code version. This information is available in our telemetry.
+
+## How to update the VS Code version
+
+To provide a good experience to users, it is recommented to update the `MIN_VERSION` in `extension.ts` first and release, and then update the `vscode` version in `package.json` and release again. By stagging this update across two releases it gives users on older VS Code versions a chance to upgrade before it silently refuses to upgrade them.
+
+## VS Code version used in tests
+
+Our integration tests are currently pinned to use an older version of VS Code due to <https://github.com/github/vscode-codeql/issues/2402>.
+This version is specified in [`jest-runner-vscode.config.base.js`](https://github.com/github/vscode-codeql/blob/d93f2b67c84e79737b0ce4bb74e31558b5f5166e/extensions/ql-vscode/test/vscode-tests/jest-runner-vscode.config.base.js#L17).
+Until this is resolved this will limit us updating our minimum supported version of VS Code.

extensions/ql-vscode/.storybook/main.ts

@@ -1,4 +1,4 @@
-import type { StorybookConfig } from "@storybook/core-common";
+import type { StorybookConfig } from "@storybook/react-webpack5";
 
 const config: StorybookConfig = {
   stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
@@ -8,13 +8,13 @@ const config: StorybookConfig = {
     "@storybook/addon-interactions",
     "./vscode-theme-addon/preset.ts",
   ],
-  framework: "@storybook/react",
-  core: {
-    builder: "@storybook/builder-webpack5",
+  framework: {
+    name: "@storybook/react-webpack5",
+    options: {},
   },
-  features: {
-    babelModeV7: true,
+  docs: {
+    autodocs: "tag",
   },
 };
 
-module.exports = config;
+export default config;

extensions/ql-vscode/.storybook/manager.ts

@@ -1,4 +1,4 @@
-import { addons } from "@storybook/addons";
+import { addons } from "@storybook/manager-api";
 import { themes } from "@storybook/theming";
 
 addons.setConfig({

extensions/ql-vscode/.storybook/preview.ts

@@ -1,31 +1,44 @@
+import { Preview } from "@storybook/react";
 import { themes } from "@storybook/theming";
 import { action } from "@storybook/addon-actions";
 
 // Allow all stories/components to use Codicons
 import "@vscode/codicons/dist/codicon.css";
 
-// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
-export const parameters = {
-  // All props starting with `on` will automatically receive an action as a prop
-  actions: { argTypesRegex: "^on[A-Z].*" },
-  // All props matching these names will automatically get the correct control
-  controls: {
-    matchers: {
-      color: /(background|color)$/i,
-      date: /Date$/,
-    },
-  },
-  // Use a dark theme to be aligned with VSCode
-  docs: {
-    theme: themes.dark,
-  },
-  backgrounds: {
-    // The background is injected by our theme CSS files
-    disable: true,
-  },
-};
+import type { VsCodeApi } from "../src/view/vscode-api";
 
-(window as any).acquireVsCodeApi = () => ({
+declare global {
+  interface Window {
+    acquireVsCodeApi: () => VsCodeApi;
+  }
+}
+
+window.acquireVsCodeApi = () => ({
   postMessage: action("post-vscode-message"),
   setState: action("set-vscode-state"),
 });
+
+// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
+const preview: Preview = {
+  parameters: {
+    // All props starting with `on` will automatically receive an action as a prop
+    actions: { argTypesRegex: "^on[A-Z].*" },
+    // All props matching these names will automatically get the correct control
+    controls: {
+      matchers: {
+        color: /(background|color)$/i,
+        date: /Date$/,
+      },
+    },
+    // Use a dark theme to be aligned with VSCode
+    docs: {
+      theme: themes.dark,
+    },
+    backgrounds: {
+      // The background is injected by our theme CSS files
+      disable: true,
+    },
+  },
+};
+
+export default preview;

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -1,14 +1,12 @@
 import * as React from "react";
 import { FunctionComponent, useCallback } from "react";
 
-import { useGlobals } from "@storybook/api";
+import { useGlobals } from "@storybook/manager-api";
 import {
   IconButton,
   Icons,
-  WithTooltip,
   TooltipLinkList,
-  Link,
-  WithHideFn,
+  WithTooltip,
 } from "@storybook/components";
 
 import { themeNames, VSCodeTheme } from "./theme";
@@ -26,7 +24,7 @@ export const ThemeSelector: FunctionComponent = () => {
   );
 
   const createLinks = useCallback(
-    (onHide: () => void): Link[] =>
+    (onHide: () => void) =>
       Object.values(VSCodeTheme).map((theme) => ({
         id: theme,
         onClick() {
@@ -44,8 +42,8 @@ export const ThemeSelector: FunctionComponent = () => {
     <WithTooltip
       placement="top"
       trigger="click"
-      closeOnClick
-      tooltip={({ onHide }: WithHideFn) => (
+      closeOnOutsideClick
+      tooltip={({ onHide }: { onHide: () => void }) => (
         <TooltipLinkList links={createLinks(onHide)} />
       )}
     >

extensions/ql-vscode/.storybook/vscode-theme-addon/manager.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { addons, types } from "@storybook/addons";
+import { addons, types } from "@storybook/manager-api";
 import { ThemeSelector } from "./ThemeSelector";
 
 const ADDON_ID = "vscode-theme-addon";

extensions/ql-vscode/.storybook/vscode-theme-addon/preset.ts

@@ -1,4 +1,4 @@
-export function config(entry = []) {
+export function previewAnnotations(entry = []) {
   return [...entry, require.resolve("./preview.ts")];
 }
 

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -1,6 +1,5 @@
-import { useEffect, useGlobals } from "@storybook/addons";
+import { useEffect } from "react";
 import type {
-  AnyFramework,
   PartialStoryFn as StoryFunction,
   StoryContext,
 } from "@storybook/csf";
@@ -34,11 +33,8 @@ const themeFiles: { [key in VSCodeTheme]: string } = {
       .default,
 };
 
-export const withTheme = (
-  StoryFn: StoryFunction<AnyFramework>,
-  context: StoryContext<AnyFramework>,
-) => {
-  const [{ vscodeTheme }] = useGlobals();
+export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {
+  const { vscodeTheme } = context.globals;
 
   useEffect(() => {
     const styleSelectorId =

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,37 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.9.1 - 29 September 2023
+
+- Add warning when using a VS Code version older than 1.82.0. [#2854](https://github.com/github/vscode-codeql/pull/2854)
+- Fix a bug when parsing large evaluation log summaries. [#2858](https://github.com/github/vscode-codeql/pull/2858)
+- Right-align and format numbers in raw result tables. [#2864](https://github.com/github/vscode-codeql/pull/2864)
+- Remove rate limit warning notifications when using Code Search to add repositories to a variant analysis list. [#2812](https://github.com/github/vscode-codeql/pull/2812)
+
+## 1.9.0 - 19 September 2023
+
+- Release the [CodeQL model editor](https://codeql.github.com/docs/codeql/codeql-for-visual-studio-code/using-the-codeql-model-editor) to create CodeQL model packs for Java frameworks. Open the editor using the "CodeQL: Open CodeQL Model Editor (Beta)" command. [#2823](https://github.com/github/vscode-codeql/pull/2823)
+
+## 1.8.12 - 11 September 2023
+
+- Fix a bug where variant analysis queries would fail for queries in the `codeql/java-queries` query pack. [#2786](https://github.com/github/vscode-codeql/pull/2786)
+
+## 1.8.11 - 7 September 2023
+
+- Update how variant analysis results are displayed. For queries with ["path-problem" or "problem" `@kind`](https://codeql.github.com/docs/writing-codeql-queries/metadata-for-codeql-queries/#metadata-properties), you can choose to display the results as rendered alerts or as a table of raw results. For queries with any other `@kind`, the results are displayed as a table. [#2745](https://github.com/github/vscode-codeql/pull/2745) & [#2749](https://github.com/github/vscode-codeql/pull/2749)
+- When running variant analyses, don't download artifacts for repositories with no results. [#2736](https://github.com/github/vscode-codeql/pull/2736)
+- Group the extension settings, so that they're easier to find in the Settings UI. [#2706](https://github.com/github/vscode-codeql/pull/2706)
+
+## 1.8.10 - 15 August 2023
+
+- Add a code lens to make the `CodeQL: Open Referenced File` command more discoverable. Click the "Open referenced file" prompt in a `.qlref` file to jump to the referenced `.ql` file. [#2704](https://github.com/github/vscode-codeql/pull/2704)
+
+## 1.8.9 - 3 August 2023
+
+- Remove "last updated" information and sorting from variant analysis results view. [#2637](https://github.com/github/vscode-codeql/pull/2637)
+- Links to code on GitHub now include column numbers as well as line numbers. [#2406](https://github.com/github/vscode-codeql/pull/2406)
+- No longer highlight trailing commas for jump to definition.  [#2615](https://github.com/github/vscode-codeql/pull/2615)
+- Fix a bug where the QHelp preview page was not being refreshed after changes to the underlying `.qhelp` file. [#2660](https://github.com/github/vscode-codeql/pull/2660)
+
 ## 1.8.8 - 17 July 2023
 
 - Remove support for CodeQL CLI versions older than 2.9.4. [#2610](https://github.com/github/vscode-codeql/pull/2610)

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -9,6 +9,7 @@ import {
 } from "fs-extra";
 import { resolve, join } from "path";
 import { isDevBuild } from "./dev";
+import type * as packageJsonType from "../package.json";
 
 export interface DeployedPackage {
   distPath: string;
@@ -46,12 +47,10 @@ async function copyPackage(
   );
 }
 
-export async function deployPackage(
-  packageJsonPath: string,
-): Promise<DeployedPackage> {
+export async function deployPackage(): Promise<DeployedPackage> {
   try {
-    const packageJson: any = JSON.parse(
-      await readFile(packageJsonPath, "utf8"),
+    const packageJson: typeof packageJsonType = JSON.parse(
+      await readFile(resolve(__dirname, "../package.json"), "utf8"),
     );
 
     const distDir = join(__dirname, "../../../dist");

extensions/ql-vscode/gulpfile.ts/package.ts

@@ -3,9 +3,7 @@ import { deployPackage } from "./deploy";
 import { spawn } from "child-process-promise";
 
 export async function packageExtension(): Promise<void> {
-  const deployedPackage = await deployPackage(
-    resolve(__dirname, "../package.json"),
-  );
+  const deployedPackage = await deployPackage();
   console.log(
     `Packaging extension '${deployedPackage.name}@${deployedPackage.version}'...`,
   );

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/tsconfig",
   "extends": "../tsconfig.json",
   "compilerOptions": {
-    "rootDir": "."
+    "rootDir": ".."
   },
   "include": ["*.ts"]
 }

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -15,9 +15,6 @@ export const config: webpack.Configuration = {
   devtool: isDevBuild ? "inline-source-map" : "source-map",
   resolve: {
     extensions: [".js", ".ts", ".tsx", ".json"],
-    fallback: {
-      path: require.resolve("path-browserify"),
-    },
   },
   module: {
     rules: [

extensions/ql-vscode/language-configuration.json

@@ -30,5 +30,5 @@
       "end": "^\\s*//\\s*#?endregion\\b"
     }
   },
-  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s\\,]+)"
 }

extensions/ql-vscode/media/dark/sort-alpha.svg

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
-	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
-<g>
-	<g>
-		<g>
-			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
-			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72
-				250.027,338.24 432,338.24 432,304.32" fill="#C5C5C5"/>
-			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
-			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
-				L113.28,140.48l41.387,110.507H71.893z" fill="#C5C5C5"/>
-		</g>
-	</g>
-</g>
-</svg>

extensions/ql-vscode/media/dark/sort-date.svg

@@ -1,3 +0,0 @@
-<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#C5C5C5"/>
-</svg>

extensions/ql-vscode/media/light/sort-alpha.svg

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
-<g>
-	<g>
-		<g>
-			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413 			"/>
-			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72 
-				250.027,338.24 432,338.24 432,304.32 			"/>
-			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227 			"/>
-			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
-				L113.28,140.48l41.387,110.507H71.893z"/>
-		</g>
-	</g>
-</g>
-</svg>

extensions/ql-vscode/media/light/sort-date.svg

@@ -1,3 +0,0 @@
-<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#424242"/>
-</svg>

extensions/ql-vscode/patches/jest-runner-vscode+3.0.1.patch

@@ -25,7 +25,7 @@ index 1ac28d5..f91f216 100644
          await super.setup();
          await (0, load_pnp_1.default)();
 diff --git a/node_modules/jest-runner-vscode/dist/child/runner.js b/node_modules/jest-runner-vscode/dist/child/runner.js
-index 0663c5c..4991663 100644
+index 0663c5c..bdf4a8b 100644
 --- a/node_modules/jest-runner-vscode/dist/child/runner.js
 +++ b/node_modules/jest-runner-vscode/dist/child/runner.js
 @@ -18,10 +18,13 @@ async function run() {
@@ -43,6 +43,16 @@ index 0663c5c..4991663 100644
          const options = JSON.parse(PARENT_JEST_OPTIONS);
          const jestOptions = [
              ...options.args,
+@@ -39,6 +42,9 @@ async function run() {
+             ...(argv.projects?.map(project => path_1.default.resolve(project)) || []),
+             options.workspacePath,
+         ]);
++        const testPaths = new Set(argv._.map(testPath => path_1.default.resolve(testPath)));
++        argv._ = [...testPaths];
++
+         await (0, core_1.runCLI)(argv, [...projects]);
+     }
+     catch (error) {
 diff --git a/node_modules/jest-runner-vscode/dist/public-types.d.ts b/node_modules/jest-runner-vscode/dist/public-types.d.ts
 index 57716e5..d8614af 100644
 --- a/node_modules/jest-runner-vscode/dist/public-types.d.ts

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -18,15 +18,15 @@ import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
 
 import { getFiles } from "./util/files";
-import type { GitHubApiRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
-import { isGetVariantAnalysisRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
+import type { GitHubApiRequest } from "../src/common/mock-gh-api/gh-api-request";
+import { isGetVariantAnalysisRequest } from "../src/common/mock-gh-api/gh-api-request";
 import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
 import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
 
 const extensionDirectory = resolve(__dirname, "..");
 const scenariosDirectory = resolve(
   extensionDirectory,
-  "src/variant-analysis/gh-api/mocks/scenarios",
+  "src/common/mock-gh-api/scenarios",
 );
 
 // Make sure we don't run into rate limits by automatically waiting until we can

extensions/ql-vscode/scripts/find-deadcode.ts

@@ -0,0 +1,48 @@
+import { basename, join, relative, resolve } from "path";
+import analyzeTsConfig from "ts-unused-exports";
+import { containsPath, pathsEqual } from "../src/common/files";
+import { exit } from "process";
+
+function ignoreFile(file: string): boolean {
+  return (
+    containsPath("gulpfile.ts", file) ||
+    containsPath(".storybook", file) ||
+    containsPath(join("src", "stories"), file) ||
+    pathsEqual(
+      join("test", "vscode-tests", "jest-runner-installed-extensions.ts"),
+      file,
+    ) ||
+    basename(file) === "jest.config.ts" ||
+    basename(file) === "index.tsx" ||
+    basename(file) === "index.ts"
+  );
+}
+
+function main() {
+  const repositoryRoot = resolve(join(__dirname, ".."));
+
+  const result = analyzeTsConfig("tsconfig.deadcode.json");
+  let foundUnusedExports = false;
+
+  for (const [filepath, exportNameAndLocations] of Object.entries(result)) {
+    const relativeFilepath = relative(repositoryRoot, filepath);
+
+    if (ignoreFile(relativeFilepath)) {
+      continue;
+    }
+
+    foundUnusedExports = true;
+
+    console.log(relativeFilepath);
+    for (const exportNameAndLocation of exportNameAndLocations) {
+      console.log(`    ${exportNameAndLocation.exportName}`);
+    }
+    console.log();
+  }
+
+  if (foundUnusedExports) {
+    exit(1);
+  }
+}
+
+main();

extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts

@@ -22,7 +22,7 @@ const scenarioName = process.argv[2];
 const extensionDirectory = resolve(__dirname, "..");
 const scenariosDirectory = resolve(
   extensionDirectory,
-  "src/variant-analysis/gh-api/mocks/scenarios",
+  "src/common/mock-gh-api/scenarios",
 );
 const scenarioDirectory = resolve(scenariosDirectory, scenarioName);
 

extensions/ql-vscode/scripts/generate-schemas.ts

@@ -0,0 +1,72 @@
+import { createGenerator } from "ts-json-schema-generator";
+import { join, resolve } from "path";
+import { outputFile } from "fs-extra";
+import { format, resolveConfig } from "prettier";
+
+const extensionDirectory = resolve(__dirname, "..");
+
+const schemas = [
+  {
+    path: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "extension-pack-metadata.ts",
+    ),
+    type: "ExtensionPackMetadata",
+    schemaPath: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "extension-pack-metadata.schema.json",
+    ),
+  },
+  {
+    path: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "model-extension-file.ts",
+    ),
+    type: "ModelExtensionFile",
+    schemaPath: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "model-extension-file.schema.json",
+    ),
+  },
+];
+
+async function generateSchema(
+  schemaDefinition: (typeof schemas)[number],
+): Promise<void> {
+  const schema = createGenerator({
+    path: schemaDefinition.path,
+    tsconfig: resolve(extensionDirectory, "tsconfig.json"),
+    type: schemaDefinition.type,
+    skipTypeCheck: true,
+    topRef: true,
+    additionalProperties: true,
+  }).createSchema(schemaDefinition.type);
+
+  const schemaJson = JSON.stringify(schema, null, 2);
+
+  const prettierOptions = await resolveConfig(schemaDefinition.schemaPath);
+
+  const formattedSchemaJson = await format(schemaJson, {
+    ...prettierOptions,
+    filepath: schemaDefinition.schemaPath,
+  });
+
+  await outputFile(schemaDefinition.schemaPath, formattedSchemaJson);
+}
+
+async function generateSchemas() {
+  await Promise.all(schemas.map(generateSchema));
+}
+
+generateSchemas().catch((e: unknown) => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -10,7 +10,7 @@ const extensionDirectory = resolve(__dirname, "..");
 const rootDirectory = resolve(extensionDirectory, "../..");
 const scenariosDirectory = resolve(
   extensionDirectory,
-  "src/variant-analysis/gh-api/mocks/scenarios",
+  "src/common/mock-gh-api/scenarios",
 );
 
 const debug = process.env.RUNNER_DEBUG || process.argv.includes("--debug");
@@ -19,7 +19,7 @@ async function lintScenarios() {
   const schema = createGenerator({
     path: resolve(
       extensionDirectory,
-      "src/variant-analysis/gh-api/mocks/gh-api-request.ts",
+      "src/common/mock-gh-api/gh-api-request.ts",
     ),
     tsconfig: resolve(extensionDirectory, "tsconfig.json"),
     type: "GitHubApiRequest",

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -6,7 +6,6 @@ import { dirname, join, delimiter } from "path";
 import * as sarif from "sarif";
 import { SemVer } from "semver";
 import { Readable } from "stream";
-import { StringDecoder } from "string_decoder";
 import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
@@ -19,6 +18,7 @@ import {
 } from "./distribution";
 import {
   assertNever,
+  getChildProcessErrorMessage,
   getErrorMessage,
   getErrorStack,
 } from "../common/helpers-pure";
@@ -30,6 +30,7 @@ import { CompilationMessage } from "../query-server/legacy-messages";
 import { sarifParser } from "../common/sarif-parser";
 import { App } from "../common/app";
 import { QueryLanguage } from "../common/query-language";
+import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
 
 /**
  * The version of the SARIF format that we are using.
@@ -123,6 +124,15 @@ export type ResolveExtensionsResult = {
   };
 };
 
+export type GenerateExtensiblePredicateMetadataResult = {
+  // There are other properties in this object, but they are
+  // not relevant for its use in the extension, so we omit them.
+  extensible_predicates: Array<{
+    // pack relative path
+    path: string;
+  }>;
+};
+
 /**
  * The expected output of `codeql resolve qlref`.
  */
@@ -538,9 +548,7 @@ export class CodeQLCliServer implements Disposable {
         yield JSON.parse(event) as EventType;
       } catch (err) {
         throw new Error(
-          `Parsing output of ${description} failed: ${
-            (err as any).stderr || getErrorMessage(err)
-          }`,
+          `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
         );
       }
     }
@@ -638,9 +646,7 @@ export class CodeQLCliServer implements Disposable {
       return JSON.parse(result) as OutputType;
     } catch (err) {
       throw new Error(
-        `Parsing output of ${description} failed: ${
-          (err as any).stderr || getErrorMessage(err)
-        }`,
+        `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
       );
     }
   }
@@ -1428,21 +1434,13 @@ export class CodeQLCliServer implements Disposable {
 
   async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
     const args = includeQueries ? [dir] : ["--no-include-queries", dir];
-    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
-    // previous versions return a list of packs.
-    const results: { paths: string[] } | string[] =
-      await this.runJsonCodeQlCliCommand(
-        ["pack", "packlist"],
-        args,
-        "Generating the pack list",
-      );
+    const results: { paths: string[] } = await this.runJsonCodeQlCliCommand(
+      ["pack", "packlist"],
+      args,
+      "Generating the pack list",
+    );
 
-    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
-    if ("paths" in results) {
-      return results.paths;
-    } else {
-      return results;
-    }
+    return results.paths;
   }
 
   async packResolveDependencies(
@@ -1466,6 +1464,17 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  async generateExtensiblePredicateMetadata(
+    packRoot: string,
+  ): Promise<GenerateExtensiblePredicateMetadataResult> {
+    return await this.runJsonCodeQlCliCommand(
+      ["generate", "extensible-predicate-metadata"],
+      [packRoot],
+      "Generating extensible predicate metadata",
+      { addFormat: false },
+    );
+  }
+
   public async getVersion() {
     if (!this._version) {
       try {
@@ -1476,13 +1485,6 @@ export class CodeQLCliServer implements Disposable {
         );
 
         // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
-        await this.app.commands.execute(
-          "setContext",
-          "codeql.supportsEvalLog",
-          newVersion.compare(
-            CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
-          ) >= 0,
-        );
         await this.app.commands.execute(
           "setContext",
           "codeql.supportsQuickEvalCount",
@@ -1642,125 +1644,18 @@ export async function runCodeQlCliCommand(
     return result.stdout;
   } catch (err) {
     throw new Error(
-      `${description} failed: ${(err as any).stderr || getErrorMessage(err)}`,
+      `${description} failed: ${getChildProcessErrorMessage(err)}`,
     );
   }
 }
 
-/**
- * Buffer to hold state used when splitting a text stream into lines.
- */
-class SplitBuffer {
-  private readonly decoder = new StringDecoder("utf8");
-  private readonly maxSeparatorLength: number;
-  private buffer = "";
-  private searchIndex = 0;
-
-  constructor(private readonly separators: readonly string[]) {
-    this.maxSeparatorLength = separators
-      .map((s) => s.length)
-      .reduce((a, b) => Math.max(a, b), 0);
-  }
-
-  /**
-   * Append new text data to the buffer.
-   * @param chunk The chunk of data to append.
-   */
-  public addChunk(chunk: Buffer): void {
-    this.buffer += this.decoder.write(chunk);
-  }
-
-  /**
-   * Signal that the end of the input stream has been reached.
-   */
-  public end(): void {
-    this.buffer += this.decoder.end();
-    this.buffer += this.separators[0]; // Append a separator to the end to ensure the last line is returned.
-  }
-
-  /**
-   * A version of startsWith that isn't overriden by a broken version of ms-python.
-   *
-   * The definition comes from
-   * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
-   * which is CC0/public domain
-   *
-   * See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
-   */
-  private static startsWith(
-    s: string,
-    searchString: string,
-    position: number,
-  ): boolean {
-    const pos = position > 0 ? position | 0 : 0;
-    return s.substring(pos, pos + searchString.length) === searchString;
-  }
-
-  /**
-   * Extract the next full line from the buffer, if one is available.
-   * @returns The text of the next available full line (without the separator), or `undefined` if no
-   * line is available.
-   */
-  public getNextLine(): string | undefined {
-    while (this.searchIndex <= this.buffer.length - this.maxSeparatorLength) {
-      for (const separator of this.separators) {
-        if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
-          const line = this.buffer.slice(0, this.searchIndex);
-          this.buffer = this.buffer.slice(this.searchIndex + separator.length);
-          this.searchIndex = 0;
-          return line;
-        }
-      }
-      this.searchIndex++;
-    }
-
-    return undefined;
-  }
-}
-
-/**
- * Splits a text stream into lines based on a list of valid line separators.
- * @param stream The text stream to split. This stream will be fully consumed.
- * @param separators The list of strings that act as line separators.
- * @returns A sequence of lines (not including separators).
- */
-async function* splitStreamAtSeparators(
-  stream: Readable,
-  separators: string[],
-): AsyncGenerator<string, void, unknown> {
-  const buffer = new SplitBuffer(separators);
-  for await (const chunk of stream) {
-    buffer.addChunk(chunk);
-    let line: string | undefined;
-    do {
-      line = buffer.getNextLine();
-      if (line !== undefined) {
-        yield line;
-      }
-    } while (line !== undefined);
-  }
-  buffer.end();
-  let line: string | undefined;
-  do {
-    line = buffer.getNextLine();
-    if (line !== undefined) {
-      yield line;
-    }
-  } while (line !== undefined);
-}
-
-/**
- *  Standard line endings for splitting human-readable text.
- */
-const lineEndings = ["\r\n", "\r", "\n"];
-
 /**
  * Log a text stream to a `Logger` interface.
  * @param stream The stream to log.
  * @param logger The logger that will consume the stream output.
  */
 async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
-  for await (const line of splitStreamAtSeparators(stream, lineEndings)) {
+  for await (const line of splitStreamAtSeparators(stream, LINE_ENDINGS)) {
     // Await the result of log here in order to ensure the logs are written in the correct order.
     await logger.log(line);
   }
@@ -1807,21 +1702,9 @@ export class CliVersionConstraint {
   );
 
   /**
-   * CLI version where the `--evaluator-log` and related options to the query server were introduced,
-   * on a per-query server basis.
+   * CLI version where the `resolve extensions` subcommand exists.
    */
-  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer("2.8.2");
-
-  /**
-   * CLI version that supports rotating structured logs to produce one per query.
-   *
-   * Note that 2.8.4 supports generating the evaluation logs and summaries,
-   * but 2.9.0 includes a new option to produce the end-of-query summary logs to
-   * the query server console. For simplicity we gate all features behind 2.9.0,
-   * but if a user is tied to the 2.8 release, we can enable evaluator logs
-   * and summaries for them.
-   */
-  public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer("2.9.0");
+  public static CLI_VERSION_WITH_RESOLVE_EXTENSIONS = new SemVer("2.10.2");
 
   /**
    * CLI version that supports the `--sourcemap` option for log generation.
@@ -1857,6 +1740,14 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
 
+  /**
+   * CLI version where the `generate extensible-predicate-metadata`
+   * command was implemented.
+   */
+  public static CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA = new SemVer(
+    "2.14.3",
+  );
+
   /**
    * CLI version where the langauge server supports visisbility change notifications.
    */
@@ -1882,15 +1773,9 @@ export class CliVersionConstraint {
     );
   }
 
-  async supportsStructuredEvalLog() {
+  async supportsResolveExtensions() {
     return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG,
-    );
-  }
-
-  async supportsPerQueryEvalLog() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+      CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS,
     );
   }
 
@@ -1949,4 +1834,10 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
     );
   }
+
+  async supportsGenerateExtensiblePredicateMetadata() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -718,7 +718,7 @@ export enum DistributionKind {
   PathEnvironmentVariable,
 }
 
-export interface Distribution {
+interface Distribution {
   codeQlPath: string;
   kind: DistributionKind;
 }
@@ -776,22 +776,22 @@ type DistributionUpdateCheckResult =
   | InvalidLocationResult
   | UpdateAvailableResult;
 
-export interface AlreadyCheckedRecentlyResult {
+interface AlreadyCheckedRecentlyResult {
   kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
 }
 
-export interface AlreadyUpToDateResult {
+interface AlreadyUpToDateResult {
   kind: DistributionUpdateCheckResultKind.AlreadyUpToDate;
 }
 
 /**
  * The distribution could not be installed or updated because it is not managed by the extension.
  */
-export interface InvalidLocationResult {
+interface InvalidLocationResult {
   kind: DistributionUpdateCheckResultKind.InvalidLocation;
 }
 
-export interface UpdateAvailableResult {
+interface UpdateAvailableResult {
   kind: DistributionUpdateCheckResultKind.UpdateAvailable;
   updatedRelease: Release;
 }
@@ -862,7 +862,7 @@ function warnDeprecatedLauncher() {
 /**
  * A release on GitHub.
  */
-export interface Release {
+interface Release {
   assets: ReleaseAsset[];
 
   /**
@@ -884,7 +884,7 @@ export interface Release {
 /**
  * An asset corresponding to a release on GitHub.
  */
-export interface ReleaseAsset {
+interface ReleaseAsset {
   /**
    * The id associated with the asset on GitHub.
    */

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -1,5 +1,3 @@
-export const PAGE_SIZE = 1000;
-
 /**
  * The single-character codes used in the bqrs format for the the kind
  * of a result column. This namespace is intentionally not an enum, see
@@ -15,7 +13,7 @@ export namespace ColumnKindCode {
   export const ENTITY = "e";
 }
 
-export type ColumnKind =
+type ColumnKind =
   | typeof ColumnKindCode.FLOAT
   | typeof ColumnKindCode.INTEGER
   | typeof ColumnKindCode.STRING
@@ -46,7 +44,7 @@ export function getResultSetSchema(
   }
   return undefined;
 }
-export interface PaginationInfo {
+interface PaginationInfo {
   "step-size": number;
   offsets: number[];
 }

extensions/ql-vscode/src/common/bqrs-utils.ts

@@ -142,5 +142,7 @@ export function tryGetRemoteLocation(
     fileLink,
     resolvableLocation.startLine,
     resolvableLocation.endLine,
+    resolvableLocation.startColumn,
+    resolvableLocation.endColumn,
   );
 }

extensions/ql-vscode/src/common/commands.ts

@@ -4,7 +4,6 @@ import type { AstItem } from "../language-support";
 import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
 import type { DatabaseItem } from "../databases/local-databases";
 import type { QueryHistoryInfo } from "../query-history/query-history-info";
-import type { RepositoriesFilterSortStateWithIds } from "../variant-analysis/shared/variant-analysis-filter-sort";
 import type { TestTreeNode } from "../query-testing/test-tree-node";
 import type {
   VariantAnalysis,
@@ -13,6 +12,7 @@ import type {
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import type { Method, Usage } from "../model-editor/method";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -56,10 +56,11 @@ export type ExplorerSelectionCommandFunction<Item> = (
 
 // Builtin commands where the implementation is provided by VS Code and not by this extension.
 // See https://code.visualstudio.com/api/references/commands
-export type BuiltInVsCodeCommands = {
+type BuiltInVsCodeCommands = {
   // The codeQLDatabases.focus command is provided by VS Code because we've registered the custom view
   "codeQLDatabases.focus": () => Promise<void>;
   "markdown.showPreviewToSide": (uri: Uri) => Promise<void>;
+  "workbench.action.closeActiveEditor": () => Promise<void>;
   revealFileInOS: (uri: Uri) => Promise<void>;
   setContext: (
     key: `${"codeql" | "codeQL"}${string}`,
@@ -134,6 +135,7 @@ export type LocalQueryCommands = {
   "codeQLQueries.runLocalQueryContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
   "codeQLQueries.runLocalQueriesContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
   "codeQLQueries.runLocalQueriesFromPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.createQuery": () => Promise<void>;
   "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise<void>;
   "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
   "codeQL.quickEval": (uri: Uri) => Promise<void>;
@@ -217,6 +219,24 @@ export type LocalDatabasesCommands = {
   "codeQLDatabases.chooseDatabaseGithub": () => Promise<void>;
   "codeQLDatabases.sortByName": () => Promise<void>;
   "codeQLDatabases.sortByDateAdded": () => Promise<void>;
+  "codeQLDatabases.displayAllLanguages": () => Promise<void>;
+  "codeQLDatabases.displayCpp": () => Promise<void>;
+  "codeQLDatabases.displayCsharp": () => Promise<void>;
+  "codeQLDatabases.displayGo": () => Promise<void>;
+  "codeQLDatabases.displayJava": () => Promise<void>;
+  "codeQLDatabases.displayJavascript": () => Promise<void>;
+  "codeQLDatabases.displayPython": () => Promise<void>;
+  "codeQLDatabases.displayRuby": () => Promise<void>;
+  "codeQLDatabases.displaySwift": () => Promise<void>;
+  "codeQLDatabases.displayAllLanguagesSelected": () => Promise<void>;
+  "codeQLDatabases.displayCppSelected": () => Promise<void>;
+  "codeQLDatabases.displayCsharpSelected": () => Promise<void>;
+  "codeQLDatabases.displayGoSelected": () => Promise<void>;
+  "codeQLDatabases.displayJavaSelected": () => Promise<void>;
+  "codeQLDatabases.displayJavascriptSelected": () => Promise<void>;
+  "codeQLDatabases.displayPythonSelected": () => Promise<void>;
+  "codeQLDatabases.displayRubySelected": () => Promise<void>;
+  "codeQLDatabases.displaySwiftSelected": () => Promise<void>;
 
   // Database panel context menu
   "codeQLDatabases.setCurrentDatabase": (
@@ -244,10 +264,6 @@ export type VariantAnalysisCommands = {
     scannedRepo: VariantAnalysisScannedRepository,
     variantAnalysisSummary: VariantAnalysis,
   ) => Promise<void>;
-  "codeQL.copyVariantAnalysisRepoList": (
-    variantAnalysisId: number,
-    filterSort?: RepositoriesFilterSortStateWithIds,
-  ) => Promise<void>;
   "codeQL.loadVariantAnalysisRepoResults": (
     variantAnalysisId: number,
     repositoryFullName: string,
@@ -305,8 +321,13 @@ export type PackagingCommands = {
   "codeQL.downloadPacks": () => Promise<void>;
 };
 
-export type DataExtensionsEditorCommands = {
-  "codeQL.openDataExtensionsEditor": () => Promise<void>;
+export type ModelEditorCommands = {
+  "codeQL.openModelEditor": () => Promise<void>;
+  "codeQLModelEditor.jumpToUsageLocation": (
+    method: Method,
+    usage: Usage,
+    databaseItem: DatabaseItem,
+  ) => Promise<void>;
 };
 
 export type EvalLogViewerCommands = {
@@ -346,7 +367,7 @@ export type AllExtensionCommands = BaseCommands &
   AstCfgCommands &
   AstViewerCommands &
   PackagingCommands &
-  DataExtensionsEditorCommands &
+  ModelEditorCommands &
   EvalLogViewerCommands &
   SummaryLanguageSupportCommands &
   Partial<TestUICommands> &

extensions/ql-vscode/src/common/helpers-pure.ts

@@ -67,3 +67,26 @@ export function asError(e: unknown): Error {
 
   return e instanceof Error ? e : new Error(String(e));
 }
+
+/**
+ * Get error message when the error may have come from a method from the `child_process` module.
+ */
+export function getChildProcessErrorMessage(e: unknown): string {
+  return isChildProcessError(e) ? e.stderr : getErrorMessage(e);
+}
+
+/**
+ * Error thrown from methods from the `child_process` module.
+ */
+interface ChildProcessError {
+  readonly stderr: string;
+}
+
+function isChildProcessError(e: unknown): e is ChildProcessError {
+  return (
+    typeof e === "object" &&
+    e !== null &&
+    "stderr" in e &&
+    typeof e.stderr === "string"
+  );
+}

extensions/ql-vscode/src/common/interface-types.ts

@@ -17,10 +17,10 @@ import {
 } from "../variant-analysis/shared/variant-analysis-filter-sort";
 import { ErrorLike } from "../common/errors";
 import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
-import { ExternalApiUsage } from "../data-extensions-editor/external-api-usage";
-import { ModeledMethod } from "../data-extensions-editor/modeled-method";
-import { DataExtensionEditorViewState } from "../data-extensions-editor/shared/view-state";
-import { Mode } from "../data-extensions-editor/shared/mode";
+import { Method, Usage } from "../model-editor/method";
+import { ModeledMethod } from "../model-editor/modeled-method";
+import { ModelEditorViewState } from "../model-editor/shared/view-state";
+import { Mode } from "../model-editor/shared/mode";
 
 /**
  * This module contains types and code that are shared between
@@ -76,11 +76,9 @@ export type GraphInterpretationData = {
   dot: string[];
 };
 
-export type InterpretationData =
-  | SarifInterpretationData
-  | GraphInterpretationData;
+type InterpretationData = SarifInterpretationData | GraphInterpretationData;
 
-export interface InterpretationT<T> {
+interface InterpretationT<T> {
   sourceLocationPrefix: string;
   numTruncatedResults: number;
   numTotalResults: number;
@@ -106,7 +104,7 @@ export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
  *
  * As a result of receiving this message, listeners might want to display a loading indicator.
  */
-export interface ResultsUpdatingMsg {
+interface ResultsUpdatingMsg {
   t: "resultsUpdating";
 }
 
@@ -114,7 +112,7 @@ export interface ResultsUpdatingMsg {
  * Message to set the initial state of the results view with a new
  * query.
  */
-export interface SetStateMsg {
+interface SetStateMsg {
   t: "setState";
   resultsPath: string;
   origResultsPaths: ResultsPaths;
@@ -143,7 +141,7 @@ export interface SetStateMsg {
  * Message indicating that the results view should display interpreted
  * results.
  */
-export interface ShowInterpretedPageMsg {
+interface ShowInterpretedPageMsg {
   t: "showInterpretedPage";
   interpretation: Interpretation;
   database: DatabaseInfo;
@@ -173,7 +171,7 @@ export interface NavigateMsg {
  * A message indicating that the results view should untoggle the
  * "Show results in Problems view" checkbox.
  */
-export interface UntoggleShowProblemsMsg {
+interface UntoggleShowProblemsMsg {
   t: "untoggleShowProblems";
 }
 
@@ -203,7 +201,7 @@ export type FromResultsViewMsg =
  * Message from the results view to open a database source
  * file at the provided location.
  */
-export interface ViewSourceFileMsg {
+interface ViewSourceFileMsg {
   t: "viewSourceFile";
   loc: ResolvableLocationValue;
   databaseUri: string;
@@ -212,7 +210,7 @@ export interface ViewSourceFileMsg {
 /**
  * Message from the results view to open a file in an editor.
  */
-export interface OpenFileMsg {
+interface OpenFileMsg {
   t: "openFile";
   /* Full path to the file to open. */
   filePath: string;
@@ -274,7 +272,7 @@ export interface RawResultsSortState {
   sortDirection: SortDirection;
 }
 
-export type InterpretedResultsSortColumn = "alert-message";
+type InterpretedResultsSortColumn = "alert-message";
 
 export interface InterpretedResultsSortState {
   sortBy: InterpretedResultsSortColumn;
@@ -318,7 +316,7 @@ export type FromCompareViewMessage =
 /**
  * Message from the compare view to request opening a query.
  */
-export interface OpenQueryMessage {
+interface OpenQueryMessage {
   readonly t: "openQuery";
   readonly kind: "from" | "to";
 }
@@ -406,12 +404,12 @@ export interface ParsedResultSets {
   resultSet: ResultSet;
 }
 
-export interface SetVariantAnalysisMessage {
+interface SetVariantAnalysisMessage {
   t: "setVariantAnalysis";
   variantAnalysis: VariantAnalysis;
 }
 
-export interface SetFilterSortStateMessage {
+interface SetFilterSortStateMessage {
   t: "setFilterSortState";
   filterSortState: RepositoriesFilterSortState;
 }
@@ -420,48 +418,48 @@ export type VariantAnalysisState = {
   variantAnalysisId: number;
 };
 
-export interface SetRepoResultsMessage {
+interface SetRepoResultsMessage {
   t: "setRepoResults";
   repoResults: VariantAnalysisScannedRepositoryResult[];
 }
 
-export interface SetRepoStatesMessage {
+interface SetRepoStatesMessage {
   t: "setRepoStates";
   repoStates: VariantAnalysisScannedRepositoryState[];
 }
 
-export interface RequestRepositoryResultsMessage {
+interface RequestRepositoryResultsMessage {
   t: "requestRepositoryResults";
   repositoryFullName: string;
 }
 
-export interface OpenQueryFileMessage {
+interface OpenQueryFileMessage {
   t: "openQueryFile";
 }
 
-export interface OpenQueryTextMessage {
+interface OpenQueryTextMessage {
   t: "openQueryText";
 }
 
-export interface CopyRepositoryListMessage {
+interface CopyRepositoryListMessage {
   t: "copyRepositoryList";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface ExportResultsMessage {
+interface ExportResultsMessage {
   t: "exportResults";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface OpenLogsMessage {
+interface OpenLogsMessage {
   t: "openLogs";
 }
 
-export interface CancelVariantAnalysisMessage {
+interface CancelVariantAnalysisMessage {
   t: "cancelVariantAnalysis";
 }
 
-export interface ShowDataFlowPathsMessage {
+interface ShowDataFlowPathsMessage {
   t: "showDataFlowPaths";
   dataFlowPaths: DataFlowPaths;
 }
@@ -483,7 +481,7 @@ export type FromVariantAnalysisMessage =
   | CancelVariantAnalysisMessage
   | ShowDataFlowPathsMessage;
 
-export interface SetDataFlowPathsMessage {
+interface SetDataFlowPathsMessage {
   t: "setDataFlowPaths";
   dataFlowPaths: DataFlowPaths;
 }
@@ -492,80 +490,137 @@ export type ToDataFlowPathsMessage = SetDataFlowPathsMessage;
 
 export type FromDataFlowPathsMessage = CommonFromViewMessages;
 
-export interface SetExtensionPackStateMessage {
-  t: "setDataExtensionEditorViewState";
-  viewState: DataExtensionEditorViewState;
+interface SetExtensionPackStateMessage {
+  t: "setModelEditorViewState";
+  viewState: ModelEditorViewState;
 }
 
-export interface SetExternalApiUsagesMessage {
-  t: "setExternalApiUsages";
-  externalApiUsages: ExternalApiUsage[];
+interface SetMethodsMessage {
+  t: "setMethods";
+  methods: Method[];
 }
 
-export interface ShowProgressMessage {
-  t: "showProgress";
-  step: number;
-  maxStep: number;
-  message: string;
+interface SetModeledMethodsMessage {
+  t: "setModeledMethods";
+  methods: Record<string, ModeledMethod>;
 }
 
-export interface LoadModeledMethodsMessage {
-  t: "loadModeledMethods";
-  modeledMethods: Record<string, ModeledMethod>;
+interface SetModifiedMethodsMessage {
+  t: "setModifiedMethods";
+  methodSignatures: string[];
 }
 
-export interface AddModeledMethodsMessage {
-  t: "addModeledMethods";
-  modeledMethods: Record<string, ModeledMethod>;
+interface SetInProgressMethodsMessage {
+  t: "setInProgressMethods";
+  packageName: string;
+  inProgressMethods: string[];
 }
 
-export interface SwitchModeMessage {
+interface SwitchModeMessage {
   t: "switchMode";
   mode: Mode;
 }
 
-export interface JumpToUsageMessage {
+interface JumpToUsageMessage {
   t: "jumpToUsage";
-  location: ResolvableLocationValue;
+  method: Method;
+  usage: Usage;
 }
 
-export interface OpenExtensionPackMessage {
+interface OpenDatabaseMessage {
+  t: "openDatabase";
+}
+
+interface OpenExtensionPackMessage {
   t: "openExtensionPack";
 }
 
-export interface RefreshExternalApiUsages {
-  t: "refreshExternalApiUsages";
+interface RefreshMethods {
+  t: "refreshMethods";
 }
 
-export interface SaveModeledMethods {
+interface SaveModeledMethods {
   t: "saveModeledMethods";
-  externalApiUsages: ExternalApiUsage[];
+  methods: Method[];
   modeledMethods: Record<string, ModeledMethod>;
 }
 
-export interface GenerateExternalApiMessage {
-  t: "generateExternalApi";
+interface GenerateMethodMessage {
+  t: "generateMethod";
 }
 
-export interface GenerateExternalApiFromLlmMessage {
-  t: "generateExternalApiFromLlm";
-  externalApiUsages: ExternalApiUsage[];
+interface GenerateMethodsFromLlmMessage {
+  t: "generateMethodsFromLlm";
+  packageName: string;
+  methods: Method[];
   modeledMethods: Record<string, ModeledMethod>;
 }
 
-export type ToDataExtensionsEditorMessage =
+interface StopGeneratingMethodsFromLlmMessage {
+  t: "stopGeneratingMethodsFromLlm";
+  packageName: string;
+}
+
+interface ModelDependencyMessage {
+  t: "modelDependency";
+}
+
+interface HideModeledMethodsMessage {
+  t: "hideModeledMethods";
+  hideModeledMethods: boolean;
+}
+
+interface SetModeledMethodMessage {
+  t: "setModeledMethod";
+  method: ModeledMethod;
+}
+
+export type ToModelEditorMessage =
   | SetExtensionPackStateMessage
-  | SetExternalApiUsagesMessage
-  | ShowProgressMessage
-  | LoadModeledMethodsMessage
-  | AddModeledMethodsMessage;
+  | SetMethodsMessage
+  | SetModeledMethodsMessage
+  | SetModifiedMethodsMessage
+  | SetInProgressMethodsMessage;
 
-export type FromDataExtensionsEditorMessage =
+export type FromModelEditorMessage =
   | ViewLoadedMsg
   | SwitchModeMessage
-  | RefreshExternalApiUsages
+  | RefreshMethods
+  | OpenDatabaseMessage
   | OpenExtensionPackMessage
   | JumpToUsageMessage
   | SaveModeledMethods
-  | GenerateExternalApiMessage
-  | GenerateExternalApiFromLlmMessage;
+  | GenerateMethodMessage
+  | GenerateMethodsFromLlmMessage
+  | StopGeneratingMethodsFromLlmMessage
+  | ModelDependencyMessage
+  | HideModeledMethodsMessage
+  | SetModeledMethodMessage;
+
+export type FromMethodModelingMessage =
+  | TelemetryMessage
+  | UnhandledErrorMessage
+  | SetModeledMethodMessage;
+
+interface SetMethodMessage {
+  t: "setMethod";
+  method: Method;
+}
+
+interface SetMethodModifiedMessage {
+  t: "setMethodModified";
+  isModified: boolean;
+}
+
+interface SetSelectedMethodMessage {
+  t: "setSelectedMethod";
+  method: Method;
+  modeledMethod: ModeledMethod;
+  isModified: boolean;
+}
+
+export type ToMethodModelingMessage =
+  | SetMethodMessage
+  | SetModeledMethodMessage
+  | SetMethodModifiedMessage
+  | SetSelectedMethodMessage;

extensions/ql-vscode/src/common/location-link-utils.ts

@@ -4,8 +4,20 @@ export function createRemoteFileRef(
   fileLink: FileLink,
   startLine?: number,
   endLine?: number,
+  startColumn?: number,
+  endColumn?: number,
 ): string {
-  if (startLine && endLine && startLine !== endLine) {
+  if (
+    startColumn &&
+    endColumn &&
+    startLine &&
+    endLine &&
+    // Verify that location information is valid; otherwise highlighting might be broken
+    ((startLine === endLine && startColumn < endColumn) || startLine < endLine)
+  ) {
+    // This relies on column highlighting of new code view on GitHub
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}C${startColumn}-L${endLine}C${endColumn}`;
+  } else if (startLine && endLine && startLine < endLine) {
     return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}-L${endLine}`;
   } else if (startLine) {
     return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}`;

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -2,7 +2,7 @@ import { NotificationLogger } from "./notification-logger";
 import { AppTelemetry } from "../telemetry";
 import { RedactableError } from "../errors";
 
-export interface ShowAndLogOptions {
+interface ShowAndLogOptions {
   /**
    * An alternate message that is added to the log, but not displayed in the popup.
    * This is useful for adding extra detail to the logs that would be too noisy for the popup.

extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts

@@ -1,5 +1,8 @@
-import { Repository } from "../repository";
-import { VariantAnalysis, VariantAnalysisRepoTask } from "../variant-analysis";
+import { Repository } from "../../variant-analysis/gh-api/repository";
+import {
+  VariantAnalysis,
+  VariantAnalysisRepoTask,
+} from "../../variant-analysis/gh-api/variant-analysis";
 
 // Types that represent requests/responses from the GitHub API
 // that we need to mock.
@@ -11,13 +14,14 @@ export enum RequestKind {
   GetVariantAnalysisRepo = "getVariantAnalysisRepo",
   GetVariantAnalysisRepoResult = "getVariantAnalysisRepoResult",
   CodeSearch = "codeSearch",
+  AutoModel = "autoModel",
 }
 
-export interface BasicErorResponse {
+interface BasicErorResponse {
   message: string;
 }
 
-export interface GetRepoRequest {
+interface GetRepoRequest {
   request: {
     kind: RequestKind.GetRepo;
   };
@@ -27,7 +31,7 @@ export interface GetRepoRequest {
   };
 }
 
-export interface SubmitVariantAnalysisRequest {
+interface SubmitVariantAnalysisRequest {
   request: {
     kind: RequestKind.SubmitVariantAnalysis;
   };
@@ -37,7 +41,7 @@ export interface SubmitVariantAnalysisRequest {
   };
 }
 
-export interface GetVariantAnalysisRequest {
+interface GetVariantAnalysisRequest {
   request: {
     kind: RequestKind.GetVariantAnalysis;
   };
@@ -47,7 +51,7 @@ export interface GetVariantAnalysisRequest {
   };
 }
 
-export interface GetVariantAnalysisRepoRequest {
+interface GetVariantAnalysisRepoRequest {
   request: {
     kind: RequestKind.GetVariantAnalysisRepo;
     repositoryId: number;
@@ -70,7 +74,7 @@ export interface GetVariantAnalysisRepoResultRequest {
   };
 }
 
-export interface CodeSearchRequest {
+interface CodeSearchRequest {
   request: {
     kind: RequestKind.CodeSearch;
     query: string;
@@ -87,13 +91,30 @@ export interface CodeSearchRequest {
   };
 }
 
+interface AutoModelRequest {
+  request: {
+    kind: RequestKind.AutoModel;
+    body?: {
+      candidates: string;
+    };
+  };
+  response: {
+    status: number;
+    body?: {
+      models: string;
+    };
+    message?: string;
+  };
+}
+
 export type GitHubApiRequest =
   | GetRepoRequest
   | SubmitVariantAnalysisRequest
   | GetVariantAnalysisRequest
   | GetVariantAnalysisRepoRequest
   | GetVariantAnalysisRepoResultRequest
-  | CodeSearchRequest;
+  | CodeSearchRequest
+  | AutoModelRequest;
 
 export const isGetRepoRequest = (
   request: GitHubApiRequest,
@@ -123,3 +144,8 @@ export const isCodeSearchRequest = (
   request: GitHubApiRequest,
 ): request is CodeSearchRequest =>
   request.request.kind === RequestKind.CodeSearch;
+
+export const isAutoModelRequest = (
+  request: GitHubApiRequest,
+): request is AutoModelRequest =>
+  request.request.kind === RequestKind.AutoModel;

extensions/ql-vscode/src/common/mock-gh-api/mock-gh-api-server.ts

@@ -2,11 +2,11 @@ import { join, resolve } from "path";
 import { pathExists } from "fs-extra";
 import { setupServer, SetupServer } from "msw/node";
 
-import { DisposableObject } from "../../../common/disposable-object";
+import { DisposableObject } from "../disposable-object";
 
 import { Recorder } from "./recorder";
 import { createRequestHandlers } from "./request-handlers";
-import { getDirectoryNamesInsidePath } from "../../../common/files";
+import { getDirectoryNamesInsidePath } from "../files";
 
 /**
  * Enables mocking of the GitHub API server via HTTP interception, using msw.
@@ -129,11 +129,11 @@ export class MockGitHubApiServer extends DisposableObject {
 
   public async getDefaultScenariosPath(): Promise<string | undefined> {
     // This should be the directory where package.json is located
-    const rootDirectory = resolve(__dirname, "../../../..");
+    const rootDirectory = resolve(__dirname, "../../..");
 
     const scenariosPath = resolve(
       rootDirectory,
-      "src/variant-analysis/gh-api/mocks/scenarios",
+      "src/common/mock-gh-api/scenarios",
     );
     if (await pathExists(scenariosPath)) {
       return scenariosPath;

extensions/ql-vscode/src/common/mock-gh-api/recorder.ts

@@ -8,7 +8,7 @@ import { IsomorphicResponse } from "@mswjs/interceptors";
 import { Headers } from "headers-polyfill";
 import fetch from "node-fetch";
 
-import { DisposableObject } from "../../../common/disposable-object";
+import { DisposableObject } from "../disposable-object";
 
 import {
   GetVariantAnalysisRepoResultRequest,
@@ -259,6 +259,21 @@ async function createGitHubApiRequest(
     };
   }
 
+  const autoModelMatch = url.match(
+    /\/repos\/github\/codeql\/code-scanning\/codeql\/auto-model/,
+  );
+  if (autoModelMatch) {
+    return {
+      request: {
+        kind: RequestKind.AutoModel,
+      },
+      response: {
+        status,
+        body: JSON.parse(body),
+      },
+    };
+  }
+
   return undefined;
 }
 

extensions/ql-vscode/src/common/mock-gh-api/request-handlers.ts

@@ -3,6 +3,7 @@ import { readdir, readJson, readFile } from "fs-extra";
 import { DefaultBodyType, MockedRequest, rest, RestHandler } from "msw";
 import {
   GitHubApiRequest,
+  isAutoModelRequest,
   isCodeSearchRequest,
   isGetRepoRequest,
   isGetVariantAnalysisRepoRequest,
@@ -13,7 +14,7 @@ import {
 
 const baseUrl = "https://api.github.com";
 
-export type RequestHandler = RestHandler<MockedRequest<DefaultBodyType>>;
+type RequestHandler = RestHandler<MockedRequest<DefaultBodyType>>;
 
 export async function createRequestHandlers(
   scenarioDirPath: string,
@@ -27,6 +28,7 @@ export async function createRequestHandlers(
     createGetVariantAnalysisRepoRequestHandler(requests),
     createGetVariantAnalysisRepoResultRequestHandler(requests),
     createCodeSearchRequestHandler(requests),
+    createAutoModelRequestHandler(requests),
   ];
 
   return handlers;
@@ -219,3 +221,30 @@ function createCodeSearchRequestHandler(
     );
   });
 }
+
+function createAutoModelRequestHandler(
+  requests: GitHubApiRequest[],
+): RequestHandler {
+  const autoModelRequests = requests.filter(isAutoModelRequest);
+  let requestIndex = 0;
+
+  // During automodeling there can be multiple API requests for each batch
+  // of candidates we want to model. We need to return different responses for each request,
+  // so keep an index of the request and return the appropriate response.
+  return rest.post(
+    `${baseUrl}/repos/github/codeql/code-scanning/codeql/auto-model`,
+    (_req, res, ctx) => {
+      const request = autoModelRequests[requestIndex];
+
+      if (requestIndex < autoModelRequests.length - 1) {
+        // If there are more requests to come, increment the index.
+        requestIndex++;
+      }
+
+      return res(
+        ctx.status(request.response.status),
+        ctx.json(request.response.body),
+      );
+    },
+  );
+}

extensions/ql-vscode/src/common/mock-gh-api/scenarios/auto-model-success/0-autoModel.json

@@ -0,0 +1,11 @@
+{
+  "request": {
+    "kind": "autoModel"
+  },
+  "response": {
+    "status": 200,
+    "body": {
+      "models": "extensions:\n- addsTo: {extensible: sinkModel, pack: codeql/java-all}\n  data:\n  - [javax.servlet.http, HttpServletResponse, true, sendRedirect, (String), '', 'Argument[this]',\n    request-forgery, ai-generated]\n  - [javax.servlet.http, HttpServletResponse, true, sendRedirect, (String), '', 'Argument[0]',\n    request-forgery, ai-generated]\n"
+    }
+  }
+}

extensions/ql-vscode/src/common/mock-gh-api/scenarios/auto-model-success/1-autoModel.json

@@ -0,0 +1,11 @@
+{
+  "request": {
+    "kind": "autoModel"
+  },
+  "response": {
+    "status": 200,
+    "body": {
+      "models": "extensions:\n- addsTo: {extensible: sinkModel, pack: codeql/java-all}\n  data:\n  - [javax.servlet, MultipartConfigElement, true, MultipartConfigElement, (String),\n    '', 'Argument[0]', request-forgery, ai-generated]\n"
+    }
+  }
+}

extensions/ql-vscode/src/common/mock-gh-api/scenarios/auto-model-success/README.md

@@ -0,0 +1 @@
+This scenario is best when modeling the `javax.servlet-api` package.

extensions/ql-vscode/src/common/mock-gh-api/scenarios/mrva-problem-query-success/19-getVariantAnalysisRepoResult.json

@@ -5,7 +5,7 @@
   },
   "response": {
     "status": 200,
-    "body": "file:20-getVariantAnalysisRepoResult.body.zip",
+    "body": "file:19-getVariantAnalysisRepoResult.body.zip",
     "contentType": "application/zip"
   }
 }