Merge pull request #2891 from github/koesie10/dispose-webview

Dispose tracked objects when panel is disposed

.github/codeql/codeql-config.yml

@@ -2,6 +2,8 @@ name: "CodeQL config"
 queries:
   - name: Run standard queries
     uses: security-and-quality
+  - name: Experimental queries
+    uses: security-experimental
   - name: Run custom javascript queries
     uses: ./.github/codeql/queries
 paths:

.github/codeql/queries/assert-no-vscode-dependency.ql

@@ -0,0 +1,37 @@
+/**
+ * @name Unwanted dependency on vscode API
+ * @kind path-problem
+ * @problem.severity error
+ * @id vscode-codeql/assert-no-vscode-dependency
+ * @description The modules stored under `common` should not have dependencies on the VS Code API
+ */
+
+import javascript
+
+class VSCodeImport extends ImportDeclaration {
+  VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
+}
+
+class CommonFile extends File {
+  CommonFile() {
+    this.getRelativePath().regexpMatch(".*/src/common/.*") and
+    not this.getRelativePath().regexpMatch(".*/vscode/.*")
+  }
+}
+
+Import getANonTypeOnlyImport(Module m) {
+  result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly()
+}
+
+query predicate edges(AstNode a, AstNode b) {
+  getANonTypeOnlyImport(a) = b or
+  a.(Import).getImportedModule() = b
+}
+
+from Module m, VSCodeImport v
+where
+  m.getFile() instanceof CommonFile and
+  edges+(m, v)
+select m, m, v,
+  "This module is in the 'common' directory but has a transitive dependency on the vscode API imported $@",
+  v, "here"

.github/codeql/queries/assert-pure.ql

@@ -1,21 +0,0 @@
-/**
- * @name Unwanted dependency on vscode API
- * @kind problem
- * @problem.severity error
- * @id vscode-codeql/assert-pure
- * @description The modules stored under `pure` and tested in the `pure-tests`
- * are intended to be "pure".
- */
-import javascript
-
-class VSCodeImport extends ASTNode {
-  VSCodeImport() {
-    this.(Import).getImportedPath().getValue() = "vscode"
-  }
-}
-
-from Module m, VSCodeImport v
-where
-  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
-  m.getAnImportedModule*().getAnImport() = v
-select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/qlpack.yml

@@ -1,3 +1,4 @@
 name: vscode-codeql-custom-queries-javascript
 version: 0.0.0
-libraryPathDependencies: codeql-javascript
+dependencies:
+  codeql/javascript-queries: "*"

.github/codeql/queries/unique-command-use.ql

@@ -15,120 +15,145 @@
  * that should be changed to fix the alert.
  */
 
- import javascript
+import javascript
 
- /**
-  * The name of a VS Code command.
-  */
- class CommandName extends string {
-   CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
- 
-   /**
-    * In how many ways is this command used. Will always be at least 1.
-    */
-   int getNumberOfUsages() { result = count(this.getAUse()) }
+/**
+ * The name of a VS Code command.
+ */
+class CommandName extends string {
+  CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
 
-   /**
-    * Get a usage of this command.
-    */
-   CommandUsage getAUse() { result.getCommandName() = this }
- 
-   /**
-    * Get the canonical first usage of this command, to use for the location
-    * of the alert. The implementation of this ordering of usages is arbitrary
-    * and the usage given may not be the one that should be changed when fixing
-    * the alert.
-    */
-   CommandUsage getFirstUsage() {
-     result =
-       max(CommandUsage use |
-         use = this.getAUse()
-       |
-         use
-         order by
-           use.getFile().getRelativePath(), use.getLocation().getStartLine(),
-           use.getLocation().getStartColumn()
-       )
-   }
- }
- 
- /**
-  * Represents a single usage of a command, either from within code or
-  * from the command's definition in package.json
-  */
- abstract class CommandUsage extends Locatable {
-   abstract string getCommandName();
- }
- 
- /**
-  * A usage of a command from the typescript code, by calling `executeCommand`.
-  */
- class CommandUsageCallExpr extends CommandUsage, CallExpr {
-   CommandUsageCallExpr() {
-     this.getCalleeName() = "executeCommand" and
-     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
-     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
-   }
- 
-   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
- }
- 
- /**
-  * A usage of a command from any menu that isn't the command palette.
-  * This means a user could invoke the command by clicking on a button in
-  * something like a menu or a dropdown.
-  */
- class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonMenuItem() {
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson, string menuName |
-       packageJson
-           .getPropValue("contributes")
-           .getPropValue("menus")
-           .getPropValue(menuName)
-           .getElementValue(_) = this and
-       menuName != "commandPalette"
-     )
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- /**
-  * Is the given command disabled for use in the command palette by
-  * a block with a `"when": "false"` field.
-  */
- predicate isDisabledInCommandPalette(string commandName) {
-   exists(PackageJson packageJson, JsonObject commandPaletteObject |
-     packageJson
-         .getPropValue("contributes")
-         .getPropValue("menus")
-         .getPropValue("commandPalette")
-         .getElementValue(_) = commandPaletteObject and
-     commandPaletteObject.getPropValue("command").getStringValue() = commandName and
-     commandPaletteObject.getPropValue("when").getStringValue() = "false"
-   )
- }
- 
- /**
-  * Represents a command being usable from the command palette.
-  * This means that a user could choose to manually invoke the command.
-  */
- class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
-   CommandUsagePackageJsonCommandPalette() {
-     this.getFile().getBaseName() = "package.json" and
-     exists(this.getPropValue("command")) and
-     exists(PackageJson packageJson |
-       packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
-     ) and
-     not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
-   }
- 
-   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
- }
- 
- from CommandName c
- where c.getNumberOfUsages() > 1
- select c.getFirstUsage(),
-   "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"
- 
+  /**
+   * In how many ways is this command used. Will always be at least 1.
+   */
+  int getNumberOfUsages() { result = count(this.getAUse()) }
+
+  /**
+   * Get a usage of this command.
+   */
+  CommandUsage getAUse() { result.getCommandName() = this }
+
+  /**
+   * Get the canonical first usage of this command, to use for the location
+   * of the alert. The implementation of this ordering of usages is arbitrary
+   * and the usage given may not be the one that should be changed when fixing
+   * the alert.
+   */
+  CommandUsage getFirstUsage() {
+    result =
+      max(CommandUsage use |
+        use = this.getAUse()
+      |
+        use
+        order by
+          use.getFile().getRelativePath(), use.getLocation().getStartLine(),
+          use.getLocation().getStartColumn()
+      )
+  }
+}
+
+/**
+ * Matches one of the members of `BuiltInVsCodeCommands` from `extensions/ql-vscode/src/common/commands.ts`.
+ */
+class BuiltInVSCodeCommand extends string {
+  BuiltInVSCodeCommand() {
+    exists(TypeAliasDeclaration tad |
+      tad.getIdentifier().getName() = "BuiltInVsCodeCommands" and
+      tad.getDefinition().(InterfaceTypeExpr).getAMember().getName() = this
+    )
+  }
+}
+
+/**
+ * Represents a single usage of a command, either from within code or
+ * from the command's definition in package.json
+ */
+abstract class CommandUsage extends Locatable {
+  abstract string getCommandName();
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `executeCommand`.
+ */
+class CommandUsageCallExpr extends CommandUsage, CallExpr {
+  CommandUsageCallExpr() {
+    this.getCalleeName() = "executeCommand" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from the typescript code, by calling `CommandManager.execute`.
+ */
+class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
+  CommandUsageCommandManagerMethodCallExpr() {
+    this.getCalleeName() = "execute" and
+    this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
+    this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+    not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+  }
+
+  override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+}
+
+/**
+ * A usage of a command from any menu that isn't the command palette.
+ * This means a user could invoke the command by clicking on a button in
+ * something like a menu or a dropdown.
+ */
+class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonMenuItem() {
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson, string menuName |
+      packageJson
+          .getPropValue("contributes")
+          .getPropValue("menus")
+          .getPropValue(menuName)
+          .getElementValue(_) = this and
+      menuName != "commandPalette"
+    )
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+/**
+ * Is the given command disabled for use in the command palette by
+ * a block with a `"when": "false"` field.
+ */
+predicate isDisabledInCommandPalette(string commandName) {
+  exists(PackageJson packageJson, JsonObject commandPaletteObject |
+    packageJson
+        .getPropValue("contributes")
+        .getPropValue("menus")
+        .getPropValue("commandPalette")
+        .getElementValue(_) = commandPaletteObject and
+    commandPaletteObject.getPropValue("command").getStringValue() = commandName and
+    commandPaletteObject.getPropValue("when").getStringValue() = "false"
+  )
+}
+
+/**
+ * Represents a command being usable from the command palette.
+ * This means that a user could choose to manually invoke the command.
+ */
+class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
+  CommandUsagePackageJsonCommandPalette() {
+    this.getFile().getBaseName() = "package.json" and
+    exists(this.getPropValue("command")) and
+    exists(PackageJson packageJson |
+      packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
+    ) and
+    not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
+  }
+
+  override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+}
+
+from CommandName c
+where c.getNumberOfUsages() > 1 and not c instanceof BuiltInVSCodeCommand
+select c.getFirstUsage(),
+  "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"

.github/workflows/bump-cli.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Bump CLI

.github/workflows/cli-test.yml

@@ -0,0 +1,111 @@
+name: Run CLI tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - .github/workflows/cli-test.yml
+      - extensions/ql-vscode/src/codeql-cli/**
+      - extensions/ql-vscode/src/language-support/**
+      - extensions/ql-vscode/src/query-server/**
+      - extensions/ql-vscode/supported_cli_versions.json
+
+jobs:
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
+
+  set-matrix:
+    name: Set Matrix for cli-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set the variables
+        id: set-variables
+        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+    outputs:
+      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
+      fail-fast: false
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.17.1'
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm ci
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Decide on ref of CodeQL repo
+        id: choose-ref
+        shell: bash
+        run: |
+          if [[ "${{ matrix.version }}" == "nightly" ]]
+          then
+            REF="codeql-cli/latest"
+          else
+            REF="codeql-cli/${{ matrix.version }}"
+          fi
+          echo "ref=$REF" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout QL
+        uses: actions/checkout@v4
+        with:
+          repository: github/codeql
+          ref: ${{ steps.choose-ref.outputs.ref }}
+          path: codeql
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          unset DBUS_SESSION_BUS_ADDRESS
+          /usr/bin/xvfb-run npm run test:cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run test:cli-integration

.github/workflows/codeql.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@main

.github/workflows/dependency-review.yml

@@ -11,6 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@v3

.github/workflows/main.yml

@@ -16,13 +16,13 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -53,34 +53,18 @@ jobs:
           name: vscode-codeql-extension
           path: artifacts
 
-  find-nightly:
-    name: Find Nightly Release
-    runs-on: ubuntu-latest
-    outputs:
-      url: ${{ steps.get-url.outputs.nightly-url }}
-    steps:
-      - name: Get Nightly Release URL
-        id: get-url
-        env:
-          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
-        shell: bash
-        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
-        run: |
-          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
-          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"
-
   lint:
     name: Lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -100,11 +84,57 @@ jobs:
         run: |
           npm run lint
 
+      - name: Lint Markdown
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run lint:markdown
+
       - name: Lint scenarios
         working-directory: extensions/ql-vscode
         run: |
           npm run lint:scenarios
 
+      - name: Find deadcode
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run find-deadcode
+
+  generated:
+    name: Check generated code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16.17.1'
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm ci
+        shell: bash
+
+      - name: Check that repo is clean
+        run: |
+          git diff --exit-code
+          git diff --exit-code --cached
+
+      - name: Generate code
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run generate
+
+      - name: Check for changes
+        run: |
+          git diff --exit-code
+          git diff --exit-code --cached
+
   unit-test:
     name: Unit Test
     runs-on: ${{ matrix.os }}
@@ -113,13 +143,13 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -147,13 +177,13 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -188,38 +218,40 @@ jobs:
         run: |
           npm run test:vscode-integration
 
-  set-matrix:
-    name: Set Matrix for cli-test
+  get-latest-cli-version:
+    name: Get latest CLI version
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Set the variables
-        id: set-variables
-        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
+        uses: actions/checkout@v4
+      - name: Set the variable
+        id: set-variable
+        run: |
+          echo "cli-version=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc '.[0]')" >> $GITHUB_OUTPUT
+          echo "$cli-version"
     outputs:
-      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+      cli-version: ${{ steps.set-variable.outputs.cli-version }}
+
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}
-    needs: [find-nightly, set-matrix]
+    needs: [get-latest-cli-version]
+    timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
       fail-fast: false
     env:
-      CLI_VERSION: ${{ matrix.version }}
-      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
+      CLI_VERSION: ${{ needs.get-latest-cli-version.outputs.cli-version }}
       TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -235,23 +267,11 @@ jobs:
           npm run build
         shell: bash
 
-      - name: Decide on ref of CodeQL repo
-        id: choose-ref
-        shell: bash
-        run: |
-          if [[ "${{ matrix.version }}" == "nightly" ]]
-          then
-            REF="codeql-cli/latest"
-          else
-            REF="codeql-cli/${{ matrix.version }}"
-          fi
-          echo "ref=$REF" >> "$GITHUB_OUTPUT"
-
       - name: Checkout QL
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: github/codeql
-          ref: ${{ steps.choose-ref.outputs.ref }}
+          ref: 'codeql-cli/${{ needs.get-latest-cli-version.outputs.cli-version }}'
           path: codeql
 
       - name: Run CLI tests (Linux)

.github/workflows/release.yml

@@ -18,11 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
 
       - name: Install dependencies
         run: |
@@ -67,49 +67,19 @@ jobs:
 
         # TODO Run tests, or check that a test run on the same branch succeeded.
 
-      - name: Create release
-        id: create-release
-        uses: actions/create-release@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref }}
-          # This gives us a chance to manually review the created release before publishing it,
-          # as well as to test the release workflow by pushing temporary tags.
-          # Once we have set all required release metadata in this step, we can set this to `false`.
-          draft: true
-          prerelease: false
-
-      - name: Upload release asset
-        uses: actions/upload-release-asset@v1.0.1
-        if: success()
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          # Get the `upload_url` from the `create-release` step above.
-          upload_url: ${{ steps.create-release.outputs.upload_url }}
-          # Get the `vsix_path` and `ref_name` from the `prepare-artifacts` step above.
-          asset_path: ${{ steps.prepare-artifacts.outputs.vsix_path }}
-          asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
-          asset_content_type: application/zip
-
       - name: Create sourcemap ZIP file
         run: |
           cd dist/vscode-codeql/out
           zip -r ../../vscode-codeql-sourcemaps.zip *.map
 
-      - name: Upload sourcemap ZIP file
-        uses: actions/upload-release-asset@v1.0.1
-        if: success()
+      - name: Create release
+        id: create-release
+        run: |
+          gh release create ${{ github.ref_name }} --draft --title "Release ${{ github.ref_name }}" \
+            '${{ steps.prepare-artifacts.outputs.vsix_path }}#${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}' \
+            'dist/vscode-codeql-sourcemaps.zip#${{ format('vscode-codeql-sourcemaps-{0}.zip', steps.prepare-artifacts.outputs.ref_name) }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          # Get the `upload_url` from the `create-release` step above.
-          upload_url: ${{ steps.create-release.outputs.upload_url }}
-          asset_path: dist/vscode-codeql-sourcemaps.zip
-          asset_name: ${{ format('vscode-codeql-sourcemaps-{0}.zip', steps.prepare-artifacts.outputs.ref_name) }}
-          asset_content_type: application/zip
 
       ###
       # Do Post release work: version bump and changelog PR
@@ -164,10 +134,7 @@ jobs:
 
       - name: Publish to Registry
         run: |
-          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix || \
-            echo "Failed to publish to VS Code Marketplace. \
-                  If this was an authentication problem, please make sure the \
-                  auth token hasn't expired."
+          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix
 
   open-vsx-publish:
     name: Publish to Open VSX Registry

.markdownlint.json

@@ -0,0 +1,4 @@
+{
+  "MD013": false,
+  "MD041": false
+}

.vscode/extensions.json

@@ -4,6 +4,7 @@
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
     "amodio.tsl-problem-matcher",
+    "DavidAnson.vscode-markdownlint",
     "dbaeumer.vscode-eslint",
     "esbenp.prettier-vscode",
     "firsttris.vscode-jest-runner",

.vscode/settings.json

@@ -42,22 +42,53 @@
     "LANG": "en-US",
     "TZ": "UTC"
   },
+
+  // These custom rules are read in extensions/ql-vscode/.markdownlint-cli2.cjs
+  // but markdownlint only considers that config when linting files in
+  // extensions/ql-vscode/ or its subfolders. Therefore, we have to explicitly
+  // load the custom rules here too.
+  "markdownlint.customRules": [
+    "./extensions/ql-vscode/node_modules/@github/markdownlint-github/src/rules/no-default-alt-text.js",
+    "./extensions/ql-vscode/node_modules/@github/markdownlint-github/src/rules/no-generic-link-text.js"
+  ],
+
+  // This ensures that the accessibility rule enablement done by github-markdownlint is
+  // considered by the extension too.
+  //
+  // Unfortunately, we can only specify a single extends, so the config here isn't
+  // identical since it can't also consider @github/markdownlint-github/style/base.json
+  // That's not as bad as it could be since the full config is considered for anything
+  // in extensions/ql-vscode/ or its subfolders anyway.
+  //
+  // Additional nonfiguration of the default rules is done in .markdownlint.json,
+  // which is picked up by the extension automatically, and read explicitly in
+  // .markdownlint-cli2.cjs
+  "markdownlint.config": {
+    "extends": "./extensions/ql-vscode/node_modules/@github/markdownlint-github/style/accessibility.json"
+  },
+
+  // These options are used by the `jestrunner.debug` command.
+  // They are not used by the `jestrunner.run` command.
+  // After clicking "debug" over a test, continually invoke the
+  // "Debug: Attach to Node Process" command until you see a
+  // process named "Code Helper (Plugin)". Then click "attach".
+  // This will attach the debugger to the test process.
   "jestrunner.debugOptions": {
     // Uncomment to debug integration tests
-    // "attachSimplePort": 9223,
+    "attachSimplePort": 9223,
     "env": {
       "LANG": "en-US",
       "TZ": "UTC",
 
       // Uncomment to set a custom path to a CodeQL checkout.
-      // "TEST_CODEQL_PATH": "../codeql",
+      // "TEST_CODEQL_PATH": "/absolute/path/to/checkout/of/codeql",
 
       // Uncomment to set a custom path to a CodeQL CLI executable.
       // This is the CodeQL version that will be used in the tests.
-      // "CLI_PATH": "/path/to/customg/codeql",
+      // "CLI_PATH": "/absolute/path/to/custom/codeql",
 
       // Uncomment to debug integration tests
-      // "VSCODE_WAIT_FOR_DEBUGGER": "true",
+      "VSCODE_WAIT_FOR_DEBUGGER": "true",
     }
   },
   "terminal.integrated.env.linux": {

CODEOWNERS

@@ -1,3 +1,5 @@
 **/* @github/codeql-vscode-reviewers
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
+**/model-editor/ @github/code-scanning-secexp-reviewers
+**/queries-panel/ @github/code-scanning-secexp-reviewers

CODE_OF_CONDUCT.md

@@ -67,10 +67,7 @@ members of the project's leadership.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
+This Code of Conduct is adapted from the [Contributor Covenant, version 1.4](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html).
 
 For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+[the Contributor Covenant FAQ](https://www.contributor-covenant.org/faq). For more about Contributor Covenant, see [the Contributor Covenant website](https://www.contributor-covenant.org).

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 [fork]: https://github.com/github/vscode-codeql/fork
 [pr]: https://github.com/github/vscode-codeql/compare
-[style]: https://primer.style
+[style]: https://github.com/microsoft/vscode-webview-ui-toolkit
 [code-of-conduct]: CODE_OF_CONDUCT.md
 
 Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great.
@@ -23,7 +23,9 @@ Please note that this project is released with a [Contributor Code of Conduct][c
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 
 * Follow the [style guide][style].
-* Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
+* Write tests:
+  * [Tests that don't require the VS Code API are located here](extensions/ql-vscode/test).
+  * [Integration tests that do require the VS Code API are located here](extensions/ql-vscode/src/vscode-tests).
 * Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
 * Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
@@ -93,7 +95,7 @@ More information about Storybook can be found inside the **Overview** page once
 
 ### Testing
 
-Information about testing can be found [here](./docs/testing.md).
+[Information about testing can be found here](./docs/testing.md).
 
 ## Resources
 

LICENSE.md

@@ -17,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -25,8 +25,8 @@ This project will track new feature development in CodeQL and, whenever appropri
 
 This extension depends on the following two extensions for required functionality. They will be installed automatically when you install VS Code CodeQL.
 
-- [Test Adapter Converter](https://marketplace.visualstudio.com/items?itemName=ms-vscode.test-adapter-converter)
-- [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer)
+* [Test Adapter Converter](https://marketplace.visualstudio.com/items?itemName=ms-vscode.test-adapter-converter)
+* [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer)
 
 ## Contributing
 

docs/node-version.md

@@ -0,0 +1,29 @@
+# Node version
+
+The CodeQL for VS Code extension defines the version of Node.js that it is intended to run with. This Node.js version is used when running most CI and unit tests.
+
+When running in production (i.e. as an extension for a VS Code application) it will use the Node.js version provided by VS Code. This can mean a different Node.js version is used by different users with different versions of VS Code.
+We should make sure the CodeQL for VS Code extension works with the Node.js version supplied by all versions of VS Code that we support.
+
+## Checking the version of Node.js supplied by VS Code
+
+You can find this info by seleting "About Visual Studio Code" from the top menu.
+
+![about-vscode](images/about-vscode.png)
+
+## Updating the Node.js version
+
+The following files will need to be updated:
+
+- `.github/workflows/cli-test.yml` - the "node-version: '[VERSION]'" setting
+- `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
+- `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting
+- `extensions/ql-vscode/.nvmrc` - this will enable nvm to automatically switch to the correct node version when you're in the project folder
+- `extensions/ql-vscode/package-lock.json` - the "engines.node: '[VERSION]'" setting
+- `extensions/ql-vscode/package.json` - the "engines.node: '[VERSION]'" setting
+
+## Node.js version used in tests
+
+Unit tests will use whatever version of Node.js is installed locally. In CI this will be the version specified in the workflow.
+
+Integration tests download a copy of VS Code and then will use whatever version of Node.js is provided by VS Code. Our integration tests are currently pinned to an older version of VS Code. See [VS Code version used in tests](./vscode-version.md#vs-code-version-used-in-tests) for more information.

docs/releasing.md

@@ -1,5 +1,6 @@
 # Releasing (write access required)
 
+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
@@ -10,10 +11,7 @@
       * New telemetry events are added.
       * Deprecation or removal of commands.
       * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
-    * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
-    * `.github/workflows/main.yml` - all the "node-version: <version>" settings
-    * `.github/workflows/release.yml` - the "node-version: <version>" setting
+1. Double-check that the node version we're using matches the one used for VS Code. See the [Node.js version instructions](./node-version.md) for more information.
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
 1. Create a PR for this release:
     * This PR will contain any missing bits from steps 1, 2 and 3. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
@@ -22,27 +20,30 @@
     * Create a PR for this branch.
     * Wait for the PR to be merged into `main`
 1. Switch to `main` branch and pull latest changes
-1. Lock the `main` branch. 
-  * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-  * Select "Lock branch"
-  * Click "Save changes"
+1. Lock the `main` branch.
+    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
+    * Select "Lock branch"
+    * Click "Save changes"
 1. Ensure that no PRs have been merged since the release PR that you merged. If there were, you might need to unlock `main` temporarily and update the CHANGELOG again.
 1. Build the extension `npm run build` and install it on your VS Code using "Install from VSIX".
 1. Go through [our test plan](./test-plan.md) to ensure that the extension is working as expected.
 1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+
     ```bash
     git checkout main
     git tag v1.3.6
     ```
 
    If you've accidentally created a badly named tag, you can delete it via
-    ```bash
-    git tag -d badly-named-tag
-    ```
+
+   ```bash
+   git tag -d badly-named-tag
+   ```
+
 1. Unlock the main branch
-  * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-  * Deselect "Lock branch"
-  * Click "Save changes"
+    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
+    * Deselect "Lock branch"
+    * Click "Save changes"
 1. Push the new tag up:
 
    a. If you're using a fork of the repo:
@@ -68,7 +69,7 @@
    or look at the source if there's any doubt the right code is being shipped.
 1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
 1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
-    - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
+    * If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
 1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
 1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
 1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
@@ -86,4 +87,4 @@ To regenerate the Open VSX token:
 1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
 1. Update the secret in the `publish-open-vsx` environment in the project settings.
 
-To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.
+To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.

docs/test-plan.md

@@ -11,16 +11,67 @@ We don't need to test features (and permutations of features) that are covered b
 
 ## Before releasing the VS Code extension
 
-- Go through the required test cases listed below
+- Run at least one local query and MRVA using the existing version of the extension (to generate "old" query history items).
+- Go through the required test cases listed below.
 - Check major PRs since the previous release for specific one-off things to test. Based on that, you might want to
 choose to go through some of the Optional Test Cases.
-- Run a query using the existing version of the extension (to generate an "old" query history item)
 
 ## Required Test Cases
 
-### Test Case 1: MRVA - Running a problem path query and viewing results
+### Local databases
 
-1. Open the [UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+#### Test case 1: Download a database from GitHub
+
+1. Click "Download Database from GitHub" and enter `angular-cn/ng-nice` and select the javascript language if prompted
+
+#### Test case 2: Import a database from an archive
+
+1. Obtain a javascript database for `babel/babel`
+   - You can do `gh api "/repos/babel/babel/code-scanning/codeql/databases/javascript" -H "Accept: application/zip" > babel.zip` to fetch a database from GitHub.
+2. Click "Choose Database from Archive" and select the file you just downloaded above.
+
+### Local queries
+
+#### Test case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+2. Select the `angular-cn/ng-nice` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that the paths can be opened and paths are rendered correctly
+   - Check that alert locations can be clicked on
+
+#### Test case 2: Running a problem query and viewing results
+
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
+2. Select the `babel/babel` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that alert locations can be clicked on
+
+#### Test case 3: Running a non-problem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+2. Select the `google/brotli` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the results table is rendered
+   - Check that result locations can be clicked on
+
+#### Test case 4: Can use AST viewer
+
+1. Click on any code location from a previous query to open a source file from a database
+2. Open the AST viewing panel and click "View AST"
+3. Once the AST is computed:
+   - Check that it can be navigated
+
+### MRVA
+
+#### Test Case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
 2. Run a MRVA against the following repo list:
 
    ```json
@@ -47,49 +98,91 @@ choose to go through some of the Optional Test Cases.
 6. Once the query completes:
    - Check that the query history item is updated to show the query status as "complete"
 
-### Test Case 2: MRVA - Running a problem query and viewing results
+#### Test Case 2: Running a problem query and viewing results
 
-1. Open the [ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
 2. Run a MRVA against the "Top 10" repositories.
-3. Check that there is a notification message.
+3. Check that a notification message pops up and the results view is opened.
 4. Check the query history. It should:
    - Show that an item has been added to the query history
    - The item should be marked as "in progress".
-5. Once the query starts:
-   - Check that a notification is shown with a link to the results view
+5. Once the query completes:
    - Check that the results are rendered with an alert message and a highlighted code snippet:
-![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-### Test Case 3: MRVA - Running a non-problem query and viewing results
+   ![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-1. Open the [FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+#### Test Case 3: Running a non-problem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
 2. Run a MRVA against a single repository (e.g. `google/brotli`).
-3. Once the query starts:
-   - Open the query results
+3. Check that a notification message pops up and the results view is opened.
+4. Check the query history. It should:
+   - Show that an item has been added to the query history
+   - The item should be marked as "in progress".
+5. Once the query completes:
    - Check that the results show up in a table:
-![results-table](images/results-table.png)
 
-### Test Case 4: MRVA - Interacting with query history
+   ![results-table](images/results-table.png)
+
+#### Test Case 4: Interacting with query history
 
 1. Click a history item (for MRVA):
    - Check that exporting results works
    - Check that sorting results works
    - Check that copying repo lists works
-2. Open the query results directory:
+2. Click "Open Results Directory":
    - Check that the correct directory is opened and there are results in it
-3. View logs
+3. Click "View Logs":
    - Check that the correct workflow is opened
 
-### Test Case 5: MRVA - Canceling a variant analysis run
+#### Test Case 5: Canceling a variant analysis run
 
 Run one of the above MRVAs, but cancel it from within VS Code:
+
 - Check that the query is canceled and the query history item is updated.
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
-### Test Case 6: MRVA - Change to a different colour theme
+### CodeQL Model Editor
 
-Open one of the above MRVAs, try changing to a different colour theme and check that everything looks sensible.
+#### Test Case 1: Opening the model editor
+
+1. Download the `sofastack/sofa-jraft` java database from GitHub.
+2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
+   - Check that the editor loads and shows methods to model.
+   - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
+   - Check that the "Open database" link works.
+
+#### Test Case 2: Model methods
+
+1. Expand one of the libraries.
+   - Change the model type and check that the other dropdowns change.
+2. Save the modeled methods.
+3. Click "Open extension pack"
+   - Check that the file explorer opens a directory with a "models" directory
+4. Open the ".model.yml" file corresponding to the library that was changed.
+   - Check that the file contrains the entries that was modeled.
+
+#### Test Case 3: Model with AI
+
+Note that this test requires the feature flag: `codeQL.model.llmGeneration`
+
+1. Click "Model with AI".
+   - Check that rows change to "Thinking".
+   - Check that results come back and rows get filled out.
+
+#### Test Case 4: Model as dependency
+
+1. Click "Model as dependency"
+   - Check that grouping are now per package (e.g. `com.alipay.sofa.rraft.option` or `com.google.protobuf`)
+2. Click "Generate".
+   - Check that rows are filled out.
+
+### General
+
+#### Test case 1: Change to a different colour theme
+
+Open at least one of the above MRVAs and at least one local query, then try changing to a different colour theme and check that everything looks sensible.
 Are there any components that are not showing up?
 
 ## Optional Test Cases
@@ -231,6 +324,7 @@ with this since it has quite a limited number of actions you can do.
 
 This requires running a MRVA query and seeing the results view.
 
+<!-- markdownlint-disable-next-line MD024 -->
 #### Test case 1: When variant analysis state is "pending"
 
 1. Can open a results view
@@ -256,10 +350,9 @@ This requires running a MRVA query and seeing the results view.
    2. When the file does not exist
 7. Can open query text
 8. Can sort repos
-   1. By name
-   2. By results
-   3. By stars
-   4. By last updated
+   1. Alphabetically
+   2. By number of results
+   3. By popularity
 9. Can filter repos
 10. Shows correct statistics
     1. Total number of results
@@ -319,6 +412,7 @@ This requires running a MRVA query and seeing the results view.
 1. Make changes via config file (ensure JSON schema is helping out)
 1. Close and re-open VS Code (ensure lists are there)
 1. Collapse/expand tree nodes
+1. Create a new list, right click and select "Add repositories with GitHub Code Search". Enter the language 'python' and the query "UserMixin". This should show a rate limiting notification after a while but eventually populate the list with roughly 770 items.
 
 Error cases that trigger an error notification:
 

docs/testing.md

@@ -18,6 +18,7 @@ Any test data you're using (sample projects, config files, etc.) must go in a `t
 ## Running the tests
 
 Pre-requisites:
+
 1. Run `npm run build`.
 2. You will need to have `npm run watch` running in the background.
 
@@ -29,7 +30,7 @@ Then, from the `extensions/ql-vscode` directory, use the appropriate command to
 * View Tests: `npm test:view`
 * VSCode integration tests: `npm run test:vscode-integration`
 
-#### CLI integration tests
+#### Running CLI integration tests from the terminal
 
 The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
 
@@ -51,7 +52,7 @@ You will need to run tests using a task from inside of VS Code, under the "Run a
 * View Tests: run the _Launch Unit Tests_ task
 * VSCode integration tests: run the _Launch Unit Tests - No Workspace_ and _Launch Unit Tests - Minimal Workspace_ tasks
 
-#### CLI integration tests
+#### Running CLI integration tests from VSCode
 
 The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
 
@@ -63,7 +64,7 @@ The CLI integration tests require the CodeQL standard libraries in order to run
 
 ## Running a single test
 
-### 1. From the terminal
+### 1. Running a single test from the terminal
 
 The easiest way to run a single test is to change the `it` of the test to `it.only` and then run the test command with some additional options
 to only run tests for this specific file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
@@ -78,7 +79,7 @@ You can also use the `--testNamePattern` option to run a specific test within a
 npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo"
 ```
 
-### 2. From VSCode
+### 2. Running a single test from VSCode
 
 Alternatively, you can run a single test inside VSCode. To do so, install the [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) extension. Then,
 you will have quicklinks to run a single test from within test files. To run a single unit or integration test, click the "Run" button. Debugging a single test is currently only supported
@@ -94,7 +95,9 @@ Multi-Repo Variant Analyses (MRVA) rely on the GitHub API. In order to make deve
 ### Using a pre-recorded test scenario
 
 To run a mock MRVA scenario, follow these steps:
+
 1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
+
 ```json
 "codeQL.mockGitHubApiServer": {
   "enabled": true
@@ -108,9 +111,11 @@ To run a mock MRVA scenario, follow these steps:
 If you want to replay the same scenario you should unload and reload it so requests are replayed from the start.
 
 ### Recording a new test scenario
+
 To record a new mock MRVA scenario, follow these steps:
 
 1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
+
 ```json
 "codeQL.mockGitHubApiServer": {
   "enabled": true
@@ -128,4 +133,4 @@ Once the scenario has been recorded, it's often useful to remove some of the req
 
 ### Scenario data location
 
-Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.
+Pre-recorded scenarios are stored in `./src/common/mock-gh-api/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.

docs/vscode-version.md

@@ -0,0 +1,33 @@
+# VS Code version
+
+The CodeQL for VS Code extension specifies the versions of VS Code that it is compatible with. VS Code will only offer to install and upgrade the extension when this version range is satisfied.
+
+## Where is the VS Code version specified
+
+1. Hard limit in [`package.json`](https://github.com/github/vscode-codeql/blob/606bfd7f877d9fffe4ff83b78015ab15f8840b12/extensions/ql-vscode/package.json#L16)
+
+    This is the value that VS Code understands and respects. If a user does not meet this version requirement then VS Code will not offer to install the CodeQL for VS Code extension, and if the extension is already installed then it will silently refuse to upgrade the extension.
+
+1. Soft limit in [`extension.ts`](https://github.com/github/vscode-codeql/blob/606bfd7f877d9fffe4ff83b78015ab15f8840b12/extensions/ql-vscode/src/extension.ts#L307)
+
+    This value is used internally by the CodeQL for VS Code extension and is used to provide a warning to users without blocking them from installing or upgrading. If the extension detects that this version range is not met it will output a warning message to the user prompting them to upgrade their VS Code version to ge the latest features of CodeQL.
+
+## When to update the VS Code version
+
+Generally we should aim to support as wide a range of VS Code versions as we can, so unless there is a reason to do so we do not update the minimum VS Code version requirement.
+Reasons for updating the minimum VS Code version include:
+
+- A new feature is included in VS Code. We may want to ensure that it is available to use so we do not have to provide an alternative code path.
+- A breaking change has happened in VS Code, and it is not possible to support both new and old versions.
+
+Also consider what percentage of our users are using each VS Code version. This information is available in our telemetry.
+
+## How to update the VS Code version
+
+To provide a good experience to users, it is recommented to update the `MIN_VERSION` in `extension.ts` first and release, and then update the `vscode` version in `package.json` and release again. By stagging this update across two releases it gives users on older VS Code versions a chance to upgrade before it silently refuses to upgrade them.
+
+## VS Code version used in tests
+
+Our integration tests are currently pinned to use an older version of VS Code due to <https://github.com/github/vscode-codeql/issues/2402>.
+This version is specified in [`jest-runner-vscode.config.base.js`](https://github.com/github/vscode-codeql/blob/d93f2b67c84e79737b0ce4bb74e31558b5f5166e/extensions/ql-vscode/test/vscode-tests/jest-runner-vscode.config.base.js#L17).
+Until this is resolved this will limit us updating our minimum supported version of VS Code.

extensions/ql-vscode/.babelrc.json

@@ -0,0 +1,16 @@
+{
+  "sourceType": "unambiguous",
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "chrome": 100
+        }
+      }
+    ],
+    "@babel/preset-typescript",
+    "@babel/preset-react"
+  ],
+  "plugins": []
+}

extensions/ql-vscode/.eslintrc.js

@@ -65,10 +65,6 @@ const baseConfig = {
     "import/no-namespace": "off",
     "import/no-unresolved": "off",
     "import/no-webpack-loader-syntax": "off",
-    "jsx-a11y/anchor-is-valid": "off",
-    "jsx-a11y/no-noninteractive-element-interactions": "off",
-    "jsx-a11y/no-static-element-interactions": "off",
-    "jsx-a11y/click-events-have-key-events": "off",
     "no-invalid-this": "off",
     "no-fallthrough": "off",
     "no-console": "off",

extensions/ql-vscode/.markdownlint-cli2.cjs

@@ -0,0 +1,16 @@
+// Having the base options in a top-level config file means
+// that the VS Code markdownlint extension can pick them up
+// too, since that only considers _this_ file when looking
+// at files in this directory or below.
+base_options = require('../../.markdownlint.json')
+
+const options = require('@github/markdownlint-github').init(
+  base_options
+)
+module.exports = {
+    config: options,
+    customRules: ["@github/markdownlint-github"],
+    outputFormatters: [
+      [ "markdownlint-cli2-formatter-pretty", { "appendLink": true } ] // ensures the error message includes a link to the rule documentation
+    ]
+} 

extensions/ql-vscode/.nvmrc

@@ -1 +1 @@
-v16.14.2
+v16.17.1

extensions/ql-vscode/.storybook/main.ts

@@ -1,4 +1,4 @@
-import type { StorybookConfig } from "@storybook/core-common";
+import type { StorybookConfig } from "@storybook/react-webpack5";
 
 const config: StorybookConfig = {
   stories: ["../src/**/*.stories.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"],
@@ -8,10 +8,13 @@ const config: StorybookConfig = {
     "@storybook/addon-interactions",
     "./vscode-theme-addon/preset.ts",
   ],
-  framework: "@storybook/react",
-  core: {
-    builder: "@storybook/builder-webpack5",
+  framework: {
+    name: "@storybook/react-webpack5",
+    options: {},
+  },
+  docs: {
+    autodocs: "tag",
   },
 };
 
-module.exports = config;
+export default config;

extensions/ql-vscode/.storybook/manager.ts

@@ -1,4 +1,4 @@
-import { addons } from "@storybook/addons";
+import { addons } from "@storybook/manager-api";
 import { themes } from "@storybook/theming";
 
 addons.setConfig({

extensions/ql-vscode/.storybook/preview.ts

@@ -1,31 +1,44 @@
+import { Preview } from "@storybook/react";
 import { themes } from "@storybook/theming";
 import { action } from "@storybook/addon-actions";
 
 // Allow all stories/components to use Codicons
 import "@vscode/codicons/dist/codicon.css";
 
-// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
-export const parameters = {
-  // All props starting with `on` will automatically receive an action as a prop
-  actions: { argTypesRegex: "^on[A-Z].*" },
-  // All props matching these names will automatically get the correct control
-  controls: {
-    matchers: {
-      color: /(background|color)$/i,
-      date: /Date$/,
-    },
-  },
-  // Use a dark theme to be aligned with VSCode
-  docs: {
-    theme: themes.dark,
-  },
-  backgrounds: {
-    // The background is injected by our theme CSS files
-    disable: true,
-  },
-};
+import type { VsCodeApi } from "../src/view/vscode-api";
 
-(window as any).acquireVsCodeApi = () => ({
+declare global {
+  interface Window {
+    acquireVsCodeApi: () => VsCodeApi;
+  }
+}
+
+window.acquireVsCodeApi = () => ({
   postMessage: action("post-vscode-message"),
   setState: action("set-vscode-state"),
 });
+
+// https://storybook.js.org/docs/react/configure/overview#configure-story-rendering
+const preview: Preview = {
+  parameters: {
+    // All props starting with `on` will automatically receive an action as a prop
+    actions: { argTypesRegex: "^on[A-Z].*" },
+    // All props matching these names will automatically get the correct control
+    controls: {
+      matchers: {
+        color: /(background|color)$/i,
+        date: /Date$/,
+      },
+    },
+    // Use a dark theme to be aligned with VSCode
+    docs: {
+      theme: themes.dark,
+    },
+    backgrounds: {
+      // The background is injected by our theme CSS files
+      disable: true,
+    },
+  },
+};
+
+export default preview;

extensions/ql-vscode/.storybook/vscode-theme-addon/ThemeSelector.tsx

@@ -1,14 +1,12 @@
 import * as React from "react";
 import { FunctionComponent, useCallback } from "react";
 
-import { useGlobals } from "@storybook/api";
+import { useGlobals } from "@storybook/manager-api";
 import {
   IconButton,
   Icons,
-  WithTooltip,
   TooltipLinkList,
-  Link,
-  WithHideFn,
+  WithTooltip,
 } from "@storybook/components";
 
 import { themeNames, VSCodeTheme } from "./theme";
@@ -26,7 +24,7 @@ export const ThemeSelector: FunctionComponent = () => {
   );
 
   const createLinks = useCallback(
-    (onHide: () => void): Link[] =>
+    (onHide: () => void) =>
       Object.values(VSCodeTheme).map((theme) => ({
         id: theme,
         onClick() {
@@ -44,8 +42,8 @@ export const ThemeSelector: FunctionComponent = () => {
     <WithTooltip
       placement="top"
       trigger="click"
-      closeOnClick
-      tooltip={({ onHide }: WithHideFn) => (
+      closeOnOutsideClick
+      tooltip={({ onHide }: { onHide: () => void }) => (
         <TooltipLinkList links={createLinks(onHide)} />
       )}
     >

extensions/ql-vscode/.storybook/vscode-theme-addon/manager.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { addons, types } from "@storybook/addons";
+import { addons, types } from "@storybook/manager-api";
 import { ThemeSelector } from "./ThemeSelector";
 
 const ADDON_ID = "vscode-theme-addon";

extensions/ql-vscode/.storybook/vscode-theme-addon/preset.ts

@@ -1,4 +1,4 @@
-export function config(entry = []) {
+export function previewAnnotations(entry = []) {
   return [...entry, require.resolve("./preview.ts")];
 }
 

extensions/ql-vscode/.storybook/vscode-theme-addon/withTheme.ts

@@ -1,6 +1,5 @@
-import { useEffect, useGlobals } from "@storybook/addons";
+import { useEffect } from "react";
 import type {
-  AnyFramework,
   PartialStoryFn as StoryFunction,
   StoryContext,
 } from "@storybook/csf";
@@ -34,11 +33,8 @@ const themeFiles: { [key in VSCodeTheme]: string } = {
       .default,
 };
 
-export const withTheme = (
-  StoryFn: StoryFunction<AnyFramework>,
-  context: StoryContext<AnyFramework>,
-) => {
-  const [{ vscodeTheme }] = useGlobals();
+export const withTheme = (StoryFn: StoryFunction, context: StoryContext) => {
+  const { vscodeTheme } = context.globals;
 
   useEffect(() => {
     const styleSelectorId =

extensions/ql-vscode/CHANGELOG.md

@@ -1,6 +1,87 @@
 # CodeQL for Visual Studio Code: Changelog
 
-## 1.8.0 - 8 March 2023
+## 1.9.1 - 29 September 2023
+
+- Add warning when using a VS Code version older than 1.82.0. [#2854](https://github.com/github/vscode-codeql/pull/2854)
+- Fix a bug when parsing large evaluation log summaries. [#2858](https://github.com/github/vscode-codeql/pull/2858)
+- Right-align and format numbers in raw result tables. [#2864](https://github.com/github/vscode-codeql/pull/2864)
+- Remove rate limit warning notifications when using Code Search to add repositories to a variant analysis list. [#2812](https://github.com/github/vscode-codeql/pull/2812)
+
+## 1.9.0 - 19 September 2023
+
+- Release the [CodeQL model editor](https://codeql.github.com/docs/codeql/codeql-for-visual-studio-code/using-the-codeql-model-editor) to create CodeQL model packs for Java frameworks. Open the editor using the "CodeQL: Open CodeQL Model Editor (Beta)" command. [#2823](https://github.com/github/vscode-codeql/pull/2823)
+
+## 1.8.12 - 11 September 2023
+
+- Fix a bug where variant analysis queries would fail for queries in the `codeql/java-queries` query pack. [#2786](https://github.com/github/vscode-codeql/pull/2786)
+
+## 1.8.11 - 7 September 2023
+
+- Update how variant analysis results are displayed. For queries with ["path-problem" or "problem" `@kind`](https://codeql.github.com/docs/writing-codeql-queries/metadata-for-codeql-queries/#metadata-properties), you can choose to display the results as rendered alerts or as a table of raw results. For queries with any other `@kind`, the results are displayed as a table. [#2745](https://github.com/github/vscode-codeql/pull/2745) & [#2749](https://github.com/github/vscode-codeql/pull/2749)
+- When running variant analyses, don't download artifacts for repositories with no results. [#2736](https://github.com/github/vscode-codeql/pull/2736)
+- Group the extension settings, so that they're easier to find in the Settings UI. [#2706](https://github.com/github/vscode-codeql/pull/2706)
+
+## 1.8.10 - 15 August 2023
+
+- Add a code lens to make the `CodeQL: Open Referenced File` command more discoverable. Click the "Open referenced file" prompt in a `.qlref` file to jump to the referenced `.ql` file. [#2704](https://github.com/github/vscode-codeql/pull/2704)
+
+## 1.8.9 - 3 August 2023
+
+- Remove "last updated" information and sorting from variant analysis results view. [#2637](https://github.com/github/vscode-codeql/pull/2637)
+- Links to code on GitHub now include column numbers as well as line numbers. [#2406](https://github.com/github/vscode-codeql/pull/2406)
+- No longer highlight trailing commas for jump to definition.  [#2615](https://github.com/github/vscode-codeql/pull/2615)
+- Fix a bug where the QHelp preview page was not being refreshed after changes to the underlying `.qhelp` file. [#2660](https://github.com/github/vscode-codeql/pull/2660)
+
+## 1.8.8 - 17 July 2023
+
+- Remove support for CodeQL CLI versions older than 2.9.4. [#2610](https://github.com/github/vscode-codeql/pull/2610)
+- Implement syntax highlighting for the `additional` and `default` keywords. [#2609](https://github.com/github/vscode-codeql/pull/2609)
+
+## 1.8.7 - 29 June 2023
+
+- Show a run button on the file tab for query files, that will start a local query. This button will only show when a local database is selected in the extension. [#2544](https://github.com/github/vscode-codeql/pull/2544)
+- Add a `CodeQL: Quick Evaluation Count` command to generate the count summary statistics of the results set
+  without spending the time to compute locations and strings. [#2475](https://github.com/github/vscode-codeql/pull/2475)
+
+## 1.8.6 - 14 June 2023
+
+- Add repositories to a variant analysis list with GitHub Code Search. [#2439](https://github.com/github/vscode-codeql/pull/2439) and [#2476](https://github.com/github/vscode-codeql/pull/2476)
+
+## 1.8.5 - 6 June 2023
+
+- Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Fix bug to ensure error messages have complete stack trace in message logs. [#2425](https://github.com/github/vscode-codeql/pull/2425)
+- Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
+- Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
+- Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
+- Retry variant analysis results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
+
+## 1.8.4 - 3 May 2023
+
+- Avoid repeated error messages when unable to monitor a variant analysis. [#2396](https://github.com/github/vscode-codeql/pull/2396)
+- Fix bug where a variant analysis didn't display the `#select` results set correctly when the [query metadata](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#query-metadata) didn't exactly match the query results. [#2395](https://github.com/github/vscode-codeql/pull/2395)
+- On the variant analysis results page, show the count of successful analyses instead of completed analyses, and indicate the reason why analyses were not successful. [#2349](https://github.com/github/vscode-codeql/pull/2349)
+- Fix bug where the "CodeQL: Set Current Database" command didn't always select the database. [#2384](https://github.com/github/vscode-codeql/pull/2384)
+
+## 1.8.3 - 26 April 2023
+
+- Added ability to filter repositories for a variant analysis to only those that have results [#2343](https://github.com/github/vscode-codeql/pull/2343)
+- Add new configuration option to allow downloading databases from http, non-secure servers. [#2332](https://github.com/github/vscode-codeql/pull/2332)
+- Remove title actions from the query history panel that depended on history items being selected. [#2350](https://github.com/github/vscode-codeql/pull/2350)
+
+## 1.8.2 - 12 April 2023
+
+- Fix bug where users could end up with the managed CodeQL CLI getting uninstalled during upgrades and not reinstalled. [#2294](https://github.com/github/vscode-codeql/pull/2294)
+- Fix bug that was causing code flows to not get updated when switching between results. [#2288](https://github.com/github/vscode-codeql/pull/2288)
+- Restart the CodeQL language server whenever the _CodeQL: Restart Query Server_ command is invoked. This avoids bugs where the CLI version changes to support new language features, but the language server is not updated. [#2238](https://github.com/github/vscode-codeql/pull/2238)
+- Avoid requiring a manual restart of the query server when the [external CLI config file](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file) changes. [#2289](https://github.com/github/vscode-codeql/pull/2289)
+
+## 1.8.1 - 23 March 2023
+
+- Show data flow paths of a variant analysis in a new tab. [#2172](https://github.com/github/vscode-codeql/pull/2172) & [#2182](https://github.com/github/vscode-codeql/pull/2182)
+- Show labels of entities in exported CSV results. [#2170](https://github.com/github/vscode-codeql/pull/2170)
+
+## 1.8.0 - 9 March 2023
 
 - Send telemetry about unhandled errors happening within the extension. [#2125](https://github.com/github/vscode-codeql/pull/2125)
 - Enable multi-repository variant analysis. [#2144](https://github.com/github/vscode-codeql/pull/2144)

extensions/ql-vscode/README.md

@@ -28,7 +28,7 @@ For information about other configurations, see the separate [CodeQL help](https
 1. [Run a query](#running-a-query).
 
 ---
-
+<!-- markdownlint-disable-next-line MD024 -->
 ## Quick start: Installing and configuring the extension
 
 ### Installing the extension
@@ -71,6 +71,7 @@ in the starter workspace directory.
 
 If you're using your own clone of the CodeQL standard libraries, you can do a `git pull` from where you have the libraries checked out.
 
+<!-- markdownlint-disable-next-line MD024 -->
 ## Quick start: Using CodeQL
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -9,6 +9,7 @@ import {
 } from "fs-extra";
 import { resolve, join } from "path";
 import { isDevBuild } from "./dev";
+import type * as packageJsonType from "../package.json";
 
 export interface DeployedPackage {
   distPath: string;
@@ -46,12 +47,10 @@ async function copyPackage(
   );
 }
 
-export async function deployPackage(
-  packageJsonPath: string,
-): Promise<DeployedPackage> {
+export async function deployPackage(): Promise<DeployedPackage> {
   try {
-    const packageJson: any = JSON.parse(
-      await readFile(packageJsonPath, "utf8"),
+    const packageJson: typeof packageJsonType = JSON.parse(
+      await readFile(resolve(__dirname, "../package.json"), "utf8"),
     );
 
     const distDir = join(__dirname, "../../../dist");
@@ -93,12 +92,6 @@ export async function deployPackage(
     );
     await copyPackage(sourcePath, distPath);
 
-    // This is necessary for vsce to know the dependencies
-    await copyDirectory(
-      resolve(sourcePath, "node_modules"),
-      resolve(distPath, "node_modules"),
-    );
-
     return {
       distPath,
       name: packageJson.name,

extensions/ql-vscode/gulpfile.ts/package.ts

@@ -3,9 +3,7 @@ import { deployPackage } from "./deploy";
 import { spawn } from "child-process-promise";
 
 export async function packageExtension(): Promise<void> {
-  const deployedPackage = await deployPackage(
-    resolve(__dirname, "../package.json"),
-  );
+  const deployedPackage = await deployPackage();
   console.log(
     `Packaging extension '${deployedPackage.name}@${deployedPackage.version}'...`,
   );
@@ -17,6 +15,7 @@ export async function packageExtension(): Promise<void> {
       "..",
       `${deployedPackage.name}-${deployedPackage.version}.vsix`,
     ),
+    "--no-dependencies",
   ];
   const proc = spawn(resolve(__dirname, "../node_modules/.bin/vsce"), args, {
     cwd: deployedPackage.distPath,

extensions/ql-vscode/gulpfile.ts/tsconfig.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/tsconfig",
   "extends": "../tsconfig.json",
   "compilerOptions": {
-    "rootDir": "."
+    "rootDir": ".."
   },
   "include": ["*.ts"]
 }

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -15,9 +15,6 @@ export const config: webpack.Configuration = {
   devtool: isDevBuild ? "inline-source-map" : "source-map",
   resolve: {
     extensions: [".js", ".ts", ".tsx", ".json"],
-    fallback: {
-      path: require.resolve("path-browserify"),
-    },
   },
   module: {
     rules: [
@@ -62,18 +59,10 @@ export const config: webpack.Configuration = {
       },
       {
         test: /\.(woff(2)?|ttf|eot)$/,
-        use: [
-          {
-            loader: "file-loader",
-            options: {
-              name: "[name].[ext]",
-              outputPath: "fonts/",
-              // We need this to make Webpack use the correct path for the fonts.
-              // Without this, the CSS file will use `url([object Module])`
-              esModule: false,
-            },
-          },
-        ],
+        type: "asset/resource",
+        generator: {
+          filename: "fonts/[hash][ext][query]",
+        },
       },
     ],
   },

extensions/ql-vscode/language-configuration.json

@@ -30,5 +30,5 @@
       "end": "^\\s*//\\s*#?endregion\\b"
     }
   },
-  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s]+)"
+  "wordPattern": "(-?\\d*\\.\\d\\w*)|([^\\~\\!\\@\\#\\%\\^\\&\\*\\(\\)\\-\\=\\+\\[\\{\\]\\}\\\\\\|\\;\\:\\'\\\"\\.\\<\\>\\/\\?\\s\\,]+)"
 }

extensions/ql-vscode/media/dark/sort-alpha.svg

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="none"
-	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
-<g>
-	<g>
-		<g>
-			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413" fill="#C5C5C5"/>
-			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72
-				250.027,338.24 432,338.24 432,304.32" fill="#C5C5C5"/>
-			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227" fill="#C5C5C5"/>
-			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
-				L113.28,140.48l41.387,110.507H71.893z" fill="#C5C5C5"/>
-		</g>
-	</g>
-</g>
-</svg>

extensions/ql-vscode/media/dark/sort-date.svg

@@ -1,3 +0,0 @@
-<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#C5C5C5"/>
-</svg>

extensions/ql-vscode/media/dark/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#C5C5C5"/></svg>

extensions/ql-vscode/media/light/sort-alpha.svg

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!-- Generator: Adobe Illustrator 19.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 432 432" style="enable-background:new 0 0 432 432;" xml:space="preserve">
-<g>
-	<g>
-		<g>
-			<polygon points="234.24,9.067 183.893,59.413 284.587,59.413 			"/>
-			<polygon points="301.44,304.32 427.947,120.853 427.947,93.973 250.88,93.973 250.88,128.107 376.32,128.107 250.027,310.72 
-				250.027,338.24 432,338.24 432,304.32 			"/>
-			<polygon points="234.24,422.933 283.947,373.227 184.533,373.227 			"/>
-			<path d="M226.773,338.24L130.987,93.76H96L0,338.24h39.253l19.627-52.267h109.013l19.627,52.267H226.773z M71.893,250.987
-				L113.28,140.48l41.387,110.507H71.893z"/>
-		</g>
-	</g>
-</g>
-</svg>

extensions/ql-vscode/media/light/sort-date.svg

@@ -1,3 +0,0 @@
-<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M7 2L6 3V6H7V3H14V5.45306L14.2071 5.29286L15 6.08576V3L14 2H7ZM8 4H10V6H8V4ZM5 9H3V11H5V9ZM2 7L1 8V13L2 14H9L10 13V8L9 7H2ZM2 13V8H9V13H2ZM8 10H6V12H8V10ZM13 4H12V7.86388L10.818 6.68192L10.1109 7.38903L12.1465 9.42454L12.8536 9.42454L14.889 7.38908L14.1819 6.68197L13 7.86388V4Z" fill="#424242"/>
-</svg>

extensions/ql-vscode/media/light/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#424242"/></svg>

extensions/ql-vscode/patches/jest-runner-vscode+3.0.1.patch

@@ -25,7 +25,7 @@ index 1ac28d5..f91f216 100644
          await super.setup();
          await (0, load_pnp_1.default)();
 diff --git a/node_modules/jest-runner-vscode/dist/child/runner.js b/node_modules/jest-runner-vscode/dist/child/runner.js
-index 0663c5c..4991663 100644
+index 0663c5c..bdf4a8b 100644
 --- a/node_modules/jest-runner-vscode/dist/child/runner.js
 +++ b/node_modules/jest-runner-vscode/dist/child/runner.js
 @@ -18,10 +18,13 @@ async function run() {
@@ -43,6 +43,16 @@ index 0663c5c..4991663 100644
          const options = JSON.parse(PARENT_JEST_OPTIONS);
          const jestOptions = [
              ...options.args,
+@@ -39,6 +42,9 @@ async function run() {
+             ...(argv.projects?.map(project => path_1.default.resolve(project)) || []),
+             options.workspacePath,
+         ]);
++        const testPaths = new Set(argv._.map(testPath => path_1.default.resolve(testPath)));
++        argv._ = [...testPaths];
++
+         await (0, core_1.runCLI)(argv, [...projects]);
+     }
+     catch (error) {
 diff --git a/node_modules/jest-runner-vscode/dist/public-types.d.ts b/node_modules/jest-runner-vscode/dist/public-types.d.ts
 index 57716e5..d8614af 100644
 --- a/node_modules/jest-runner-vscode/dist/public-types.d.ts

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -18,13 +18,16 @@ import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
 
 import { getFiles } from "./util/files";
-import type { GitHubApiRequest } from "../src/mocks/gh-api-request";
-import { isGetVariantAnalysisRequest } from "../src/mocks/gh-api-request";
+import type { GitHubApiRequest } from "../src/common/mock-gh-api/gh-api-request";
+import { isGetVariantAnalysisRequest } from "../src/common/mock-gh-api/gh-api-request";
 import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
 import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/common/mock-gh-api/scenarios",
+);
 
 // Make sure we don't run into rate limits by automatically waiting until we can
 // make another request.

extensions/ql-vscode/scripts/find-deadcode.ts

@@ -0,0 +1,48 @@
+import { basename, join, relative, resolve } from "path";
+import analyzeTsConfig from "ts-unused-exports";
+import { containsPath, pathsEqual } from "../src/common/files";
+import { exit } from "process";
+
+function ignoreFile(file: string): boolean {
+  return (
+    containsPath("gulpfile.ts", file) ||
+    containsPath(".storybook", file) ||
+    containsPath(join("src", "stories"), file) ||
+    pathsEqual(
+      join("test", "vscode-tests", "jest-runner-installed-extensions.ts"),
+      file,
+    ) ||
+    basename(file) === "jest.config.ts" ||
+    basename(file) === "index.tsx" ||
+    basename(file) === "index.ts"
+  );
+}
+
+function main() {
+  const repositoryRoot = resolve(join(__dirname, ".."));
+
+  const result = analyzeTsConfig("tsconfig.deadcode.json");
+  let foundUnusedExports = false;
+
+  for (const [filepath, exportNameAndLocations] of Object.entries(result)) {
+    const relativeFilepath = relative(repositoryRoot, filepath);
+
+    if (ignoreFile(relativeFilepath)) {
+      continue;
+    }
+
+    foundUnusedExports = true;
+
+    console.log(relativeFilepath);
+    for (const exportNameAndLocation of exportNameAndLocations) {
+      console.log(`    ${exportNameAndLocation.exportName}`);
+    }
+    console.log();
+  }
+
+  if (foundUnusedExports) {
+    exit(1);
+  }
+}
+
+main();

extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts

@@ -20,7 +20,10 @@ if (process.argv.length !== 3) {
 const scenarioName = process.argv[2];
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/common/mock-gh-api/scenarios",
+);
 const scenarioDirectory = resolve(scenariosDirectory, scenarioName);
 
 async function fixScenarioFiles() {

extensions/ql-vscode/scripts/generate-schemas.ts

@@ -0,0 +1,72 @@
+import { createGenerator } from "ts-json-schema-generator";
+import { join, resolve } from "path";
+import { outputFile } from "fs-extra";
+import { format, resolveConfig } from "prettier";
+
+const extensionDirectory = resolve(__dirname, "..");
+
+const schemas = [
+  {
+    path: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "extension-pack-metadata.ts",
+    ),
+    type: "ExtensionPackMetadata",
+    schemaPath: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "extension-pack-metadata.schema.json",
+    ),
+  },
+  {
+    path: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "model-extension-file.ts",
+    ),
+    type: "ModelExtensionFile",
+    schemaPath: join(
+      extensionDirectory,
+      "src",
+      "model-editor",
+      "model-extension-file.schema.json",
+    ),
+  },
+];
+
+async function generateSchema(
+  schemaDefinition: (typeof schemas)[number],
+): Promise<void> {
+  const schema = createGenerator({
+    path: schemaDefinition.path,
+    tsconfig: resolve(extensionDirectory, "tsconfig.json"),
+    type: schemaDefinition.type,
+    skipTypeCheck: true,
+    topRef: true,
+    additionalProperties: true,
+  }).createSchema(schemaDefinition.type);
+
+  const schemaJson = JSON.stringify(schema, null, 2);
+
+  const prettierOptions = await resolveConfig(schemaDefinition.schemaPath);
+
+  const formattedSchemaJson = await format(schemaJson, {
+    ...prettierOptions,
+    filepath: schemaDefinition.schemaPath,
+  });
+
+  await outputFile(schemaDefinition.schemaPath, formattedSchemaJson);
+}
+
+async function generateSchemas() {
+  await Promise.all(schemas.map(generateSchema));
+}
+
+generateSchemas().catch((e: unknown) => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -8,13 +8,19 @@ import { getFiles } from "./util/files";
 
 const extensionDirectory = resolve(__dirname, "..");
 const rootDirectory = resolve(extensionDirectory, "../..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/common/mock-gh-api/scenarios",
+);
 
 const debug = process.env.RUNNER_DEBUG || process.argv.includes("--debug");
 
 async function lintScenarios() {
   const schema = createGenerator({
-    path: resolve(extensionDirectory, "src/mocks/gh-api-request.ts"),
+    path: resolve(
+      extensionDirectory,
+      "src/common/mock-gh-api/gh-api-request.ts",
+    ),
     tsconfig: resolve(extensionDirectory, "tsconfig.json"),
     type: "GitHubApiRequest",
     skipTypeCheck: true,

extensions/ql-vscode/src/code-tour/code-tour.ts

@@ -0,0 +1,55 @@
+import { AppCommandManager } from "../common/commands";
+import { Uri, workspace } from "vscode";
+import { join } from "path";
+import { pathExists } from "fs-extra";
+import { isCodespacesTemplate } from "../config";
+import { showBinaryChoiceDialog } from "../common/vscode/dialog";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Check if the current workspace is the CodeTour and open the workspace folder.
+ * Without this, we can't run the code tour correctly.
+ **/
+export async function prepareCodeTour(
+  commandManager: AppCommandManager,
+): Promise<void> {
+  if (workspace.workspaceFolders?.length) {
+    const currentFolder = workspace.workspaceFolders[0].uri.fsPath;
+
+    const tutorialWorkspacePath = join(
+      currentFolder,
+      "tutorial.code-workspace",
+    );
+    const toursFolderPath = join(currentFolder, ".tours");
+
+    /** We're opening the tutorial workspace, if we detect it.
+     * This will only happen if the following three conditions are met:
+     * - the .tours folder exists
+     * - the tutorial.code-workspace file exists
+     * - the CODESPACES_TEMPLATE setting doesn't exist (it's only set if the user has already opened
+     * the tutorial workspace so it's a good indicator that the user is in the folder but has ignored
+     * the prompt to open the workspace)
+     */
+    if (
+      (await pathExists(tutorialWorkspacePath)) &&
+      (await pathExists(toursFolderPath)) &&
+      !isCodespacesTemplate()
+    ) {
+      const answer = await showBinaryChoiceDialog(
+        "We've detected you're in the CodeQL Tour repo. We will need to open the workspace file to continue. Reload?",
+      );
+
+      if (!answer) {
+        return;
+      }
+
+      const tutorialWorkspaceUri = Uri.file(tutorialWorkspacePath);
+
+      void extLogger.log(
+        `In prepareCodeTour() method, going to open the tutorial workspace file: ${tutorialWorkspacePath}`,
+      );
+
+      await commandManager.execute("vscode.openFolder", tutorialWorkspaceUri);
+    }
+  }
+}

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,7 +1,7 @@
 import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { Logger } from "../common/logging";
+import { getErrorMessage } from "../common/helpers-pure";
 
 /**
  * Get the version of a CodeQL CLI.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -6,29 +6,31 @@ import { dirname, join, delimiter } from "path";
 import * as sarif from "sarif";
 import { SemVer } from "semver";
 import { Readable } from "stream";
-import { StringDecoder } from "string_decoder";
 import tk from "tree-kill";
 import { promisify } from "util";
-import { CancellationToken, commands, Disposable, Uri } from "vscode";
+import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "./pure/bqrs-cli-types";
-import { allowCanaryQueryServer, CliConfig } from "./config";
+import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
   FindDistributionResultKind,
 } from "./distribution";
 import {
   assertNever,
+  getChildProcessErrorMessage,
   getErrorMessage,
   getErrorStack,
-} from "./pure/helpers-pure";
-import { QueryMetadata, SortDirection } from "./pure/interface-types";
-import { Logger, ProgressReporter } from "./common";
-import { CompilationMessage } from "./pure/legacy-messages";
-import { sarifParser } from "./sarif-parser";
-import { walkDirectory } from "./helpers";
-import { App } from "./common/app";
-import { QueryLanguage } from "./common/query-language";
+} from "../common/helpers-pure";
+import { walkDirectory } from "../common/files";
+import { QueryMetadata, SortDirection } from "../common/interface-types";
+import { BaseLogger, Logger } from "../common/logging";
+import { ProgressReporter } from "../common/logging/vscode";
+import { CompilationMessage } from "../query-server/legacy-messages";
+import { sarifParser } from "../common/sarif-parser";
+import { App } from "../common/app";
+import { QueryLanguage } from "../common/query-language";
+import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
 
 /**
  * The version of the SARIF format that we are using.
@@ -107,6 +109,30 @@ export type MlModelInfo = {
 /** The expected output of `codeql resolve ml-models`. */
 export type MlModelsInfo = { models: MlModelInfo[] };
 
+/** Information about a data extension predicate, as resolved by `codeql resolve extensions`. */
+export type DataExtensionResult = {
+  predicate: string;
+  file: string;
+  index: number;
+};
+
+/** The expected output of `codeql resolve extensions`. */
+export type ResolveExtensionsResult = {
+  models: MlModelInfo[];
+  data: {
+    [path: string]: DataExtensionResult[];
+  };
+};
+
+export type GenerateExtensiblePredicateMetadataResult = {
+  // There are other properties in this object, but they are
+  // not relevant for its use in the extension, so we omit them.
+  extensible_predicates: Array<{
+    // pack relative path
+    path: string;
+  }>;
+};
+
 /**
  * The expected output of `codeql resolve qlref`.
  */
@@ -119,6 +145,11 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries`.
+ */
+export type ResolvedQueries = string[];
+
 /**
  * The expected output of `codeql resolve tests`.
  */
@@ -134,6 +165,7 @@ export interface TestCompleted {
   compilationMs: number;
   evaluationMs: number;
   expected: string;
+  actual?: string;
   diff: string[] | undefined;
   failureDescription?: string;
   failureStage?: string;
@@ -155,6 +187,8 @@ export type OnLineCallback = (
   line: string,
 ) => Promise<string | undefined> | string | undefined;
 
+type VersionChangedListener = (newVersion: SemVer | undefined) => void;
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -172,7 +206,9 @@ export class CodeQLCliServer implements Disposable {
   nullBuffer: Buffer;
 
   /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: Promise<SemVer> | undefined;
+  private _version: SemVer | undefined;
+
+  private _versionChangedListeners: VersionChangedListener[] = [];
 
   /**
    * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
@@ -193,7 +229,7 @@ export class CodeQLCliServer implements Disposable {
     private readonly app: App,
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    public readonly logger: Logger,
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -305,6 +341,7 @@ export class CodeQLCliServer implements Disposable {
     commandArgs: string[],
     description: string,
     onLine?: OnLineCallback,
+    silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
@@ -324,7 +361,12 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(" ");
-      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      // If we are running silently, we don't want to print anything to the console.
+      if (!silent) {
+        void this.logger.log(
+          `${description} using CodeQL CLI: ${argsString}...`,
+        );
+      }
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -370,24 +412,30 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        void this.logger.log("CLI command succeeded.");
+        if (!silent) {
+          void this.logger.log("CLI command succeeded.");
+        }
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
+        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
         const newError =
           stderrBuffers.length === 0
-            ? new Error(`${description} failed: ${err}`)
+            ? new Error(
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
+              )
             : new Error(
-                `${description} failed: ${Buffer.concat(stderrBuffers).toString(
-                  "utf8",
-                )}`,
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
+                  stderrBuffers,
+                ).toString("utf8")}`,
               );
         newError.stack += getErrorStack(err);
         throw newError;
       } finally {
-        void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent) {
+          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
         process.stderr.removeAllListeners("data");
@@ -424,7 +472,7 @@ export class CodeQLCliServer implements Disposable {
     command: string[],
     commandArgs: string[],
     cancellationToken?: CancellationToken,
-    logger?: Logger,
+    logger?: BaseLogger,
   ): AsyncGenerator<string, void, unknown> {
     // Add format argument first, in case commandArgs contains positional parameters.
     const args = [...command, "--format", "jsonz", ...commandArgs];
@@ -432,6 +480,11 @@ export class CodeQLCliServer implements Disposable {
     // Spawn the CodeQL process
     const codeqlPath = await this.getCodeQlPath();
     const childPromise = spawn(codeqlPath, args);
+    // Avoid a runtime message about unhandled rejection.
+    childPromise.catch(() => {
+      /**/
+    });
+
     const child = childPromise.childProcess;
 
     let cancellationRegistration: Disposable | undefined = undefined;
@@ -482,7 +535,7 @@ export class CodeQLCliServer implements Disposable {
       logger,
     }: {
       cancellationToken?: CancellationToken;
-      logger?: Logger;
+      logger?: BaseLogger;
     } = {},
   ): AsyncGenerator<EventType, void, unknown> {
     for await (const event of this.runAsyncCodeQlCliCommandInternal(
@@ -495,9 +548,7 @@ export class CodeQLCliServer implements Disposable {
         yield JSON.parse(event) as EventType;
       } catch (err) {
         throw new Error(
-          `Parsing output of ${description} failed: ${
-            (err as any).stderr || getErrorMessage(err)
-          }`,
+          `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
         );
       }
     }
@@ -519,9 +570,11 @@ export class CodeQLCliServer implements Disposable {
     {
       progressReporter,
       onLine,
+      silent = false,
     }: {
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<string> {
     if (progressReporter) {
@@ -537,6 +590,7 @@ export class CodeQLCliServer implements Disposable {
             commandArgs,
             description,
             onLine,
+            silent,
           ).then(resolve, reject);
         } catch (err) {
           reject(err);
@@ -570,10 +624,12 @@ export class CodeQLCliServer implements Disposable {
       addFormat = true,
       progressReporter,
       onLine,
+      silent = false,
     }: {
       addFormat?: boolean;
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
@@ -584,14 +640,13 @@ export class CodeQLCliServer implements Disposable {
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
       onLine,
+      silent,
     });
     try {
       return JSON.parse(result) as OutputType;
     } catch (err) {
       throw new Error(
-        `Parsing output of ${description} failed: ${
-          (err as any).stderr || getErrorMessage(err)
-        }`,
+        `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
       );
     }
   }
@@ -670,6 +725,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(
     workspaces: string[],
     queryPath: string,
+    silent = false,
   ): Promise<QuerySetup> {
     const subcommandArgs = [
       "--query",
@@ -680,6 +736,7 @@ export class CodeQLCliServer implements Disposable {
       ["resolve", "library-path"],
       subcommandArgs,
       "Resolving library paths",
+      { silent },
     );
   }
 
@@ -706,6 +763,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Finds all available queries in a given directory.
+   * @param queryDir Root of directory tree to search for queries.
+   * @param silent If true, don't print logs to the CodeQL extension log.
+   * @returns The list of queries that were found.
+   */
+  public async resolveQueries(
+    queryDir: string,
+    silent?: boolean,
+  ): Promise<ResolvedQueries> {
+    const subcommandArgs = [queryDir];
+    return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
+      ["resolve", "queries"],
+      subcommandArgs,
+      "Resolving queries",
+      { silent },
+    );
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.
@@ -761,7 +837,7 @@ export class CodeQLCliServer implements Disposable {
       logger,
     }: {
       cancellationToken?: CancellationToken;
-      logger?: Logger;
+      logger?: BaseLogger;
     },
   ): AsyncGenerator<TestCompleted, void, unknown> {
     const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
@@ -1006,6 +1082,7 @@ export class CodeQLCliServer implements Disposable {
     resultsPath: string,
     interpretedResultsPath: string,
     sourceInfo?: SourceInfo,
+    args?: string[],
   ): Promise<sarif.Log> {
     const additionalArgs = [
       // TODO: This flag means that we don't group interpreted results
@@ -1013,6 +1090,7 @@ export class CodeQLCliServer implements Disposable {
       // interpretation with and without this flag, or do some
       // grouping client-side.
       "--no-group-results",
+      ...(args ?? []),
     ];
 
     await this.runInterpretCommand(
@@ -1163,24 +1241,55 @@ export class CodeQLCliServer implements Disposable {
 
   /**
    * Gets information about available qlpacks
-   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
-   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
-   *   the default CLI search path is used.
+   * @param additionalPacks A list of directories to search for qlpacks.
+   * @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
+   *    be returned. If false, all packs will be returned.
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
-  resolveQlpacks(
+  async resolveQlpacks(
     additionalPacks: string[],
-    searchPath?: string[],
+    extensionPacksOnly = false,
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
-    if (searchPath?.length) {
-      args.push("--search-path", join(...searchPath));
+    if (extensionPacksOnly) {
+      if (!(await this.cliConstraints.supportsQlpacksKind())) {
+        void this.logger.log(
+          "Warning: Running with extension packs is only supported by CodeQL CLI v2.12.3 or later.",
+        );
+        return {};
+      }
+      args.push("--kind", "extension", "--no-recursive");
     }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(
       ["resolve", "qlpacks"],
       args,
-      "Resolving qlpack information",
+      `Resolving qlpack information${
+        extensionPacksOnly ? " (extension packs only)" : ""
+      }`,
+    );
+  }
+
+  /**
+   * Gets information about available extensions
+   * @param suite The suite to resolve.
+   * @param additionalPacks A list of directories to search for qlpacks.
+   * @returns An object containing the list of models and extensions
+   */
+  async resolveExtensions(
+    suite: string,
+    additionalPacks: string[],
+  ): Promise<ResolveExtensionsResult> {
+    const args = this.getAdditionalPacksArg(additionalPacks);
+    args.push(suite);
+
+    return this.runJsonCodeQlCliCommand<ResolveExtensionsResult>(
+      ["resolve", "extensions"],
+      args,
+      "Resolving extensions",
+      {
+        addFormat: false,
+      },
     );
   }
 
@@ -1276,11 +1385,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  async packInstall(dir: string, forceUpdate = false) {
+  async packInstall(
+    dir: string,
+    { forceUpdate = false, workspaceFolders = [] as string[] } = {},
+  ) {
     const args = [dir];
     if (forceUpdate) {
       args.push("--mode", "update");
     }
+    if (workspaceFolders?.length > 0) {
+      if (await this.cliConstraints.supportsAdditionalPacksInstall()) {
+        args.push(
+          // Allow prerelease packs from the ql submodule.
+          "--allow-prerelease",
+          // Allow the use of --additional-packs argument without issueing a warning
+          "--no-strict-mode",
+          ...this.getAdditionalPacksArg(workspaceFolders),
+        );
+      }
+    }
     return this.runJsonCodeQlCliCommandWithAuthentication(
       ["pack", "install"],
       args,
@@ -1311,21 +1434,13 @@ export class CodeQLCliServer implements Disposable {
 
   async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
     const args = includeQueries ? [dir] : ["--no-include-queries", dir];
-    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
-    // previous versions return a list of packs.
-    const results: { paths: string[] } | string[] =
-      await this.runJsonCodeQlCliCommand(
-        ["pack", "packlist"],
-        args,
-        "Generating the pack list",
-      );
+    const results: { paths: string[] } = await this.runJsonCodeQlCliCommand(
+      ["pack", "packlist"],
+      args,
+      "Generating the pack list",
+    );
 
-    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
-    if ("paths" in results) {
-      return results.paths;
-    } else {
-      return results;
-    }
+    return results.paths;
   }
 
   async packResolveDependencies(
@@ -1349,17 +1464,49 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  async generateExtensiblePredicateMetadata(
+    packRoot: string,
+  ): Promise<GenerateExtensiblePredicateMetadataResult> {
+    return await this.runJsonCodeQlCliCommand(
+      ["generate", "extensible-predicate-metadata"],
+      [packRoot],
+      "Generating extensible predicate metadata",
+      { addFormat: false },
+    );
+  }
+
   public async getVersion() {
     if (!this._version) {
-      this._version = this.refreshVersion();
-      // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
-      await commands.executeCommand(
-        "setContext",
-        "codeql.supportsEvalLog",
-        await this.cliConstraints.supportsPerQueryEvalLog(),
-      );
+      try {
+        const newVersion = await this.refreshVersion();
+        this._version = newVersion;
+        this._versionChangedListeners.forEach((listener) =>
+          listener(newVersion),
+        );
+
+        // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
+        await this.app.commands.execute(
+          "setContext",
+          "codeql.supportsQuickEvalCount",
+          newVersion.compare(
+            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+          ) >= 0,
+        );
+      } catch (e) {
+        this._versionChangedListeners.forEach((listener) =>
+          listener(undefined),
+        );
+        throw e;
+      }
     }
-    return await this._version;
+    return this._version;
+  }
+
+  public addVersionChangedListener(listener: VersionChangedListener) {
+    if (this._version) {
+      listener(this._version);
+    }
+    this._versionChangedListeners.push(listener);
   }
 
   private async refreshVersion() {
@@ -1380,6 +1527,17 @@ export class CodeQLCliServer implements Disposable {
   private getAdditionalPacksArg(paths: string[]): string[] {
     return paths.length ? ["--additional-packs", paths.join(delimiter)] : [];
   }
+
+  public async useExtensionPacks(): Promise<boolean> {
+    return (
+      this.cliConfig.useExtensionPacks &&
+      (await this.cliConstraints.supportsQlpacksKind())
+    );
+  }
+
+  public async setUseExtensionPacks(useExtensionPacks: boolean) {
+    await this.cliConfig.setUseExtensionPacks(useExtensionPacks);
+  }
 }
 
 /**
@@ -1423,10 +1581,23 @@ export function spawnServer(
     );
   }
 
+  let lastStdout: any = undefined;
+  child.stdout!.on("data", (data) => {
+    lastStdout = data;
+  });
   // Set up event listeners.
-  child.on("close", (code) =>
-    logger.log(`Child process exited with code ${code}`),
-  );
+  child.on("close", async (code, signal) => {
+    if (code !== null)
+      void logger.log(`Child process exited with code ${code}`);
+    if (signal)
+      void logger.log(
+        `Child process exited due to receipt of signal ${signal}`,
+      );
+    // If the process exited abnormally, log the last stdout message,
+    // It may be from the jvm.
+    if (code !== 0 && lastStdout !== undefined)
+      void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+  });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
     child.stdout!.on("data", stdoutListener);
@@ -1473,155 +1644,49 @@ export async function runCodeQlCliCommand(
     return result.stdout;
   } catch (err) {
     throw new Error(
-      `${description} failed: ${(err as any).stderr || getErrorMessage(err)}`,
+      `${description} failed: ${getChildProcessErrorMessage(err)}`,
     );
   }
 }
 
-/**
- * Buffer to hold state used when splitting a text stream into lines.
- */
-class SplitBuffer {
-  private readonly decoder = new StringDecoder("utf8");
-  private readonly maxSeparatorLength: number;
-  private buffer = "";
-  private searchIndex = 0;
-
-  constructor(private readonly separators: readonly string[]) {
-    this.maxSeparatorLength = separators
-      .map((s) => s.length)
-      .reduce((a, b) => Math.max(a, b), 0);
-  }
-
-  /**
-   * Append new text data to the buffer.
-   * @param chunk The chunk of data to append.
-   */
-  public addChunk(chunk: Buffer): void {
-    this.buffer += this.decoder.write(chunk);
-  }
-
-  /**
-   * Signal that the end of the input stream has been reached.
-   */
-  public end(): void {
-    this.buffer += this.decoder.end();
-    this.buffer += this.separators[0]; // Append a separator to the end to ensure the last line is returned.
-  }
-
-  /**
-   * A version of startsWith that isn't overriden by a broken version of ms-python.
-   *
-   * The definition comes from
-   * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
-   * which is CC0/public domain
-   *
-   * See https://github.com/github/vscode-codeql/issues/802 for more context as to why we need it.
-   */
-  private static startsWith(
-    s: string,
-    searchString: string,
-    position: number,
-  ): boolean {
-    const pos = position > 0 ? position | 0 : 0;
-    return s.substring(pos, pos + searchString.length) === searchString;
-  }
-
-  /**
-   * Extract the next full line from the buffer, if one is available.
-   * @returns The text of the next available full line (without the separator), or `undefined` if no
-   * line is available.
-   */
-  public getNextLine(): string | undefined {
-    while (this.searchIndex <= this.buffer.length - this.maxSeparatorLength) {
-      for (const separator of this.separators) {
-        if (SplitBuffer.startsWith(this.buffer, separator, this.searchIndex)) {
-          const line = this.buffer.slice(0, this.searchIndex);
-          this.buffer = this.buffer.slice(this.searchIndex + separator.length);
-          this.searchIndex = 0;
-          return line;
-        }
-      }
-      this.searchIndex++;
-    }
-
-    return undefined;
-  }
-}
-
-/**
- * Splits a text stream into lines based on a list of valid line separators.
- * @param stream The text stream to split. This stream will be fully consumed.
- * @param separators The list of strings that act as line separators.
- * @returns A sequence of lines (not including separators).
- */
-async function* splitStreamAtSeparators(
-  stream: Readable,
-  separators: string[],
-): AsyncGenerator<string, void, unknown> {
-  const buffer = new SplitBuffer(separators);
-  for await (const chunk of stream) {
-    buffer.addChunk(chunk);
-    let line: string | undefined;
-    do {
-      line = buffer.getNextLine();
-      if (line !== undefined) {
-        yield line;
-      }
-    } while (line !== undefined);
-  }
-  buffer.end();
-  let line: string | undefined;
-  do {
-    line = buffer.getNextLine();
-    if (line !== undefined) {
-      yield line;
-    }
-  } while (line !== undefined);
-}
-
-/**
- *  Standard line endings for splitting human-readable text.
- */
-const lineEndings = ["\r\n", "\r", "\n"];
-
 /**
  * Log a text stream to a `Logger` interface.
  * @param stream The stream to log.
  * @param logger The logger that will consume the stream output.
  */
-async function logStream(stream: Readable, logger: Logger): Promise<void> {
-  for await (const line of splitStreamAtSeparators(stream, lineEndings)) {
+async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
+  for await (const line of splitStreamAtSeparators(stream, LINE_ENDINGS)) {
     // Await the result of log here in order to ensure the logs are written in the correct order.
     await logger.log(line);
   }
 }
 
-export function shouldDebugIdeServer() {
+function isEnvTrue(name: string): boolean {
   return (
-    "IDE_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.IDE_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
+    name in process.env &&
+    process.env[name] !== "0" &&
+    // Use en-US since we expect the value to be either "false" or "FALSE", not a localized version.
+    process.env[name]?.toLocaleLowerCase("en-US") !== "false"
   );
 }
 
+export function shouldDebugIdeServer() {
+  return isEnvTrue("IDE_SERVER_JAVA_DEBUG");
+}
+
 export function shouldDebugQueryServer() {
-  return (
-    "QUERY_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.QUERY_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("QUERY_SERVER_JAVA_DEBUG");
 }
 
 export function shouldDebugCliServer() {
-  return (
-    "CLI_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.CLI_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("CLI_SERVER_JAVA_DEBUG");
 }
 
 export class CliVersionConstraint {
+  // The oldest version of the CLI that we support. This is used to determine
+  // whether to show a warning about the CLI being too old on startup.
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.9.4");
+
   /**
    * CLI version where building QLX packs for remote queries is supported.
    * (The options were _accepted_ by a few earlier versions, but only from
@@ -1637,21 +1702,9 @@ export class CliVersionConstraint {
   );
 
   /**
-   * CLI version where the `--evaluator-log` and related options to the query server were introduced,
-   * on a per-query server basis.
+   * CLI version where the `resolve extensions` subcommand exists.
    */
-  public static CLI_VERSION_WITH_STRUCTURED_EVAL_LOG = new SemVer("2.8.2");
-
-  /**
-   * CLI version that supports rotating structured logs to produce one per query.
-   *
-   * Note that 2.8.4 supports generating the evaluation logs and summaries,
-   * but 2.9.0 includes a new option to produce the end-of-query summary logs to
-   * the query server console. For simplicity we gate all features behind 2.9.0,
-   * but if a user is tied to the 2.8 release, we can enable evaluator logs
-   * and summaries for them.
-   */
-  public static CLI_VERSION_WITH_PER_QUERY_EVAL_LOG = new SemVer("2.9.0");
+  public static CLI_VERSION_WITH_RESOLVE_EXTENSIONS = new SemVer("2.10.2");
 
   /**
    * CLI version that supports the `--sourcemap` option for log generation.
@@ -1668,6 +1721,40 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_WORKSPACE_RFERENCES = new SemVer("2.11.3");
 
+  /**
+   * CLI version that supports the `--kind` option for the `resolve qlpacks` command.
+   */
+  public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
+
+  /**
+   * CLI version that supports the `--additional-packs` option for the `pack install` command.
+   */
+  public static CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL = new SemVer(
+    "2.12.4",
+  );
+
+  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
+
+  /**
+   * CLI version where the query server supports quick-eval count mode.
+   */
+  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
+
+  /**
+   * CLI version where the `generate extensible-predicate-metadata`
+   * command was implemented.
+   */
+  public static CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA = new SemVer(
+    "2.14.3",
+  );
+
+  /**
+   * CLI version where the langauge server supports visisbility change notifications.
+   */
+  public static CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS = new SemVer(
+    "2.14.0",
+  );
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1686,15 +1773,9 @@ export class CliVersionConstraint {
     );
   }
 
-  async supportsStructuredEvalLog() {
+  async supportsResolveExtensions() {
     return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_STRUCTURED_EVAL_LOG,
-    );
-  }
-
-  async supportsPerQueryEvalLog() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+      CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS,
     );
   }
 
@@ -1725,4 +1806,38 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_WORKSPACE_RFERENCES,
     );
   }
+
+  async supportsQlpacksKind() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
+    );
+  }
+
+  async supportsAdditionalPacksInstall() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
+    );
+  }
+
+  async usesGlobalCompilationCache() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
+  }
+
+  async supportsVisibilityNotifications() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
+    );
+  }
+
+  async supportsQuickEvalCount() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+    );
+  }
+
+  async supportsGenerateExtensiblePredicateMetadata() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -3,24 +3,29 @@ import { pathExists, mkdtemp, createWriteStream, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
 import * as semver from "semver";
-import { parse } from "url";
+import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
-import { DistributionConfig } from "./config";
-import {
-  InvocationRateLimiter,
-  InvocationRateLimiterResultKind,
-  showAndLogErrorMessage,
-  showAndLogWarningMessage,
-} from "./helpers";
-import { extLogger } from "./common";
+import { DistributionConfig } from "../config";
+import { extLogger } from "../common/logging/vscode";
 import { getCodeQlCliVersion } from "./cli-version";
-import { ProgressCallback, reportStreamProgress } from "./commandRunner";
+import {
+  ProgressCallback,
+  reportStreamProgress,
+} from "../common/vscode/progress";
 import {
   codeQlLauncherName,
   deprecatedCodeQlLauncherName,
   extractZipArchive,
   getRequiredAssetName,
-} from "./pure/distribution";
+} from "../common/distribution";
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+} from "../common/invocation-rate-limiter";
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+} from "../common/logging";
 
 /**
  * distribution.ts
@@ -73,7 +78,7 @@ export class DistributionManager implements DistributionProvider {
         extensionContext,
       );
     this.updateCheckRateLimiter = new InvocationRateLimiter(
-      extensionContext,
+      extensionContext.globalState,
       "extensionSpecificDistributionUpdateCheck",
       () =>
         this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),
@@ -155,6 +160,7 @@ export class DistributionManager implements DistributionProvider {
     if (this.config.customCodeQlPath) {
       if (!(await pathExists(this.config.customCodeQlPath))) {
         void showAndLogErrorMessage(
+          extLogger,
           `The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
             "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
             "that a CodeQL executable exists at the specified path or remove the setting.",
@@ -215,6 +221,9 @@ export class DistributionManager implements DistributionProvider {
     minSecondsSinceLastUpdateCheck: number,
   ): Promise<DistributionUpdateCheckResult> {
     const distribution = await this.getDistributionWithoutVersionCheck();
+    if (distribution === undefined) {
+      minSecondsSinceLastUpdateCheck = 0;
+    }
     const extensionManagedCodeQlPath =
       await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
@@ -499,7 +508,7 @@ class ExtensionSpecificDistributionManager {
         0,
       ) || "";
     return join(
-      this.extensionContext.globalStoragePath,
+      this.extensionContext.globalStorageUri.fsPath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
         distributionFolderIndex,
     );
@@ -664,7 +673,7 @@ export class ReleasesApiConsumer {
       redirectUrl &&
       redirectCount < ReleasesApiConsumer._maxRedirects
     ) {
-      const parsedRedirectUrl = parse(redirectUrl);
+      const parsedRedirectUrl = new URL(redirectUrl);
       if (parsedRedirectUrl.protocol !== "https:") {
         throw new Error("Encountered a non-https redirect, rejecting");
       }
@@ -709,7 +718,7 @@ export enum DistributionKind {
   PathEnvironmentVariable,
 }
 
-export interface Distribution {
+interface Distribution {
   codeQlPath: string;
   kind: DistributionKind;
 }
@@ -767,22 +776,22 @@ type DistributionUpdateCheckResult =
   | InvalidLocationResult
   | UpdateAvailableResult;
 
-export interface AlreadyCheckedRecentlyResult {
+interface AlreadyCheckedRecentlyResult {
   kind: DistributionUpdateCheckResultKind.AlreadyCheckedRecentlyResult;
 }
 
-export interface AlreadyUpToDateResult {
+interface AlreadyUpToDateResult {
   kind: DistributionUpdateCheckResultKind.AlreadyUpToDate;
 }
 
 /**
  * The distribution could not be installed or updated because it is not managed by the extension.
  */
-export interface InvalidLocationResult {
+interface InvalidLocationResult {
   kind: DistributionUpdateCheckResultKind.InvalidLocation;
 }
 
-export interface UpdateAvailableResult {
+interface UpdateAvailableResult {
   kind: DistributionUpdateCheckResultKind.UpdateAvailable;
   updatedRelease: Release;
 }
@@ -844,6 +853,7 @@ export async function getExecutableFromDirectory(
 
 function warnDeprecatedLauncher() {
   void showAndLogWarningMessage(
+    extLogger,
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
       `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`,
   );
@@ -852,7 +862,7 @@ function warnDeprecatedLauncher() {
 /**
  * A release on GitHub.
  */
-export interface Release {
+interface Release {
   assets: ReleaseAsset[];
 
   /**
@@ -874,7 +884,7 @@ export interface Release {
 /**
  * An asset corresponding to a release on GitHub.
  */
-export interface ReleaseAsset {
+interface ReleaseAsset {
   /**
    * The id associated with the asset on GitHub.
    */
@@ -944,7 +954,10 @@ export interface GithubReleaseAsset {
 }
 
 export class GithubApiError extends Error {
-  constructor(public status: number, public body: string) {
+  constructor(
+    public status: number,
+    public body: string,
+  ) {
     super(`API call failed with status code ${status}, body: ${body}`);
   }
 }

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -0,0 +1,80 @@
+import { CodeQLCliServer } from "./cli";
+import { Uri, window } from "vscode";
+import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { UserCancellationException } from "../common/vscode/progress";
+import { showAndLogErrorMessage } from "../common/logging";
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined,
+): Promise<QueryLanguage | undefined> {
+  const uri = queryUri || window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(
+        getOnDiskWorkspaceFolders(),
+        uri,
+      );
+      const language = Object.keys(queryInfo.byLanguage)[0];
+      void extLogger.log(`Detected query language: ${language}`);
+
+      if (isQueryLanguage(language)) {
+        return language;
+      }
+
+      void extLogger.log(
+        "Query language is unsupported. Select language manually.",
+      );
+    } catch (e) {
+      void extLogger.log(
+        "Could not autodetect query language. Select language manually.",
+      );
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(
+  cliServer: CodeQLCliServer,
+  throwOnEmpty = true,
+): Promise<QueryLanguage | undefined> {
+  const language = await window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    {
+      placeHolder: "Select target language for your query",
+      ignoreFocusOut: true,
+    },
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException("Cancelled.");
+    } else {
+      void showAndLogErrorMessage(
+        extLogger,
+        "Language not found. Language must be specified manually.",
+      );
+    }
+    return undefined;
+  }
+
+  if (!isQueryLanguage(language)) {
+    void showAndLogErrorMessage(
+      extLogger,
+      `Language '${language}' is not supported. Only languages ${Object.values(
+        QueryLanguage,
+      ).join(", ")} are supported.`,
+    );
+    return undefined;
+  }
+
+  return language;
+}

extensions/ql-vscode/src/codeql-cli/query-metadata.ts

@@ -0,0 +1,22 @@
+import { CodeQLCliServer } from "./cli";
+import { QueryMetadata } from "../common/interface-types";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Gets metadata for a query, if it exists.
+ * @param cliServer The CLI server.
+ * @param queryPath The path to the query.
+ * @returns A promise that resolves to the query metadata, if available.
+ */
+export async function tryGetQueryMetadata(
+  cliServer: CodeQLCliServer,
+  queryPath: string,
+): Promise<QueryMetadata | undefined> {
+  try {
+    return await cliServer.resolveMetadata(queryPath);
+  } catch (e) {
+    // Ignore errors and provide no metadata.
+    void extLogger.log(`Couldn't resolve metadata for ${queryPath}: ${e}`);
+    return;
+  }
+}

extensions/ql-vscode/src/commandRunner.ts

@@ -1,268 +0,0 @@
-import {
-  CancellationToken,
-  ProgressOptions,
-  window as Window,
-  commands,
-  Disposable,
-  ProgressLocation,
-} from "vscode";
-import {
-  showAndLogExceptionWithTelemetry,
-  showAndLogWarningMessage,
-} from "./helpers";
-import { extLogger } from "./common";
-import { asError, getErrorMessage, getErrorStack } from "./pure/helpers-pure";
-import { telemetryListener } from "./telemetry";
-import { redactableError } from "./pure/errors";
-
-export class UserCancellationException extends Error {
-  /**
-   * @param message The error message
-   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
-   */
-  constructor(message?: string, public readonly silent = false) {
-    super(message);
-  }
-}
-
-export interface ProgressUpdate {
-  /**
-   * The current step
-   */
-  step: number;
-  /**
-   * The maximum step. This *should* be constant for a single job.
-   */
-  maxStep: number;
-  /**
-   * The current progress message
-   */
-  message: string;
-}
-
-export type ProgressCallback = (p: ProgressUpdate) => void;
-
-/**
- * A task that handles command invocations from `commandRunner`
- * and includes a progress monitor.
- *
- *
- * Arguments passed to the command handler are passed along,
- * untouched to this `ProgressTask` instance.
- *
- * @param progress a progress handler function. Call this
- * function with a `ProgressUpdate` instance in order to
- * denote some progress being achieved on this task.
- * @param token a cencellation token
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-export type ProgressTask<R> = (
-  progress: ProgressCallback,
-  token: CancellationToken,
-  ...args: any[]
-) => Thenable<R>;
-
-/**
- * A task that handles command invocations from `commandRunner`.
- * Arguments passed to the command handler are passed along,
- * untouched to this `NoProgressTask` instance.
- *
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-type NoProgressTask = (...args: any[]) => Promise<any>;
-
-/**
- * This mediates between the kind of progress callbacks we want to
- * write (where we *set* current progress position and give
- * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%).
- *
- * Where possible, the `commandRunner` function below should be used
- * instead of this function. The commandRunner is meant for wrapping
- * top-level commands and provides error handling and other support
- * automatically.
- *
- * Only use this function if you need a progress monitor and the
- * control flow does not always come from a command (eg- during
- * extension activation, or from an internal language server
- * request).
- */
-export function withProgress<R>(
-  options: ProgressOptions,
-  task: ProgressTask<R>,
-  ...args: any[]
-): Thenable<R> {
-  let progressAchieved = 0;
-  return Window.withProgress(options, (progress, token) => {
-    return task(
-      (p) => {
-        const { message, step, maxStep } = p;
-        const increment = (100 * (step - progressAchieved)) / maxStep;
-        progressAchieved = step;
-        progress.report({ message, increment });
-      },
-      token,
-      ...args,
-    );
-  });
-}
-
-/**
- * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
- *
- * In this variant of the command runner, no progress monitor is used.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task.
- */
-export function commandRunner(
-  commandId: string,
-  task: NoProgressTask,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const startTime = Date.now();
-    let error: Error | undefined;
-
-    try {
-      return await task(...args);
-    } catch (e) {
-      error = asError(e);
-      const errorMessage = redactableError(error)`${
-        getErrorMessage(e) || e
-      } (${commandId})`;
-      const errorStack = getErrorStack(e);
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          void extLogger.log(errorMessage.fullMessage);
-        } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage);
-        }
-      } else {
-        // Include the full stack in the error log only.
-        const fullMessage = errorStack
-          ? `${errorMessage.fullMessage}\n${errorStack}`
-          : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      }
-      return undefined;
-    } finally {
-      const executionTime = Date.now() - startTime;
-      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
-    }
-  });
-}
-
-/**
- * A generic wrapper for command registration.  This wrapper adds uniform error handling,
- * progress monitoring, and cancellation for commands.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task after the progress callback
- * and cancellation token.
- * @param progressOptions Progress options to be sent to the progress monitor.
- */
-export function commandRunnerWithProgress<R>(
-  commandId: string,
-  task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>,
-  outputLogger = extLogger,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const startTime = Date.now();
-    let error: Error | undefined;
-    const progressOptionsWithDefaults = {
-      location: ProgressLocation.Notification,
-      ...progressOptions,
-    };
-    try {
-      return await withProgress(progressOptionsWithDefaults, task, ...args);
-    } catch (e) {
-      error = asError(e);
-      const errorMessage = redactableError`${
-        getErrorMessage(e) || e
-      } (${commandId})`;
-      const errorStack = getErrorStack(e);
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          void outputLogger.log(errorMessage.fullMessage);
-        } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage, {
-            outputLogger,
-          });
-        }
-      } else {
-        // Include the full stack in the error log only.
-        const fullMessage = errorStack
-          ? `${errorMessage.fullMessage}\n${errorStack}`
-          : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          outputLogger,
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      }
-      return undefined;
-    } finally {
-      const executionTime = Date.now() - startTime;
-      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
-    }
-  });
-}
-
-/**
- * Displays a progress monitor that indicates how much progess has been made
- * reading from a stream.
- *
- * @param readable The stream to read progress from
- * @param messagePrefix A prefix for displaying the message
- * @param totalNumBytes Total number of bytes in this stream
- * @param progress The progress callback used to set messages
- */
-export function reportStreamProgress(
-  readable: NodeJS.ReadableStream,
-  messagePrefix: string,
-  totalNumBytes?: number,
-  progress?: ProgressCallback,
-) {
-  if (progress && totalNumBytes) {
-    let numBytesDownloaded = 0;
-    const bytesToDisplayMB = (numBytes: number): string =>
-      `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-    const updateProgress = () => {
-      progress({
-        step: numBytesDownloaded,
-        maxStep: totalNumBytes,
-        message: `${messagePrefix} [${bytesToDisplayMB(
-          numBytesDownloaded,
-        )} of ${bytesToDisplayMB(totalNumBytes)}]`,
-      });
-    };
-
-    // Display the progress straight away rather than waiting for the first chunk.
-    updateProgress();
-
-    readable.on("data", (data) => {
-      numBytesDownloaded += data.length;
-      updateProgress();
-    });
-  } else if (progress) {
-    progress({
-      step: 1,
-      maxStep: 2,
-      message: `${messagePrefix} (Size unknown)`,
-    });
-  }
-}

extensions/ql-vscode/src/common/app.ts

@@ -1,20 +1,24 @@
 import { Credentials } from "./authentication";
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 import { AppEventEmitter } from "./events";
-import { Logger } from "./logging";
+import { NotificationLogger } from "./logging";
 import { Memento } from "./memento";
+import { AppCommandManager } from "./commands";
+import { AppTelemetry } from "./telemetry";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
-  executeCommand(command: string, ...args: any): Thenable<void>;
   readonly mode: AppMode;
-  readonly logger: Logger;
+  readonly logger: NotificationLogger;
+  readonly telemetry?: AppTelemetry;
   readonly subscriptions: Disposable[];
   readonly extensionPath: string;
   readonly globalStoragePath: string;
   readonly workspaceStoragePath?: string;
   readonly workspaceState: Memento;
   readonly credentials: Credentials;
+  readonly commands: AppCommandManager;
+  readonly environment: EnvironmentContext;
 }
 
 export enum AppMode {
@@ -22,3 +26,7 @@ export enum AppMode {
   Development = 2,
   Test = 3,
 }
+
+export interface EnvironmentContext {
+  language: string;
+}

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -1,5 +1,3 @@
-export const PAGE_SIZE = 1000;
-
 /**
  * The single-character codes used in the bqrs format for the the kind
  * of a result column. This namespace is intentionally not an enum, see
@@ -15,7 +13,7 @@ export namespace ColumnKindCode {
   export const ENTITY = "e";
 }
 
-export type ColumnKind =
+type ColumnKind =
   | typeof ColumnKindCode.FLOAT
   | typeof ColumnKindCode.INTEGER
   | typeof ColumnKindCode.STRING
@@ -46,7 +44,7 @@ export function getResultSetSchema(
   }
   return undefined;
 }
-export interface PaginationInfo {
+interface PaginationInfo {
   "step-size": number;
   offsets: number[];
 }
@@ -115,7 +113,7 @@ export type BqrsKind =
   | "Entity";
 
 interface BqrsColumn {
-  name: string;
+  name?: string;
   kind: BqrsKind;
 }
 export interface DecodedBqrsChunk {

extensions/ql-vscode/src/common/bqrs-utils.ts

@@ -4,7 +4,7 @@ import {
   LineColumnLocation,
   WholeFileLocation,
 } from "./bqrs-cli-types";
-import { createRemoteFileRef } from "./location-link-utils";
+import { createRemoteFileRef } from "../common/location-link-utils";
 
 /**
  * The CodeQL filesystem libraries use this pattern in `getURL()` predicates
@@ -142,5 +142,7 @@ export function tryGetRemoteLocation(
     fileLink,
     resolvableLocation.startLine,
     resolvableLocation.endLine,
+    resolvableLocation.startColumn,
+    resolvableLocation.endColumn,
   );
 }

extensions/ql-vscode/src/common/commands.ts

@@ -0,0 +1,384 @@
+import type { CommandManager } from "../packages/commands";
+import type { Uri, Range, TextDocumentShowOptions } from "vscode";
+import type { AstItem } from "../language-support";
+import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
+import type { DatabaseItem } from "../databases/local-databases";
+import type { QueryHistoryInfo } from "../query-history/query-history-info";
+import type { TestTreeNode } from "../query-testing/test-tree-node";
+import type {
+  VariantAnalysis,
+  VariantAnalysisScannedRepository,
+  VariantAnalysisScannedRepositoryResult,
+} from "../variant-analysis/shared/variant-analysis";
+import type { QLDebugConfiguration } from "../debugger/debug-configuration";
+import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import type { Method, Usage } from "../model-editor/method";
+
+// A command function matching the signature that VS Code calls when
+// a command is invoked from a context menu on a TreeView with
+// canSelectMany set to true.
+//
+// singleItem will always be defined and corresponds to the item that
+// was hovered or right-clicked. If precisely one item was selected then
+// multiSelect will be undefined. If more than one item is selected then
+// multiSelect will contain all selected items, including singleItem.
+export type TreeViewContextMultiSelectionCommandFunction<Item> = (
+  singleItem: Item,
+  multiSelect: Item[] | undefined,
+) => Promise<void>;
+
+// A command function matching the signature that VS Code calls when
+// a command is invoked from a context menu on a TreeView with
+// canSelectMany set to false.
+//
+// It is guaranteed that precisely one item will be selected.
+export type TreeViewContextSingleSelectionCommandFunction<Item> = (
+  singleItem: Item,
+) => Promise<void>;
+
+// A command function matching the signature that VS Code calls when
+// a command is invoked from a context menu on the file explorer.
+//
+// singleItem corresponds to the item that was right-clicked.
+// multiSelect will always been defined and non-empty and contains
+// all selected items, including singleItem.
+export type ExplorerSelectionCommandFunction<Item> = (
+  singleItem: Item,
+  multiSelect: Item[],
+) => Promise<void>;
+
+/**
+ * Contains type definitions for all commands used by the extension.
+ *
+ * To add a new command first define its type here, then provide
+ * the implementation in the corresponding `getCommands` function.
+ */
+
+// Builtin commands where the implementation is provided by VS Code and not by this extension.
+// See https://code.visualstudio.com/api/references/commands
+type BuiltInVsCodeCommands = {
+  // The codeQLDatabases.focus command is provided by VS Code because we've registered the custom view
+  "codeQLDatabases.focus": () => Promise<void>;
+  "markdown.showPreviewToSide": (uri: Uri) => Promise<void>;
+  "workbench.action.closeActiveEditor": () => Promise<void>;
+  revealFileInOS: (uri: Uri) => Promise<void>;
+  setContext: (
+    key: `${"codeql" | "codeQL"}${string}`,
+    value: unknown,
+  ) => Promise<void>;
+  "workbench.action.reloadWindow": () => Promise<void>;
+  "vscode.diff": (
+    leftSideResource: Uri,
+    rightSideResource: Uri,
+    title?: string,
+    columnOrOptions?: TextDocumentShowOptions,
+  ) => Promise<void>;
+  "vscode.open": (uri: Uri) => Promise<void>;
+  "vscode.openFolder": (uri: Uri) => Promise<void>;
+  revealInExplorer: (uri: Uri) => Promise<void>;
+  // We type the `config` property specifically as a CodeQL debug configuration, since that's the
+  // only kinds we specify anyway.
+  "workbench.action.debug.start": (options?: {
+    config?: Partial<QLDebugConfiguration>;
+    noDebug?: boolean;
+  }) => Promise<void>;
+  "workbench.action.debug.stepInto": () => Promise<void>;
+  "workbench.action.debug.stepOver": () => Promise<void>;
+  "workbench.action.debug.stepOut": () => Promise<void>;
+};
+
+// Commands that are available before the extension is fully activated.
+// These commands are *not* registered using the command manager, but can
+// be invoked using the command manager.
+export type PreActivationCommands = {
+  "codeQL.checkForUpdatesToCLI": () => Promise<void>;
+};
+
+// Base commands not tied directly to a module like e.g. variant analysis.
+export type BaseCommands = {
+  "codeQL.openDocumentation": () => Promise<void>;
+  "codeQL.showLogs": () => Promise<void>;
+  "codeQL.authenticateToGitHub": () => Promise<void>;
+
+  "codeQL.copyVersion": () => Promise<void>;
+  "codeQL.restartQueryServer": () => Promise<void>;
+  "codeQL.restartQueryServerOnConfigChange": () => Promise<void>;
+  "codeQL.restartLegacyQueryServerOnConfigChange": () => Promise<void>;
+  "codeQL.restartQueryServerOnExternalConfigChange": () => Promise<void>;
+};
+
+// Commands used when working with queries in the editor
+export type QueryEditorCommands = {
+  "codeQL.openReferencedFile": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.openReferencedFileContextEditor": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
+  "codeQL.openReferencedFileContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
+  "codeQL.previewQueryHelp": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextEditor": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
+};
+
+// Commands used for running local queries
+export type LocalQueryCommands = {
+  "codeQL.runQuery": (uri?: Uri) => Promise<void>;
+  "codeQL.runQueryContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQL.runQueryOnMultipleDatabases": (uri?: Uri) => Promise<void>;
+  "codeQL.runQueryOnMultipleDatabasesContextEditor": (
+    uri?: Uri,
+  ) => Promise<void>;
+  "codeQLQueries.runLocalQueryFromQueriesPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueryContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesFromPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.createQuery": () => Promise<void>;
+  "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise<void>;
+  "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
+  "codeQL.quickEval": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalCount": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
+  "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
+  "codeQL.quickQuery": () => Promise<void>;
+  "codeQL.getCurrentQuery": () => Promise<string>;
+  "codeQL.createQuery": () => Promise<void>;
+};
+
+// Debugger commands
+export type DebuggerCommands = {
+  "codeQL.debugQuery": (uri: Uri | undefined) => Promise<void>;
+  "codeQL.debugQueryContextEditor": (uri: Uri) => Promise<void>;
+  "codeQL.startDebuggingSelection": () => Promise<void>;
+  "codeQL.startDebuggingSelectionContextEditor": () => Promise<void>;
+  "codeQL.continueDebuggingSelection": () => Promise<void>;
+  "codeQL.continueDebuggingSelectionContextEditor": () => Promise<void>;
+};
+
+export type ResultsViewCommands = {
+  "codeQLQueryResults.up": () => Promise<void>;
+  "codeQLQueryResults.down": () => Promise<void>;
+  "codeQLQueryResults.left": () => Promise<void>;
+  "codeQLQueryResults.right": () => Promise<void>;
+  "codeQLQueryResults.nextPathStep": () => Promise<void>;
+  "codeQLQueryResults.previousPathStep": () => Promise<void>;
+};
+
+// Commands used for the query history panel
+export type QueryHistoryCommands = {
+  // Commands in the "navigation" group
+  "codeQLQueryHistory.sortByName": () => Promise<void>;
+  "codeQLQueryHistory.sortByDate": () => Promise<void>;
+  "codeQLQueryHistory.sortByCount": () => Promise<void>;
+
+  // Commands in the context menu or in the hover menu
+  "codeQLQueryHistory.openQueryContextMenu": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.removeHistoryItemContextMenu": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showQueryLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showQueryText": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.openQueryDirectory": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.cancel": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.exportResults": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewCsvResults": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewCsvAlerts": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewSarifAlerts": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewDil": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.itemClicked": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.openOnGithub": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.copyRepoList": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+
+  // Commands in the command palette
+  "codeQL.exportSelectedVariantAnalysisResults": () => Promise<void>;
+};
+
+// Commands used for the local databases panel
+export type LocalDatabasesCommands = {
+  // Command palette commands
+  "codeQL.chooseDatabaseFolder": () => Promise<void>;
+  "codeQL.chooseDatabaseArchive": () => Promise<void>;
+  "codeQL.chooseDatabaseInternet": () => Promise<void>;
+  "codeQL.chooseDatabaseGithub": () => Promise<void>;
+  "codeQL.upgradeCurrentDatabase": () => Promise<void>;
+  "codeQL.clearCache": () => Promise<void>;
+
+  // Explorer context menu
+  "codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
+
+  // Database panel view title commands
+  "codeQLDatabases.chooseDatabaseFolder": () => Promise<void>;
+  "codeQLDatabases.chooseDatabaseArchive": () => Promise<void>;
+  "codeQLDatabases.chooseDatabaseInternet": () => Promise<void>;
+  "codeQLDatabases.chooseDatabaseGithub": () => Promise<void>;
+  "codeQLDatabases.sortByName": () => Promise<void>;
+  "codeQLDatabases.sortByDateAdded": () => Promise<void>;
+  "codeQLDatabases.displayAllLanguages": () => Promise<void>;
+  "codeQLDatabases.displayCpp": () => Promise<void>;
+  "codeQLDatabases.displayCsharp": () => Promise<void>;
+  "codeQLDatabases.displayGo": () => Promise<void>;
+  "codeQLDatabases.displayJava": () => Promise<void>;
+  "codeQLDatabases.displayJavascript": () => Promise<void>;
+  "codeQLDatabases.displayPython": () => Promise<void>;
+  "codeQLDatabases.displayRuby": () => Promise<void>;
+  "codeQLDatabases.displaySwift": () => Promise<void>;
+  "codeQLDatabases.displayAllLanguagesSelected": () => Promise<void>;
+  "codeQLDatabases.displayCppSelected": () => Promise<void>;
+  "codeQLDatabases.displayCsharpSelected": () => Promise<void>;
+  "codeQLDatabases.displayGoSelected": () => Promise<void>;
+  "codeQLDatabases.displayJavaSelected": () => Promise<void>;
+  "codeQLDatabases.displayJavascriptSelected": () => Promise<void>;
+  "codeQLDatabases.displayPythonSelected": () => Promise<void>;
+  "codeQLDatabases.displayRubySelected": () => Promise<void>;
+  "codeQLDatabases.displaySwiftSelected": () => Promise<void>;
+
+  // Database panel context menu
+  "codeQLDatabases.setCurrentDatabase": (
+    databaseItem: DatabaseItem,
+  ) => Promise<void>;
+
+  // Database panel selection commands
+  "codeQLDatabases.removeDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.upgradeDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.renameDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.openDatabaseFolder": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.addDatabaseSource": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+
+  // Codespace template commands
+  "codeQL.setDefaultTourDatabase": () => Promise<void>;
+
+  // Internal commands
+  "codeQLDatabases.removeOrphanedDatabases": () => Promise<void>;
+  "codeQL.getCurrentDatabase": () => Promise<string | undefined>;
+};
+
+// Commands tied to variant analysis
+export type VariantAnalysisCommands = {
+  "codeQL.autoDownloadVariantAnalysisResult": (
+    scannedRepo: VariantAnalysisScannedRepository,
+    variantAnalysisSummary: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.loadVariantAnalysisRepoResults": (
+    variantAnalysisId: number,
+    repositoryFullName: string,
+  ) => Promise<VariantAnalysisScannedRepositoryResult>;
+  "codeQL.monitorNewVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.monitorRehydratedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.monitorReauthenticatedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.openVariantAnalysisLogs": (
+    variantAnalysisId: number,
+  ) => Promise<void>;
+  "codeQL.openVariantAnalysisView": (
+    variantAnalysisId: number,
+  ) => Promise<void>;
+  "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
+  "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQLQueries.runVariantAnalysisContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+};
+
+export type DatabasePanelCommands = {
+  "codeQLVariantAnalysisRepositories.openConfigFile": () => Promise<void>;
+  "codeQLVariantAnalysisRepositories.addNewDatabase": () => Promise<void>;
+  "codeQLVariantAnalysisRepositories.addNewList": () => Promise<void>;
+  "codeQLVariantAnalysisRepositories.setupControllerRepository": () => Promise<void>;
+
+  "codeQLVariantAnalysisRepositories.setSelectedItem": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+};
+
+export type AstCfgCommands = {
+  "codeQL.viewAst": (selectedFile: Uri) => Promise<void>;
+  "codeQL.viewAstContextExplorer": (selectedFile: Uri) => Promise<void>;
+  "codeQL.viewAstContextEditor": (selectedFile: Uri) => Promise<void>;
+  "codeQL.viewCfg": () => Promise<void>;
+  "codeQL.viewCfgContextExplorer": () => Promise<void>;
+  "codeQL.viewCfgContextEditor": () => Promise<void>;
+};
+
+export type AstViewerCommands = {
+  "codeQLAstViewer.clear": () => Promise<void>;
+  "codeQLAstViewer.gotoCode": (item: AstItem) => Promise<void>;
+};
+
+export type PackagingCommands = {
+  "codeQL.installPackDependencies": () => Promise<void>;
+  "codeQL.downloadPacks": () => Promise<void>;
+};
+
+export type ModelEditorCommands = {
+  "codeQL.openModelEditor": () => Promise<void>;
+  "codeQLModelEditor.jumpToUsageLocation": (
+    method: Method,
+    usage: Usage,
+    databaseItem: DatabaseItem,
+  ) => Promise<void>;
+};
+
+export type EvalLogViewerCommands = {
+  "codeQLEvalLogViewer.clear": () => Promise<void>;
+};
+
+export type SummaryLanguageSupportCommands = {
+  "codeQL.gotoQL": () => Promise<void>;
+  "codeQL.gotoQLContextEditor": () => Promise<void>;
+};
+
+export type TestUICommands = {
+  "codeQLTests.showOutputDifferences": (node: TestTreeNode) => Promise<void>;
+  "codeQLTests.acceptOutput": (node: TestTreeNode) => Promise<void>;
+  "codeQLTests.acceptOutputContextTestItem": (
+    node: TestTreeNode,
+  ) => Promise<void>;
+};
+
+export type MockGitHubApiServerCommands = {
+  "codeQL.mockGitHubApiServer.startRecording": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.saveScenario": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.cancelRecording": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.loadScenario": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.unloadScenario": () => Promise<void>;
+};
+
+// All commands where the implementation is provided by this activated extension.
+export type AllExtensionCommands = BaseCommands &
+  QueryEditorCommands &
+  ResultsViewCommands &
+  QueryHistoryCommands &
+  LocalDatabasesCommands &
+  DebuggerCommands &
+  VariantAnalysisCommands &
+  DatabasePanelCommands &
+  AstCfgCommands &
+  AstViewerCommands &
+  PackagingCommands &
+  ModelEditorCommands &
+  EvalLogViewerCommands &
+  SummaryLanguageSupportCommands &
+  Partial<TestUICommands> &
+  MockGitHubApiServerCommands;
+
+export type AllCommands = AllExtensionCommands &
+  PreActivationCommands &
+  BuiltInVsCodeCommands;
+
+export type AppCommandManager = CommandManager<AllCommands>;
+
+// Separate command manager because it uses a different logger
+export type QueryServerCommands = LocalQueryCommands;
+export type QueryServerCommandManager = CommandManager<QueryServerCommands>;

extensions/ql-vscode/src/common/discovery.ts

@@ -1,25 +1,38 @@
-import { DisposableObject } from "./pure/disposable-object";
-import { extLogger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { DisposableObject } from "./disposable-object";
+import { getErrorMessage } from "./helpers-pure";
+import { Logger } from "./logging";
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
  * files. This class automatically prevents more than one discovery operation from running at the
  * same time.
  */
-export abstract class Discovery<T> extends DisposableObject {
-  private retry = false;
-  private discoveryInProgress = false;
+export abstract class Discovery extends DisposableObject {
+  private restartWhenFinished = false;
+  private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string) {
+  constructor(
+    protected readonly name: string,
+    private readonly logger: Logger,
+  ) {
     super();
   }
 
+  /**
+   * Returns the promise of the currently running refresh operation, if one is in progress.
+   * Otherwise returns a promise that resolves immediately.
+   */
+  public waitForCurrentRefresh(): Promise<void> {
+    return this.currentDiscoveryPromise ?? Promise.resolve();
+  }
+
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
+   *
+   * Returns a promise that resolves when the refresh is complete, including any retries.
    */
-  public refresh(): void {
+  public refresh(): Promise<void> {
     // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
     // got a storm of refresh requests due to, say, the copying or deletion of a large directory
     // tree, we could potentially spawn a separate simultaneous discovery operation for each
@@ -36,14 +49,16 @@ export abstract class Discovery<T> extends DisposableObject {
     // other change notifications that might be coming along. However, this would create more
     // latency in the common case, in order to save a bit of latency in the uncommon case.
 
-    if (this.discoveryInProgress) {
+    if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
-      this.retry = true;
+      this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
-      this.discoveryInProgress = true;
-      this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery().finally(() => {
+        this.currentDiscoveryPromise = undefined;
+      });
     }
+    return this.currentDiscoveryPromise;
   }
 
   /**
@@ -51,46 +66,28 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery operation completes, the `update` function will be invoked with the results of the
    * discovery.
    */
-  private launchDiscovery(): void {
-    const discoveryPromise = this.discover();
-    discoveryPromise
-      .then((results) => {
-        if (!this.retry) {
-          // Update any listeners with the results of the discovery.
-          this.discoveryInProgress = false;
-          this.update(results);
-        }
-      })
+  private async launchDiscovery(): Promise<void> {
+    try {
+      await this.discover();
+    } catch (err) {
+      void this.logger.log(
+        `${this.name} failed. Reason: ${getErrorMessage(err)}`,
+      );
+    }
 
-      .catch((err: unknown) => {
-        void extLogger.log(
-          `${this.name} failed. Reason: ${getErrorMessage(err)}`,
-        );
-      })
-
-      .finally(() => {
-        if (this.retry) {
-          // Another refresh request came in while we were still running a previous discovery
-          // operation. Since the discovery results we just computed are now stale, we'll launch
-          // another discovery operation instead of updating.
-          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-          // initial discovery operation failed.
-          this.retry = false;
-          this.launchDiscovery();
-        }
-      });
+    if (this.restartWhenFinished) {
+      // Another refresh request came in while we were still running a previous discovery
+      // operation. Since the discovery results we just computed are now stale, we'll launch
+      // another discovery operation instead of updating.
+      // We want to relaunch discovery regardless of if the initial discovery operation
+      // succeeded or failed.
+      this.restartWhenFinished = false;
+      await this.launchDiscovery();
+    }
   }
 
   /**
    * Overridden by the derived class to spawn the actual discovery operation, returning the results.
    */
-  protected abstract discover(): Promise<T>;
-
-  /**
-   * Overridden by the derived class to atomically update the `Discovery` object with the results of
-   * the discovery operation, and to notify any listeners that the discovery results may have
-   * changed.
-   * @param results The discovery results returned by the `discover` function.
-   */
-  protected abstract update(results: T): void;
+  protected abstract discover(): Promise<void>;
 }

extensions/ql-vscode/src/common/errors.ts

@@ -1,6 +1,6 @@
 export class RedactableError extends Error {
   constructor(
-    cause: Error | undefined,
+    cause: ErrorLike | undefined,
     private readonly strings: TemplateStringsArray,
     private readonly values: unknown[],
   ) {
@@ -54,19 +54,34 @@ export function redactableError(
   ...values: unknown[]
 ): RedactableError;
 export function redactableError(
-  error: Error,
+  error: ErrorLike,
 ): (strings: TemplateStringsArray, ...values: unknown[]) => RedactableError;
 
 export function redactableError(
-  errorOrStrings: Error | TemplateStringsArray,
+  errorOrStrings: ErrorLike | TemplateStringsArray,
   ...values: unknown[]
 ):
   | ((strings: TemplateStringsArray, ...values: unknown[]) => RedactableError)
   | RedactableError {
-  if (errorOrStrings instanceof Error) {
+  if (isErrorLike(errorOrStrings)) {
     return (strings: TemplateStringsArray, ...values: unknown[]) =>
       new RedactableError(errorOrStrings, strings, values);
   } else {
     return new RedactableError(undefined, errorOrStrings, values);
   }
 }
+
+export interface ErrorLike {
+  message: string;
+  stack?: string;
+}
+
+function isErrorLike(error: any): error is ErrorLike {
+  if (
+    typeof error.message === "string" &&
+    (error.stack === undefined || typeof error.stack === "string")
+  ) {
+    return true;
+  }
+  return false;
+}

extensions/ql-vscode/src/common/events.ts

@@ -1,10 +1,10 @@
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 
 export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;
 }
 
-export interface AppEventEmitter<T> {
+export interface AppEventEmitter<T> extends Disposable {
   event: AppEvent<T>;
   fire(data: T): void;
 }

extensions/ql-vscode/src/common/file-tree-nodes.ts

@@ -0,0 +1,122 @@
+import { basename, dirname, join } from "path";
+import { EnvironmentContext } from "./app";
+
+/**
+ * A node in the tree of files. This will be either a `FileTreeDirectory` or a `FileTreeLeaf`.
+ */
+export abstract class FileTreeNode<T = undefined> {
+  constructor(
+    private _path: string,
+    private _name: string,
+    private _data?: T,
+  ) {}
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public get data(): T | undefined {
+    return this._data;
+  }
+
+  public abstract get children(): ReadonlyArray<FileTreeNode<T>>;
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more files or other directories.
+ */
+export class FileTreeDirectory<T = undefined> extends FileTreeNode<T> {
+  constructor(
+    _path: string,
+    _name: string,
+    protected readonly env: EnvironmentContext,
+    private _children: Array<FileTreeNode<T>> = [],
+  ) {
+    super(_path, _name);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return this._children;
+  }
+
+  public addChild(child: FileTreeNode<T>): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): FileTreeDirectory<T> {
+    if (relativePath === ".") {
+      return this;
+    }
+    const dirName = dirname(relativePath);
+    if (dirName === ".") {
+      return this.createChildDirectory(relativePath);
+    } else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    // remove empty directories
+    this._children.filter(
+      (child) => child instanceof FileTreeLeaf || child.children.length > 0,
+    );
+    this._children.sort((a, b) =>
+      a.name.localeCompare(b.name, this.env.language),
+    );
+    this._children.forEach((child, i) => {
+      child.finish();
+      if (
+        child.children?.length === 1 &&
+        child.children[0] instanceof FileTreeDirectory
+      ) {
+        // collapse children
+        const replacement = new FileTreeDirectory<T>(
+          child.children[0].path,
+          `${child.name} / ${child.children[0].name}`,
+          this.env,
+          Array.from(child.children[0].children),
+        );
+        this._children[i] = replacement;
+      }
+    });
+  }
+
+  private createChildDirectory(name: string): FileTreeDirectory<T> {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return existingChild as FileTreeDirectory<T>;
+    } else {
+      const newChild = new FileTreeDirectory<T>(
+        join(this.path, name),
+        name,
+        this.env,
+      );
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single file.
+ */
+export class FileTreeLeaf<T = undefined> extends FileTreeNode<T> {
+  constructor(_path: string, _name: string, _data?: T) {
+    super(_path, _name, _data);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return [];
+  }
+
+  public finish(): void {
+    /**/
+  }
+}

extensions/ql-vscode/src/common/files.ts

@@ -0,0 +1,129 @@
+import { pathExists, stat, readdir, opendir } from "fs-extra";
+import { isAbsolute, join, relative, resolve } from "path";
+import { tmpdir as osTmpdir } from "os";
+
+/**
+ * Recursively finds all .ql files in this set of Uris.
+ *
+ * @param paths The list of Uris to search through
+ *
+ * @returns list of ql files and a boolean describing whether or not a directory was found/
+ */
+export async function gatherQlFiles(
+  paths: string[],
+): Promise<[string[], boolean]> {
+  const gatheredUris: Set<string> = new Set();
+  let dirFound = false;
+  for (const nextPath of paths) {
+    if ((await pathExists(nextPath)) && (await stat(nextPath)).isDirectory()) {
+      dirFound = true;
+      const subPaths = await readdir(nextPath);
+      const fullPaths = subPaths.map((p) => join(nextPath, p));
+      const nestedFiles = (await gatherQlFiles(fullPaths))[0];
+      nestedFiles.forEach((nested) => gatheredUris.add(nested));
+    } else if (nextPath.endsWith(".ql")) {
+      gatheredUris.add(nextPath);
+    }
+  }
+  return [Array.from(gatheredUris), dirFound];
+}
+
+/**
+ * Lists the names of directories inside the given path.
+ * @param path The path to the directory to read.
+ * @returns the names of the directories inside the given path.
+ */
+export async function getDirectoryNamesInsidePath(
+  path: string,
+): Promise<string[]> {
+  if (!(await pathExists(path))) {
+    throw Error(`Path does not exist: ${path}`);
+  }
+  if (!(await stat(path)).isDirectory()) {
+    throw Error(`Path is not a directory: ${path}`);
+  }
+
+  const dirItems = await readdir(path, { withFileTypes: true });
+
+  const dirNames = dirItems
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => dirent.name);
+
+  return dirNames;
+}
+
+export function normalizePath(path: string): string {
+  // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
+  // to normalize the paths to ensure they all get resolved to the
+  // same format. On Windows, we also need to do the comparison
+  // case-insensitively.
+  path = resolve(path);
+  if (process.platform === "win32") {
+    path = path.toLowerCase();
+  }
+  return path;
+}
+
+export function pathsEqual(path1: string, path2: string): boolean {
+  return normalizePath(path1) === normalizePath(path2);
+}
+
+/**
+ * Returns true if `parent` contains `child`, or if they are equal.
+ */
+export function containsPath(parent: string, child: string): boolean {
+  const relativePath = relative(parent, child);
+  return (
+    !relativePath.startsWith("..") &&
+    // On windows, if the two paths are in different drives, then the
+    // relative path will be an absolute path to the other drive.
+    !isAbsolute(relativePath)
+  );
+}
+
+export async function readDirFullPaths(path: string): Promise<string[]> {
+  const baseNames = await readdir(path);
+  return baseNames.map((baseName) => join(path, baseName));
+}
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(
+  dir: string,
+): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await opendir(dir)) {
+    const entry = join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}
+
+/**
+ * Error thrown from methods from the `fs` module.
+ *
+ * In practice, any error matching this is likely an instance of `NodeJS.ErrnoException`.
+ * If desired in the future, we could model more fields or use `NodeJS.ErrnoException` directly.
+ */
+export interface IOError {
+  readonly code: string;
+}
+
+export function isIOError(e: any): e is IOError {
+  return e.code !== undefined && typeof e.code === "string";
+}
+
+// This function is a wrapper around `os.tmpdir()` to make it easier to mock in tests.
+export function tmpdir(): string {
+  return osTmpdir();
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -1,4 +1,4 @@
-import { OWNER_REGEX, REPO_REGEX } from "../pure/helpers-pure";
+import { OWNER_REGEX, REPO_REGEX } from "./helpers-pure";
 
 /**
  * Checks if a string is a valid GitHub NWO.

extensions/ql-vscode/src/common/helpers-pure.ts

@@ -67,3 +67,26 @@ export function asError(e: unknown): Error {
 
   return e instanceof Error ? e : new Error(String(e));
 }
+
+/**
+ * Get error message when the error may have come from a method from the `child_process` module.
+ */
+export function getChildProcessErrorMessage(e: unknown): string {
+  return isChildProcessError(e) ? e.stderr : getErrorMessage(e);
+}
+
+/**
+ * Error thrown from methods from the `child_process` module.
+ */
+interface ChildProcessError {
+  readonly stderr: string;
+}
+
+function isChildProcessError(e: unknown): e is ChildProcessError {
+  return (
+    typeof e === "object" &&
+    e !== null &&
+    "stderr" in e &&
+    typeof e.stderr === "string"
+  );
+}

extensions/ql-vscode/src/common/index.ts

@@ -1 +0,0 @@
-export * from "./logging";

extensions/ql-vscode/src/common/interface-types.ts

@@ -5,13 +5,22 @@ import {
   ResultSetSchema,
   Column,
   ResolvableLocationValue,
-} from "./bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import {
   VariantAnalysis,
   VariantAnalysisScannedRepositoryResult,
   VariantAnalysisScannedRepositoryState,
 } from "../variant-analysis/shared/variant-analysis";
-import { RepositoriesFilterSortStateWithIds } from "./variant-analysis-filter-sort";
+import {
+  RepositoriesFilterSortState,
+  RepositoriesFilterSortStateWithIds,
+} from "../variant-analysis/shared/variant-analysis-filter-sort";
+import { ErrorLike } from "../common/errors";
+import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
+import { Method, Usage } from "../model-editor/method";
+import { ModeledMethod } from "../model-editor/modeled-method";
+import { ModelEditorViewState } from "../model-editor/shared/view-state";
+import { Mode } from "../model-editor/shared/mode";
 
 /**
  * This module contains types and code that are shared between
@@ -67,11 +76,9 @@ export type GraphInterpretationData = {
   dot: string[];
 };
 
-export type InterpretationData =
-  | SarifInterpretationData
-  | GraphInterpretationData;
+type InterpretationData = SarifInterpretationData | GraphInterpretationData;
 
-export interface InterpretationT<T> {
+interface InterpretationT<T> {
   sourceLocationPrefix: string;
   numTruncatedResults: number;
   numTotalResults: number;
@@ -97,7 +104,7 @@ export type SortedResultsMap = { [resultSet: string]: SortedResultSetInfo };
  *
  * As a result of receiving this message, listeners might want to display a loading indicator.
  */
-export interface ResultsUpdatingMsg {
+interface ResultsUpdatingMsg {
   t: "resultsUpdating";
 }
 
@@ -105,7 +112,7 @@ export interface ResultsUpdatingMsg {
  * Message to set the initial state of the results view with a new
  * query.
  */
-export interface SetStateMsg {
+interface SetStateMsg {
   t: "setState";
   resultsPath: string;
   origResultsPaths: ResultsPaths;
@@ -134,7 +141,7 @@ export interface SetStateMsg {
  * Message indicating that the results view should display interpreted
  * results.
  */
-export interface ShowInterpretedPageMsg {
+interface ShowInterpretedPageMsg {
   t: "showInterpretedPage";
   interpretation: Interpretation;
   database: DatabaseInfo;
@@ -164,7 +171,7 @@ export interface NavigateMsg {
  * A message indicating that the results view should untoggle the
  * "Show results in Problems view" checkbox.
  */
-export interface UntoggleShowProblemsMsg {
+interface UntoggleShowProblemsMsg {
   t: "untoggleShowProblems";
 }
 
@@ -182,20 +189,19 @@ export type IntoResultsViewMsg =
  * A message sent from the results view.
  */
 export type FromResultsViewMsg =
+  | CommonFromViewMessages
   | ViewSourceFileMsg
   | ToggleDiagnostics
   | ChangeRawResultsSortMsg
   | ChangeInterpretedResultsSortMsg
-  | ViewLoadedMsg
   | ChangePage
-  | OpenFileMsg
-  | TelemetryMessage;
+  | OpenFileMsg;
 
 /**
  * Message from the results view to open a database source
  * file at the provided location.
  */
-export interface ViewSourceFileMsg {
+interface ViewSourceFileMsg {
   t: "viewSourceFile";
   loc: ResolvableLocationValue;
   databaseUri: string;
@@ -204,7 +210,7 @@ export interface ViewSourceFileMsg {
 /**
  * Message from the results view to open a file in an editor.
  */
-export interface OpenFileMsg {
+interface OpenFileMsg {
   t: "openFile";
   /* Full path to the file to open. */
   filePath: string;
@@ -231,6 +237,21 @@ interface ViewLoadedMsg {
   viewName: string;
 }
 
+interface TelemetryMessage {
+  t: "telemetry";
+  action: string;
+}
+
+interface UnhandledErrorMessage {
+  t: "unhandledError";
+  error: ErrorLike;
+}
+
+type CommonFromViewMessages =
+  | ViewLoadedMsg
+  | TelemetryMessage
+  | UnhandledErrorMessage;
+
 /**
  * Message from the results view to signal a request to change the
  * page.
@@ -251,7 +272,7 @@ export interface RawResultsSortState {
   sortDirection: SortDirection;
 }
 
-export type InterpretedResultsSortColumn = "alert-message";
+type InterpretedResultsSortColumn = "alert-message";
 
 export interface InterpretedResultsSortState {
   sortBy: InterpretedResultsSortColumn;
@@ -287,16 +308,15 @@ interface ChangeInterpretedResultsSortMsg {
  * Message from the compare view to the extension.
  */
 export type FromCompareViewMessage =
-  | ViewLoadedMsg
+  | CommonFromViewMessages
   | ChangeCompareMessage
   | ViewSourceFileMsg
-  | OpenQueryMessage
-  | TelemetryMessage;
+  | OpenQueryMessage;
 
 /**
  * Message from the compare view to request opening a query.
  */
-export interface OpenQueryMessage {
+interface OpenQueryMessage {
   readonly t: "openQuery";
   readonly kind: "from" | "to";
 }
@@ -384,68 +404,74 @@ export interface ParsedResultSets {
   resultSet: ResultSet;
 }
 
-export interface SetVariantAnalysisMessage {
+interface SetVariantAnalysisMessage {
   t: "setVariantAnalysis";
   variantAnalysis: VariantAnalysis;
 }
 
+interface SetFilterSortStateMessage {
+  t: "setFilterSortState";
+  filterSortState: RepositoriesFilterSortState;
+}
+
 export type VariantAnalysisState = {
   variantAnalysisId: number;
 };
 
-export interface SetRepoResultsMessage {
+interface SetRepoResultsMessage {
   t: "setRepoResults";
   repoResults: VariantAnalysisScannedRepositoryResult[];
 }
 
-export interface SetRepoStatesMessage {
+interface SetRepoStatesMessage {
   t: "setRepoStates";
   repoStates: VariantAnalysisScannedRepositoryState[];
 }
 
-export interface RequestRepositoryResultsMessage {
+interface RequestRepositoryResultsMessage {
   t: "requestRepositoryResults";
   repositoryFullName: string;
 }
 
-export interface OpenQueryFileMessage {
+interface OpenQueryFileMessage {
   t: "openQueryFile";
 }
 
-export interface OpenQueryTextMessage {
+interface OpenQueryTextMessage {
   t: "openQueryText";
 }
 
-export interface CopyRepositoryListMessage {
+interface CopyRepositoryListMessage {
   t: "copyRepositoryList";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface ExportResultsMessage {
+interface ExportResultsMessage {
   t: "exportResults";
   filterSort?: RepositoriesFilterSortStateWithIds;
 }
 
-export interface OpenLogsMessage {
+interface OpenLogsMessage {
   t: "openLogs";
 }
 
-export interface CancelVariantAnalysisMessage {
+interface CancelVariantAnalysisMessage {
   t: "cancelVariantAnalysis";
 }
 
-export interface TelemetryMessage {
-  t: "telemetry";
-  action: string;
+interface ShowDataFlowPathsMessage {
+  t: "showDataFlowPaths";
+  dataFlowPaths: DataFlowPaths;
 }
 
 export type ToVariantAnalysisMessage =
   | SetVariantAnalysisMessage
+  | SetFilterSortStateMessage
   | SetRepoResultsMessage
   | SetRepoStatesMessage;
 
 export type FromVariantAnalysisMessage =
-  | ViewLoadedMsg
+  | CommonFromViewMessages
   | RequestRepositoryResultsMessage
   | OpenQueryFileMessage
   | OpenQueryTextMessage
@@ -453,4 +479,148 @@ export type FromVariantAnalysisMessage =
   | ExportResultsMessage
   | OpenLogsMessage
   | CancelVariantAnalysisMessage
-  | TelemetryMessage;
+  | ShowDataFlowPathsMessage;
+
+interface SetDataFlowPathsMessage {
+  t: "setDataFlowPaths";
+  dataFlowPaths: DataFlowPaths;
+}
+
+export type ToDataFlowPathsMessage = SetDataFlowPathsMessage;
+
+export type FromDataFlowPathsMessage = CommonFromViewMessages;
+
+interface SetExtensionPackStateMessage {
+  t: "setModelEditorViewState";
+  viewState: ModelEditorViewState;
+}
+
+interface SetMethodsMessage {
+  t: "setMethods";
+  methods: Method[];
+}
+
+interface SetModeledMethodsMessage {
+  t: "setModeledMethods";
+  methods: Record<string, ModeledMethod>;
+}
+
+interface SetModifiedMethodsMessage {
+  t: "setModifiedMethods";
+  methodSignatures: string[];
+}
+
+interface SetInProgressMethodsMessage {
+  t: "setInProgressMethods";
+  packageName: string;
+  inProgressMethods: string[];
+}
+
+interface SwitchModeMessage {
+  t: "switchMode";
+  mode: Mode;
+}
+
+interface JumpToUsageMessage {
+  t: "jumpToUsage";
+  method: Method;
+  usage: Usage;
+}
+
+interface OpenDatabaseMessage {
+  t: "openDatabase";
+}
+
+interface OpenExtensionPackMessage {
+  t: "openExtensionPack";
+}
+
+interface RefreshMethods {
+  t: "refreshMethods";
+}
+
+interface SaveModeledMethods {
+  t: "saveModeledMethods";
+  methods: Method[];
+  modeledMethods: Record<string, ModeledMethod>;
+}
+
+interface GenerateMethodMessage {
+  t: "generateMethod";
+}
+
+interface GenerateMethodsFromLlmMessage {
+  t: "generateMethodsFromLlm";
+  packageName: string;
+  methods: Method[];
+  modeledMethods: Record<string, ModeledMethod>;
+}
+
+interface StopGeneratingMethodsFromLlmMessage {
+  t: "stopGeneratingMethodsFromLlm";
+  packageName: string;
+}
+
+interface ModelDependencyMessage {
+  t: "modelDependency";
+}
+
+interface HideModeledMethodsMessage {
+  t: "hideModeledMethods";
+  hideModeledMethods: boolean;
+}
+
+interface SetModeledMethodMessage {
+  t: "setModeledMethod";
+  method: ModeledMethod;
+}
+
+export type ToModelEditorMessage =
+  | SetExtensionPackStateMessage
+  | SetMethodsMessage
+  | SetModeledMethodsMessage
+  | SetModifiedMethodsMessage
+  | SetInProgressMethodsMessage;
+
+export type FromModelEditorMessage =
+  | ViewLoadedMsg
+  | SwitchModeMessage
+  | RefreshMethods
+  | OpenDatabaseMessage
+  | OpenExtensionPackMessage
+  | JumpToUsageMessage
+  | SaveModeledMethods
+  | GenerateMethodMessage
+  | GenerateMethodsFromLlmMessage
+  | StopGeneratingMethodsFromLlmMessage
+  | ModelDependencyMessage
+  | HideModeledMethodsMessage
+  | SetModeledMethodMessage;
+
+export type FromMethodModelingMessage =
+  | TelemetryMessage
+  | UnhandledErrorMessage
+  | SetModeledMethodMessage;
+
+interface SetMethodMessage {
+  t: "setMethod";
+  method: Method;
+}
+
+interface SetMethodModifiedMessage {
+  t: "setMethodModified";
+  isModified: boolean;
+}
+
+interface SetSelectedMethodMessage {
+  t: "setSelectedMethod";
+  method: Method;
+  modeledMethod: ModeledMethod;
+  isModified: boolean;
+}
+
+export type ToMethodModelingMessage =
+  | SetMethodMessage
+  | SetModeledMethodMessage
+  | SetMethodModifiedMessage
+  | SetSelectedMethodMessage;

extensions/ql-vscode/src/common/invocation-rate-limiter.ts

@@ -0,0 +1,89 @@
+import { Memento } from "./memento";
+
+/**
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
+ */
+export class InvocationRateLimiter<T> {
+  constructor(
+    private readonly globalState: Memento,
+    private readonly funcIdentifier: string,
+    private readonly func: () => Promise<T>,
+    private readonly createDate: (dateString?: string) => Date = (s) =>
+      s ? new Date(s) : new Date(),
+  ) {}
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(
+    minSecondsSinceLastInvocation: number,
+  ): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this.createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
+        updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this.func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined = this.globalState.get(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+    );
+    return maybeDateString ? this.createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this.globalState.update(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+      date,
+    );
+  }
+
+  private static readonly _invocationRateLimiterPrefix =
+    "invocationRateLimiter_lastInvocationDate_";
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited,
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result,
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited,
+  };
+}

extensions/ql-vscode/src/common/location-link-utils.ts

@@ -0,0 +1,27 @@
+import { FileLink } from "../variant-analysis/shared/analysis-result";
+
+export function createRemoteFileRef(
+  fileLink: FileLink,
+  startLine?: number,
+  endLine?: number,
+  startColumn?: number,
+  endColumn?: number,
+): string {
+  if (
+    startColumn &&
+    endColumn &&
+    startLine &&
+    endLine &&
+    // Verify that location information is valid; otherwise highlighting might be broken
+    ((startLine === endLine && startColumn < endColumn) || startLine < endLine)
+  ) {
+    // This relies on column highlighting of new code view on GitHub
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}C${startColumn}-L${endLine}C${endColumn}`;
+  } else if (startLine && endLine && startLine < endLine) {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}-L${endLine}`;
+  } else if (startLine) {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}`;
+  } else {
+    return `${fileLink.fileLinkPrefix}/${fileLink.filePath}`;
+  }
+}

extensions/ql-vscode/src/common/logging/index.ts

@@ -1,3 +1,4 @@
 export * from "./logger";
-export * from "./vscode/loggers";
-export * from "./vscode/output-channel-logger";
+export * from "./notification-logger";
+export * from "./notifications";
+export * from "./tee-logger";

extensions/ql-vscode/src/common/logging/logger.ts

@@ -1,12 +1,10 @@
 export interface LogOptions {
   // If false, don't output a trailing newline for the log entry. Default true.
   trailingNewline?: boolean;
-
-  // If specified, add this log entry to the log file at the specified location.
-  additionalLogLocation?: string;
 }
 
-export interface Logger {
+/** Minimal logger interface. */
+export interface BaseLogger {
   /**
    * Writes the given log message, optionally followed by a newline.
    * This function is asynchronous and will only resolve once the message is written
@@ -18,18 +16,14 @@ export interface Logger {
    * @param options Optional settings.
    */
   log(message: string, options?: LogOptions): Promise<void>;
+}
 
+/** Full logger interface, including a function to show the log in the UI. */
+export interface Logger extends BaseLogger {
   /**
    * Reveal the logger channel in the UI.
    *
    * @param preserveFocus When `true` the channel will not take focus.
    */
   show(preserveFocus?: boolean): void;
-
-  /**
-   * Remove the log at the specified location.
-   *
-   * @param location log to remove
-   */
-  removeAdditionalLogLocation(location: string | undefined): void;
 }

extensions/ql-vscode/src/common/logging/notification-logger.ts

@@ -0,0 +1,7 @@
+import { Logger } from "./logger";
+
+export interface NotificationLogger extends Logger {
+  showErrorMessage(message: string): Promise<void>;
+  showWarningMessage(message: string): Promise<void>;
+  showInformationMessage(message: string): Promise<void>;
+}

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -0,0 +1,116 @@
+import { NotificationLogger } from "./notification-logger";
+import { AppTelemetry } from "../telemetry";
+import { RedactableError } from "../errors";
+
+interface ShowAndLogOptions {
+  /**
+   * An alternate message that is added to the log, but not displayed in the popup.
+   * This is useful for adding extra detail to the logs that would be too noisy for the popup.
+   */
+  fullMessage?: string;
+}
+
+/**
+ * Show an error message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogErrorMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    dropLinesExceptInitial(message),
+    logger.showErrorMessage,
+    { fullMessage: message, ...options },
+  );
+}
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join("\n");
+}
+
+/**
+ * Show a warning message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogWarningMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showWarningMessage,
+    options,
+  );
+}
+
+/**
+ * Show an information message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogInformationMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showInformationMessage,
+    options,
+  );
+}
+
+async function internalShowAndLog(
+  logger: NotificationLogger,
+  message: string,
+  fn: (message: string) => Promise<void>,
+  { fullMessage }: ShowAndLogOptions = {},
+): Promise<void> {
+  void logger.log(fullMessage || message);
+  await fn.bind(logger)(message);
+}
+
+interface ShowAndLogExceptionOptions extends ShowAndLogOptions {
+  /** Custom properties to include in the telemetry report. */
+  extraTelemetryProperties?: { [key: string]: string };
+}
+
+/**
+ * Show an error message, log it to the console, and emit redacted information as telemetry
+ *
+ * @param logger The logger that will receive the message.
+ * @param telemetry The telemetry instance to use for reporting.
+ * @param error The error to show. Only redacted information will be included in the telemetry.
+ * @param options See individual fields on `ShowAndLogExceptionOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogExceptionWithTelemetry(
+  logger: NotificationLogger,
+  telemetry: AppTelemetry | undefined,
+  error: RedactableError,
+  options: ShowAndLogExceptionOptions = {},
+): Promise<void> {
+  telemetry?.sendError(error, options.extraTelemetryProperties);
+  return showAndLogErrorMessage(logger, error.fullMessage, options);
+}

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -0,0 +1,68 @@
+import { appendFile, ensureFile } from "fs-extra";
+import { isAbsolute } from "path";
+import { getErrorMessage } from "../helpers-pure";
+import { Logger, LogOptions } from "./logger";
+
+/**
+ * An implementation of {@link Logger} that sends the output both to another {@link Logger}
+ * and to a file.
+ *
+ * The first time a message is written, an additional banner is written to the underlying logger
+ * pointing the user to the "side log" file.
+ */
+export class TeeLogger implements Logger {
+  private emittedRedirectMessage = false;
+  private error = false;
+
+  public constructor(
+    private readonly logger: Logger,
+    private readonly location: string,
+  ) {
+    if (!isAbsolute(location)) {
+      throw new Error(
+        `Additional Log Location must be an absolute path: ${location}`,
+      );
+    }
+  }
+
+  async log(message: string, options = {} as LogOptions): Promise<void> {
+    if (!this.emittedRedirectMessage) {
+      this.emittedRedirectMessage = true;
+      const msg = `| Log being saved to ${this.location} |`;
+      const separator = new Array(msg.length).fill("-").join("");
+      await this.logger.log(separator);
+      await this.logger.log(msg);
+      await this.logger.log(separator);
+    }
+
+    if (!this.error) {
+      try {
+        const trailingNewline = options.trailingNewline ?? true;
+        await ensureFile(this.location);
+
+        await appendFile(
+          this.location,
+          message + (trailingNewline ? "\n" : ""),
+          {
+            encoding: "utf8",
+          },
+        );
+      } catch (e) {
+        // Write an error message to the primary log, and stop trying to write to the side log.
+        this.error = true;
+        const errorMessage = getErrorMessage(e);
+        await this.logger.log(
+          `Error writing to additional log file: ${errorMessage}`,
+        );
+      }
+    }
+
+    if (!this.error) {
+      await this.logger.log(message, options);
+    }
+  }
+
+  show(preserveFocus?: boolean): void {
+    this.logger.show(preserveFocus);
+  }
+}

extensions/ql-vscode/src/common/logging/vscode/index.ts

@@ -0,0 +1,2 @@
+export * from "./loggers";
+export * from "./output-channel-logger";

extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts

@@ -1,18 +1,16 @@
 import { window as Window, OutputChannel, Progress } from "vscode";
-import { ensureFile, appendFile } from "fs-extra";
-import { isAbsolute } from "path";
 import { Logger, LogOptions } from "../logger";
-import { DisposableObject } from "../../../pure/disposable-object";
+import { DisposableObject } from "../../disposable-object";
+import { NotificationLogger } from "../notification-logger";
 
 /**
  * A logger that writes messages to an output channel in the VS Code Output tab.
  */
-export class OutputChannelLogger extends DisposableObject implements Logger {
+export class OutputChannelLogger
+  extends DisposableObject
+  implements Logger, NotificationLogger
+{
   public readonly outputChannel: OutputChannel;
-  private readonly additionalLocations = new Map<
-    string,
-    AdditionalLogLocation
-  >();
   isCustomLogDirectory: boolean;
 
   constructor(title: string) {
@@ -32,27 +30,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
       } else {
         this.outputChannel.append(message);
       }
-
-      if (options.additionalLogLocation) {
-        if (!isAbsolute(options.additionalLogLocation)) {
-          throw new Error(
-            `Additional Log Location must be an absolute path: ${options.additionalLogLocation}`,
-          );
-        }
-        const logPath = options.additionalLogLocation;
-        let additional = this.additionalLocations.get(logPath);
-        if (!additional) {
-          const msg = `| Log being saved to ${logPath} |`;
-          const separator = new Array(msg.length).fill("-").join("");
-          this.outputChannel.appendLine(separator);
-          this.outputChannel.appendLine(msg);
-          this.outputChannel.appendLine(separator);
-          additional = new AdditionalLogLocation(logPath);
-          this.additionalLocations.set(logPath, additional);
-        }
-
-        await additional.log(message, options);
-      }
     } catch (e) {
       if (e instanceof Error && e.message === "Channel has been closed") {
         // Output channel is closed logging to console instead
@@ -70,29 +47,28 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
     this.outputChannel.show(preserveFocus);
   }
 
-  removeAdditionalLogLocation(location: string | undefined): void {
-    if (location) {
-      this.additionalLocations.delete(location);
-    }
+  async showErrorMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showErrorMessage);
   }
-}
 
-class AdditionalLogLocation {
-  constructor(private location: string) {}
+  async showInformationMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showInformationMessage);
+  }
 
-  async log(message: string, options = {} as LogOptions): Promise<void> {
-    if (options.trailingNewline === undefined) {
-      options.trailingNewline = true;
+  async showWarningMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showWarningMessage);
+  }
+
+  private async showMessage(
+    message: string,
+    show: (message: string, ...items: string[]) => Thenable<string | undefined>,
+  ): Promise<void> {
+    const label = "Show Log";
+    const result = await show(message, label);
+
+    if (result === label) {
+      this.show();
     }
-    await ensureFile(this.location);
-
-    await appendFile(
-      this.location,
-      message + (options.trailingNewline ? "\n" : ""),
-      {
-        encoding: "utf8",
-      },
-    );
   }
 }
 

extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts

@@ -1,8 +1,8 @@
-import { Repository } from "../variant-analysis/gh-api/repository";
+import { Repository } from "../../variant-analysis/gh-api/repository";
 import {
   VariantAnalysis,
   VariantAnalysisRepoTask,
-} from "../variant-analysis/gh-api/variant-analysis";
+} from "../../variant-analysis/gh-api/variant-analysis";
 
 // Types that represent requests/responses from the GitHub API
 // that we need to mock.
@@ -13,13 +13,15 @@ export enum RequestKind {
   GetVariantAnalysis = "getVariantAnalysis",
   GetVariantAnalysisRepo = "getVariantAnalysisRepo",
   GetVariantAnalysisRepoResult = "getVariantAnalysisRepoResult",
+  CodeSearch = "codeSearch",
+  AutoModel = "autoModel",
 }
 
-export interface BasicErorResponse {
+interface BasicErorResponse {
   message: string;
 }
 
-export interface GetRepoRequest {
+interface GetRepoRequest {
   request: {
     kind: RequestKind.GetRepo;
   };
@@ -29,7 +31,7 @@ export interface GetRepoRequest {
   };
 }
 
-export interface SubmitVariantAnalysisRequest {
+interface SubmitVariantAnalysisRequest {
   request: {
     kind: RequestKind.SubmitVariantAnalysis;
   };
@@ -39,7 +41,7 @@ export interface SubmitVariantAnalysisRequest {
   };
 }
 
-export interface GetVariantAnalysisRequest {
+interface GetVariantAnalysisRequest {
   request: {
     kind: RequestKind.GetVariantAnalysis;
   };
@@ -49,7 +51,7 @@ export interface GetVariantAnalysisRequest {
   };
 }
 
-export interface GetVariantAnalysisRepoRequest {
+interface GetVariantAnalysisRepoRequest {
   request: {
     kind: RequestKind.GetVariantAnalysisRepo;
     repositoryId: number;
@@ -72,12 +74,47 @@ export interface GetVariantAnalysisRepoResultRequest {
   };
 }
 
+interface CodeSearchRequest {
+  request: {
+    kind: RequestKind.CodeSearch;
+    query: string;
+  };
+  response: {
+    status: number;
+    body?: {
+      total_count?: number;
+      items?: Array<{
+        repository: Repository;
+      }>;
+    };
+    message?: string;
+  };
+}
+
+interface AutoModelRequest {
+  request: {
+    kind: RequestKind.AutoModel;
+    body?: {
+      candidates: string;
+    };
+  };
+  response: {
+    status: number;
+    body?: {
+      models: string;
+    };
+    message?: string;
+  };
+}
+
 export type GitHubApiRequest =
   | GetRepoRequest
   | SubmitVariantAnalysisRequest
   | GetVariantAnalysisRequest
   | GetVariantAnalysisRepoRequest
-  | GetVariantAnalysisRepoResultRequest;
+  | GetVariantAnalysisRepoResultRequest
+  | CodeSearchRequest
+  | AutoModelRequest;
 
 export const isGetRepoRequest = (
   request: GitHubApiRequest,
@@ -102,3 +139,13 @@ export const isGetVariantAnalysisRepoResultRequest = (
   request: GitHubApiRequest,
 ): request is GetVariantAnalysisRepoResultRequest =>
   request.request.kind === RequestKind.GetVariantAnalysisRepoResult;
+
+export const isCodeSearchRequest = (
+  request: GitHubApiRequest,
+): request is CodeSearchRequest =>
+  request.request.kind === RequestKind.CodeSearch;
+
+export const isAutoModelRequest = (
+  request: GitHubApiRequest,
+): request is AutoModelRequest =>
+  request.request.kind === RequestKind.AutoModel;

extensions/ql-vscode/src/common/mock-gh-api/mock-gh-api-server.ts

@@ -1,12 +1,12 @@
 import { join, resolve } from "path";
 import { pathExists } from "fs-extra";
-import { setupServer, SetupServerApi } from "msw/node";
+import { setupServer, SetupServer } from "msw/node";
 
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../disposable-object";
 
 import { Recorder } from "./recorder";
 import { createRequestHandlers } from "./request-handlers";
-import { getDirectoryNamesInsidePath } from "../pure/files";
+import { getDirectoryNamesInsidePath } from "../files";
 
 /**
  * Enables mocking of the GitHub API server via HTTP interception, using msw.
@@ -14,7 +14,7 @@ import { getDirectoryNamesInsidePath } from "../pure/files";
 export class MockGitHubApiServer extends DisposableObject {
   private _isListening: boolean;
 
-  private readonly server: SetupServerApi;
+  private readonly server: SetupServer;
   private readonly recorder: Recorder;
 
   constructor() {
@@ -129,9 +129,12 @@ export class MockGitHubApiServer extends DisposableObject {
 
   public async getDefaultScenariosPath(): Promise<string | undefined> {
     // This should be the directory where package.json is located
-    const rootDirectory = resolve(__dirname, "../..");
+    const rootDirectory = resolve(__dirname, "../../..");
 
-    const scenariosPath = resolve(rootDirectory, "src/mocks/scenarios");
+    const scenariosPath = resolve(
+      rootDirectory,
+      "src/common/mock-gh-api/scenarios",
+    );
     if (await pathExists(scenariosPath)) {
       return scenariosPath;
     }

extensions/ql-vscode/src/common/mock-gh-api/recorder.ts

@@ -2,13 +2,13 @@ import { ensureDir, writeFile } from "fs-extra";
 import { join } from "path";
 
 import { MockedRequest } from "msw";
-import { SetupServerApi } from "msw/node";
+import { SetupServer } from "msw/node";
 import { IsomorphicResponse } from "@mswjs/interceptors";
 
 import { Headers } from "headers-polyfill";
 import fetch from "node-fetch";
 
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../disposable-object";
 
 import {
   GetVariantAnalysisRepoResultRequest,
@@ -22,7 +22,7 @@ export class Recorder extends DisposableObject {
 
   private _isRecording = false;
 
-  constructor(private readonly server: SetupServerApi) {
+  constructor(private readonly server: SetupServer) {
     super();
     this.onRequestStart = this.onRequestStart.bind(this);
     this.onResponseBypass = this.onResponseBypass.bind(this);
@@ -88,13 +88,18 @@ export class Recorder extends DisposableObject {
 
         const bodyFileName = `${i}-${writtenRequest.request.kind}.body.${extension}`;
         const bodyFilePath = join(scenarioDirectory, bodyFileName);
-        await writeFile(bodyFilePath, writtenRequest.response.body);
+
+        let bodyFileLink = undefined;
+        if (writtenRequest.response.body) {
+          await writeFile(bodyFilePath, writtenRequest.response.body || "");
+          bodyFileLink = `file:${bodyFileName}`;
+        }
 
         writtenRequest = {
           ...writtenRequest,
           response: {
             ...writtenRequest.response,
-            body: `file:${bodyFileName}`,
+            body: bodyFileLink,
           },
         };
       }
@@ -240,6 +245,35 @@ async function createGitHubApiRequest(
     };
   }
 
+  const codeSearchMatch = url.match(/\/search\/code\?q=(?<query>.*)$/);
+  if (codeSearchMatch?.groups?.query) {
+    return {
+      request: {
+        kind: RequestKind.CodeSearch,
+        query: codeSearchMatch?.groups?.query,
+      },
+      response: {
+        status,
+        body: JSON.parse(body),
+      },
+    };
+  }
+
+  const autoModelMatch = url.match(
+    /\/repos\/github\/codeql\/code-scanning\/codeql\/auto-model/,
+  );
+  if (autoModelMatch) {
+    return {
+      request: {
+        kind: RequestKind.AutoModel,
+      },
+      response: {
+        status,
+        body: JSON.parse(body),
+      },
+    };
+  }
+
   return undefined;
 }
 

extensions/ql-vscode/src/common/mock-gh-api/request-handlers.ts

@@ -3,6 +3,8 @@ import { readdir, readJson, readFile } from "fs-extra";
 import { DefaultBodyType, MockedRequest, rest, RestHandler } from "msw";
 import {
   GitHubApiRequest,
+  isAutoModelRequest,
+  isCodeSearchRequest,
   isGetRepoRequest,
   isGetVariantAnalysisRepoRequest,
   isGetVariantAnalysisRepoResultRequest,
@@ -12,7 +14,7 @@ import {
 
 const baseUrl = "https://api.github.com";
 
-export type RequestHandler = RestHandler<MockedRequest<DefaultBodyType>>;
+type RequestHandler = RestHandler<MockedRequest<DefaultBodyType>>;
 
 export async function createRequestHandlers(
   scenarioDirPath: string,
@@ -25,6 +27,8 @@ export async function createRequestHandlers(
     createGetVariantAnalysisRequestHandler(requests),
     createGetVariantAnalysisRepoRequestHandler(requests),
     createGetVariantAnalysisRepoResultRequestHandler(requests),
+    createCodeSearchRequestHandler(requests),
+    createAutoModelRequestHandler(requests),
   ];
 
   return handlers;
@@ -193,3 +197,54 @@ function createGetVariantAnalysisRepoResultRequestHandler(
     },
   );
 }
+
+function createCodeSearchRequestHandler(
+  requests: GitHubApiRequest[],
+): RequestHandler {
+  const codeSearchRequests = requests.filter(isCodeSearchRequest);
+  let requestIndex = 0;
+
+  // During a code search, there are multiple request to get pages of results. We
+  // need to return different responses for each request, so keep an index of the
+  // request and return the appropriate response.
+  return rest.get(`${baseUrl}/search/code?q=*`, (_req, res, ctx) => {
+    const request = codeSearchRequests[requestIndex];
+
+    if (requestIndex < codeSearchRequests.length - 1) {
+      // If there are more requests to come, increment the index.
+      requestIndex++;
+    }
+
+    return res(
+      ctx.status(request.response.status),
+      ctx.json(request.response.body),
+    );
+  });
+}
+
+function createAutoModelRequestHandler(
+  requests: GitHubApiRequest[],
+): RequestHandler {
+  const autoModelRequests = requests.filter(isAutoModelRequest);
+  let requestIndex = 0;
+
+  // During automodeling there can be multiple API requests for each batch
+  // of candidates we want to model. We need to return different responses for each request,
+  // so keep an index of the request and return the appropriate response.
+  return rest.post(
+    `${baseUrl}/repos/github/codeql/code-scanning/codeql/auto-model`,
+    (_req, res, ctx) => {
+      const request = autoModelRequests[requestIndex];
+
+      if (requestIndex < autoModelRequests.length - 1) {
+        // If there are more requests to come, increment the index.
+        requestIndex++;
+      }
+
+      return res(
+        ctx.status(request.response.status),
+        ctx.json(request.response.body),
+      );
+    },
+  );
+}

extensions/ql-vscode/src/common/mock-gh-api/scenarios/auto-model-success/0-autoModel.json

@@ -0,0 +1,11 @@
+{
+  "request": {
+    "kind": "autoModel"
+  },
+  "response": {
+    "status": 200,
+    "body": {
+      "models": "extensions:\n- addsTo: {extensible: sinkModel, pack: codeql/java-all}\n  data:\n  - [javax.servlet.http, HttpServletResponse, true, sendRedirect, (String), '', 'Argument[this]',\n    request-forgery, ai-generated]\n  - [javax.servlet.http, HttpServletResponse, true, sendRedirect, (String), '', 'Argument[0]',\n    request-forgery, ai-generated]\n"
+    }
+  }
+}

extensions/ql-vscode/src/common/mock-gh-api/scenarios/auto-model-success/1-autoModel.json

@@ -0,0 +1,11 @@
+{
+  "request": {
+    "kind": "autoModel"
+  },
+  "response": {
+    "status": 200,
+    "body": {
+      "models": "extensions:\n- addsTo: {extensible: sinkModel, pack: codeql/java-all}\n  data:\n  - [javax.servlet, MultipartConfigElement, true, MultipartConfigElement, (String),\n    '', 'Argument[0]', request-forgery, ai-generated]\n"
+    }
+  }
+}

extensions/ql-vscode/src/common/mock-gh-api/scenarios/auto-model-success/README.md

@@ -0,0 +1 @@
+This scenario is best when modeling the `javax.servlet-api` package.