Merge pull request #2466 from github/v1.8.5

v1.8.5

.github/codeql/codeql-config.yml

@@ -2,6 +2,8 @@ name: "CodeQL config"
 queries:
   - name: Run standard queries
     uses: security-and-quality
+  - name: Experimental queries
+    uses: security-experimental
   - name: Run custom javascript queries
     uses: ./.github/codeql/queries
 paths:

.github/codeql/queries/assert-pure.ql

@@ -1,21 +1,40 @@
 /**
  * @name Unwanted dependency on vscode API
- * @kind problem
+ * @kind path-problem
  * @problem.severity error
  * @id vscode-codeql/assert-pure
  * @description The modules stored under `pure` and tested in the `pure-tests`
  * are intended to be "pure".
  */
+
 import javascript
 
-class VSCodeImport extends ASTNode {
-  VSCodeImport() {
-    this.(Import).getImportedPath().getValue() = "vscode"
+class VSCodeImport extends ImportDeclaration {
+  VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
+}
+
+class PureFile extends File {
+  PureFile() {
+    (
+      this.getRelativePath().regexpMatch(".*/src/pure/.*") or
+      this.getRelativePath().regexpMatch(".*/src/common/.*")
+    ) and
+    not this.getRelativePath().regexpMatch(".*/vscode/.*")
   }
 }
 
+Import getANonTypeOnlyImport(Module m) {
+  result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly()
+}
+
+query predicate edges(AstNode a, AstNode b) {
+  getANonTypeOnlyImport(a) = b or
+  a.(Import).getImportedModule() = b
+}
+
 from Module m, VSCodeImport v
 where
-  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
-  m.getAnImportedModule*().getAnImport() = v
-select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"
+  m.getFile() instanceof PureFile and
+  edges+(m, v)
+select m, m, v,
+  "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/unique-command-use.ql

@@ -0,0 +1,148 @@
+/**
+ * @name A VS Code command should not be used in multiple locations
+ * @kind problem
+ * @problem.severity warning
+ * @id vscode-codeql/unique-command-use
+ * @description Using each VS Code command from only one location makes
+ * our telemetry more useful, because we can differentiate more user
+ * interactions and know which features of the UI our users are using.
+ * To fix this alert, new commands will need to be made so that each one
+ * is only used from one location. The commands should share the same
+ * implementation so we do not introduce duplicate code.
+ * When fixing this alert, search the codebase for all other references
+ * to the command name. The location of the alert is an arbitrarily
+ * chosen usage of the command, and may not necessarily be the location
+ * that should be changed to fix the alert.
+ */
+
+ import javascript
+
+ /**
+  * The name of a VS Code command.
+  */
+ class CommandName extends string {
+   CommandName() { exists(CommandUsage e | e.getCommandName() = this) }
+ 
+   /**
+    * In how many ways is this command used. Will always be at least 1.
+    */
+   int getNumberOfUsages() { result = count(this.getAUse()) }
+
+   /**
+    * Get a usage of this command.
+    */
+   CommandUsage getAUse() { result.getCommandName() = this }
+ 
+   /**
+    * Get the canonical first usage of this command, to use for the location
+    * of the alert. The implementation of this ordering of usages is arbitrary
+    * and the usage given may not be the one that should be changed when fixing
+    * the alert.
+    */
+   CommandUsage getFirstUsage() {
+     result =
+       max(CommandUsage use |
+         use = this.getAUse()
+       |
+         use
+         order by
+           use.getFile().getRelativePath(), use.getLocation().getStartLine(),
+           use.getLocation().getStartColumn()
+       )
+   }
+ }
+ 
+ /**
+  * Represents a single usage of a command, either from within code or
+  * from the command's definition in package.json
+  */
+ abstract class CommandUsage extends Locatable {
+   abstract string getCommandName();
+ }
+ 
+ /**
+  * A usage of a command from the typescript code, by calling `executeCommand`.
+  */
+ class CommandUsageCallExpr extends CommandUsage, CallExpr {
+   CommandUsageCallExpr() {
+     this.getCalleeName() = "executeCommand" and
+     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+   }
+ 
+   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+ }
+
+ /**
+  * A usage of a command from the typescript code, by calling `CommandManager.execute`.
+  */
+ class CommandUsageCommandManagerMethodCallExpr extends CommandUsage, MethodCallExpr {
+   CommandUsageCommandManagerMethodCallExpr() {
+     this.getCalleeName() = "execute" and
+     this.getReceiver().getType().unfold().(TypeReference).getTypeName().getName() = "CommandManager" and
+     this.getArgument(0).(StringLiteral).getValue().matches("%codeQL%") and
+     not this.getFile().getRelativePath().matches("extensions/ql-vscode/test/%")
+   }
+
+   override string getCommandName() { result = this.getArgument(0).(StringLiteral).getValue() }
+ }
+ 
+ /**
+  * A usage of a command from any menu that isn't the command palette.
+  * This means a user could invoke the command by clicking on a button in
+  * something like a menu or a dropdown.
+  */
+ class CommandUsagePackageJsonMenuItem extends CommandUsage, JsonObject {
+   CommandUsagePackageJsonMenuItem() {
+     exists(this.getPropValue("command")) and
+     exists(PackageJson packageJson, string menuName |
+       packageJson
+           .getPropValue("contributes")
+           .getPropValue("menus")
+           .getPropValue(menuName)
+           .getElementValue(_) = this and
+       menuName != "commandPalette"
+     )
+   }
+ 
+   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+ }
+ 
+ /**
+  * Is the given command disabled for use in the command palette by
+  * a block with a `"when": "false"` field.
+  */
+ predicate isDisabledInCommandPalette(string commandName) {
+   exists(PackageJson packageJson, JsonObject commandPaletteObject |
+     packageJson
+         .getPropValue("contributes")
+         .getPropValue("menus")
+         .getPropValue("commandPalette")
+         .getElementValue(_) = commandPaletteObject and
+     commandPaletteObject.getPropValue("command").getStringValue() = commandName and
+     commandPaletteObject.getPropValue("when").getStringValue() = "false"
+   )
+ }
+ 
+ /**
+  * Represents a command being usable from the command palette.
+  * This means that a user could choose to manually invoke the command.
+  */
+ class CommandUsagePackageJsonCommandPalette extends CommandUsage, JsonObject {
+   CommandUsagePackageJsonCommandPalette() {
+     this.getFile().getBaseName() = "package.json" and
+     exists(this.getPropValue("command")) and
+     exists(PackageJson packageJson |
+       packageJson.getPropValue("contributes").getPropValue("commands").getElementValue(_) = this
+     ) and
+     not isDisabledInCommandPalette(this.getPropValue("command").getStringValue())
+   }
+ 
+   override string getCommandName() { result = this.getPropValue("command").getStringValue() }
+ }
+ 
+ from CommandName c
+ where c.getNumberOfUsages() > 1
+ select c.getFirstUsage(),
+   "The " + c + " command is used from " + c.getNumberOfUsages() + " locations"
+ 

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -80,7 +80,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -100,6 +100,11 @@ jobs:
         run: |
           npm run lint
 
+      - name: Lint Markdown
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run lint:markdown
+
       - name: Lint scenarios
         working-directory: extensions/ql-vscode
         run: |
@@ -119,7 +124,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -153,7 +158,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -199,6 +204,7 @@ jobs:
         run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
     outputs:
       cli-versions: ${{ steps.set-variables.outputs.cli-versions }}
+
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}
@@ -219,7 +225,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
 
       - name: Install dependencies
         run: |
@@ -54,7 +54,7 @@ jobs:
           echo "ref_name=$REF_NAME" >> "$GITHUB_OUTPUT"
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: vscode-codeql-extension
           path: artifacts
@@ -67,49 +67,19 @@ jobs:
 
         # TODO Run tests, or check that a test run on the same branch succeeded.
 
-      - name: Create release
-        id: create-release
-        uses: actions/create-release@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref }}
-          # This gives us a chance to manually review the created release before publishing it,
-          # as well as to test the release workflow by pushing temporary tags.
-          # Once we have set all required release metadata in this step, we can set this to `false`.
-          draft: true
-          prerelease: false
-
-      - name: Upload release asset
-        uses: actions/upload-release-asset@v1.0.1
-        if: success()
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          # Get the `upload_url` from the `create-release` step above.
-          upload_url: ${{ steps.create-release.outputs.upload_url }}
-          # Get the `vsix_path` and `ref_name` from the `prepare-artifacts` step above.
-          asset_path: ${{ steps.prepare-artifacts.outputs.vsix_path }}
-          asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
-          asset_content_type: application/zip
-
       - name: Create sourcemap ZIP file
         run: |
           cd dist/vscode-codeql/out
           zip -r ../../vscode-codeql-sourcemaps.zip *.map
 
-      - name: Upload sourcemap ZIP file
-        uses: actions/upload-release-asset@v1.0.1
-        if: success()
+      - name: Create release
+        id: create-release
+        run: |
+          gh release create ${{ github.ref_name }} --draft --title "Release ${{ github.ref_name }}" \
+            '${{ steps.prepare-artifacts.outputs.vsix_path }}#${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}' \
+            'dist/vscode-codeql-sourcemaps.zip#${{ format('vscode-codeql-sourcemaps-{0}.zip', steps.prepare-artifacts.outputs.ref_name) }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          # Get the `upload_url` from the `create-release` step above.
-          upload_url: ${{ steps.create-release.outputs.upload_url }}
-          asset_path: dist/vscode-codeql-sourcemaps.zip
-          asset_name: ${{ format('vscode-codeql-sourcemaps-{0}.zip', steps.prepare-artifacts.outputs.ref_name) }}
-          asset_content_type: application/zip
 
       ###
       # Do Post release work: version bump and changelog PR
@@ -164,10 +134,7 @@ jobs:
 
       - name: Publish to Registry
         run: |
-          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix || \
-            echo "Failed to publish to VS Code Marketplace. \
-                  If this was an authentication problem, please make sure the \
-                  auth token hasn't expired."
+          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix
 
   open-vsx-publish:
     name: Publish to Open VSX Registry

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+cd extensions/ql-vscode && npm run format-staged

.husky/pre-push

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+cd extensions/ql-vscode && ./scripts/forbid-test-only

.markdownlint.json

@@ -0,0 +1,4 @@
+{
+  "MD013": false,
+  "MD041": false
+}

.vscode/extensions.json

@@ -4,6 +4,7 @@
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
     "amodio.tsl-problem-matcher",
+    "DavidAnson.vscode-markdownlint",
     "dbaeumer.vscode-eslint",
     "esbenp.prettier-vscode",
     "firsttris.vscode-jest-runner",

.vscode/settings.json

@@ -42,22 +42,53 @@
     "LANG": "en-US",
     "TZ": "UTC"
   },
+
+  // These custom rules are read in extensions/ql-vscode/.markdownlint-cli2.cjs
+  // but markdownlint only considers that config when linting files in
+  // extensions/ql-vscode/ or its subfolders. Therefore, we have to explicitly
+  // load the custom rules here too.
+  "markdownlint.customRules": [
+    "./extensions/ql-vscode/node_modules/@github/markdownlint-github/src/rules/no-default-alt-text.js",
+    "./extensions/ql-vscode/node_modules/@github/markdownlint-github/src/rules/no-generic-link-text.js"
+  ],
+
+  // This ensures that the accessibility rule enablement done by github-markdownlint is
+  // considered by the extension too.
+  //
+  // Unfortunately, we can only specify a single extends, so the config here isn't
+  // identical since it can't also consider @github/markdownlint-github/style/base.json
+  // That's not as bad as it could be since the full config is considered for anything
+  // in extensions/ql-vscode/ or its subfolders anyway.
+  //
+  // Additional nonfiguration of the default rules is done in .markdownlint.json,
+  // which is picked up by the extension automatically, and read explicitly in
+  // .markdownlint-cli2.cjs
+  "markdownlint.config": {
+    "extends": "./extensions/ql-vscode/node_modules/@github/markdownlint-github/style/accessibility.json"
+  },
+
+  // These options are used by the `jestrunner.debug` command.
+  // They are not used by the `jestrunner.run` command.
+  // After clicking "debug" over a test, continually invoke the
+  // "Debug: Attach to Node Process" command until you see a
+  // process named "Code Helper (Plugin)". Then click "attach".
+  // This will attach the debugger to the test process.
   "jestrunner.debugOptions": {
     // Uncomment to debug integration tests
-    // "attachSimplePort": 9223,
+    "attachSimplePort": 9223,
     "env": {
       "LANG": "en-US",
       "TZ": "UTC",
 
       // Uncomment to set a custom path to a CodeQL checkout.
-      // "TEST_CODEQL_PATH": "../codeql",
+      // "TEST_CODEQL_PATH": "/absolute/path/to/checkout/of/codeql",
 
       // Uncomment to set a custom path to a CodeQL CLI executable.
       // This is the CodeQL version that will be used in the tests.
-      // "CLI_PATH": "/path/to/customg/codeql",
+      // "CLI_PATH": "/absolute/path/to/custom/codeql",
 
       // Uncomment to debug integration tests
-      // "VSCODE_WAIT_FOR_DEBUGGER": "true",
+      "VSCODE_WAIT_FOR_DEBUGGER": "true",
     }
   },
   "terminal.integrated.env.linux": {

CODEOWNERS

@@ -1,3 +1,4 @@
 **/* @github/codeql-vscode-reviewers
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
+**/data-extensions-editor/ @github/code-scanning-secexp-reviewers

CODE_OF_CONDUCT.md

@@ -67,10 +67,7 @@ members of the project's leadership.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
+This Code of Conduct is adapted from the [Contributor Covenant, version 1.4](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html).
 
 For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+[the Contributor Covenant FAQ](https://www.contributor-covenant.org/faq). For more about Contributor Covenant, see [the Contributor Covenant website](https://www.contributor-covenant.org).

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 [fork]: https://github.com/github/vscode-codeql/fork
 [pr]: https://github.com/github/vscode-codeql/compare
-[style]: https://primer.style
+[style]: https://github.com/microsoft/vscode-webview-ui-toolkit
 [code-of-conduct]: CODE_OF_CONDUCT.md
 
 Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great.
@@ -23,7 +23,9 @@ Please note that this project is released with a [Contributor Code of Conduct][c
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 
 * Follow the [style guide][style].
-* Write tests. Tests that don't require the VS Code API are located [here](extensions/ql-vscode/test). Integration tests that do require the VS Code API are located [here](extensions/ql-vscode/src/vscode-tests).
+* Write tests:
+  * [Tests that don't require the VS Code API are located here](extensions/ql-vscode/test).
+  * [Integration tests that do require the VS Code API are located here](extensions/ql-vscode/src/vscode-tests).
 * Keep your change as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests.
 * Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
@@ -93,214 +95,7 @@ More information about Storybook can be found inside the **Overview** page once
 
 ### Testing
 
-We have several types of tests:
-
-* Unit tests: these live in the `tests/unit-tests/` directory
-* View tests: these live in `src/view/variant-analysis/__tests__/`
-* VSCode integration tests:
-  * `test/vscode-tests/activated-extension` tests: These are intended to cover functionality that require the full extension to be activated but don't require the CLI. This suite is not run against multiple versions of the CLI in CI.
-  * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality around not having a workspace. The extension is not activated in these tests.
-  * `test/vscode-tests/minimal-workspace` tests: These are intended to cover functionality that need a workspace but don't require the full extension to be activated.
-* CLI integration tests: these live in `test/vscode-tests/cli-integration`
-  * These tests are intended to cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against each supported versions of the CLI in CI.
-
-The CLI integration tests require an instance of the CodeQL CLI to run so they will require some extra setup steps. When adding new tests to our test suite, please be mindful of whether they need to be in the cli-integration folder. If the tests don't depend on the CLI, they are better suited to being a VSCode integration test.
-
-Any test data you're using (sample projects, config files, etc.) must go in a `test/vscode-tests/*/data` directory. When you run the tests, the test runner will copy the data directory to `out/vscode-tests/*/data`.
-
-#### Running the tests
-
-Pre-requisites:
-1. Run `npm run build`.
-2. You will need to have `npm run watch` running in the background.
-
-##### 1. From the terminal
-
-Then, from the `extensions/ql-vscode` directory, use the appropriate command to run the tests:
-
-* Unit tests: `npm run test:unit`
-* View Tests: `npm test:view`
-* VSCode integration tests: `npm run test:vscode-integration`
-
-###### CLI integration tests
-
-The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
-
-1. Set the `TEST_CODEQL_PATH` environment variable: running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
-
-2. Run your test command:
-
-```shell
-cd extensions/ql-vscode && npm run test:cli-integration
-```
-
-##### 2. From VSCode
-
-Alternatively, you can run the tests inside of VSCode. There are several VSCode launch configurations defined that run the unit and integration tests.
-
-You will need to run tests using a task from inside of VS Code, under the "Run and Debug" view:
-
-* Unit tests: run the _Launch Unit Tests - React_ task
-* View Tests: run the _Launch Unit Tests_ task
-* VSCode integration tests: run the _Launch Unit Tests - No Workspace_ and _Launch Unit Tests - Minimal Workspace_ tasks
-
-###### CLI integration tests
-
-The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
-
-1. Set the `TEST_CODEQL_PATH` environment variable: running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
-
-2. Set the codeql path in VSCode's launch configuration: open `launch.json` and under the _Launch Integration Tests - With CLI_ section, uncomment the `"${workspaceRoot}/../codeql"` line. If you've cloned the `github/codeql` repo to a different path, replace the value with the correct path.
-
-3. Run the VSCode task from the "Run and Debug" view called _Launch Integration Tests - With CLI_.
-
-#### Running a single test
-
-##### 1. From the terminal
-
-The easiest way to run a single test is to change the `it` of the test to `it.only` and then run the test command with some additional options
-to only run tests for this specific file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
-
-```shell
-npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts
-```
-
-You can also use the `--testNamePattern` option to run a specific test within a file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
-
-```shell
-npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo"
-```
-
-##### 2. From VSCode
-
-Alternatively, you can run a single test inside VSCode. To do so, install the [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) extension. Then,
-you will have quicklinks to run a single test from within test files. To run a single unit or integration test, click the "Run" button. Debugging a single test is currently only supported
-for unit tests by default. To debug integration tests, open the `.vscode/settings.json` file and uncomment the `jestrunner.debugOptions` lines. This will allow you to debug integration tests.
-Please make sure to revert this change before committing; with this setting enabled, it is not possible to debug unit tests.
-
-Without the Jest Runner extension, you can also use the "Launch Selected Unit Test (vscode-codeql)" launch configuration to run a single unit test.
-
-#### Using a mock GitHub API server
-
-Multi-Repo Variant Analyses (MRVA) rely on the GitHub API. In order to make development and testing easy, we have functionality that allows us to intercept requests to the GitHub API and provide mock responses.
-
-##### Using a pre-recorded test scenario
-
-To run a mock MRVA scenario, follow these steps:
-1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
-```json
-"codeQL.mockGitHubApiServer": {
-  "enabled": true
-}
-```
-
-1. Run the `CodeQL: Mock GitHub API Server: Load Scenario` command from the command pallet, and choose one of the scenarios to load.
-1. Execute a normal MRVA. At this point you should see the scenario being played out, rather than an actual MRVA running.
-1. Once you're done, you can stop using the mock scenario with `CodeQL: Mock GitHub API Server: Unload Scenario`
-
-If you want to replay the same scenario you should unload and reload it so requests are replayed from the start.
-
-##### Recording a new test scenario
-To record a new mock MRVA scenario, follow these steps:
-
-1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
-```json
-"codeQL.mockGitHubApiServer": {
-  "enabled": true
-}
-```
-
-1. Run the `CodeQL: Mock GitHub API Server: Start Scenario Recording` VS Code command from the command pallet.
-1. Execute a normal MRVA.
-1. Once what you wanted to record is done (e.g. the MRVA has finished), then run the `CodeQL: Mock GitHub API Server: Save Scenario` command from the command pallet.
-1. The scenario should then be available for replaying.
-
-If you want to cancel recording, run the `CodeQL: Mock GitHub API Server: Cancel Scenario Recording` command.
-
-Once the scenario has been recorded, it's often useful to remove some of the requests to speed up the replay, particularly ones that fetch the variant analysis status. Once some of the request files have manually been removed, the [fix-scenario-file-numbering script](./extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts) can be used to update the number of the files. See the script file for details on how to use.
-
-#### Scenario data location
-
-Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.
-
-## Releasing (write access required)
-
-1. Go through [our test plan](/extensions/ql-vscode/docs/test-plan.md) to ensure that the extension is working as expected.
-1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
-    * Go through all recent PRs and make sure they are properly accounted for.
-    * Make sure all changelog entries have links back to their PR(s) if appropriate.
-    * For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
-      * Making substantial new features available to all users. This can include lifting a feature flag.
-      * Breakage in compatibility with recent versions of the CLI.
-      * Minimum required version of VS Code is increased.
-      * New telemetry events are added.
-      * Deprecation or removal of commands.
-      * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
-    * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
-    * `.github/workflows/main.yml` - all the "node-version: <version>" settings
-    * `.github/workflows/release.yml` - the "node-version: <version>" setting
-1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
-1. Create a PR for this release:
-    * This PR will contain any missing bits from steps 1 and 2. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
-    * Create a new branch for the release named after the new version. For example: `v1.3.6`
-    * Create a new commit with a message the same as the branch name.
-    * Create a PR for this branch.
-    * Wait for the PR to be merged into `main`
-1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
-    ```bash
-    git checkout main
-    git tag v1.3.6
-    ```
-
-   If you've accidentally created a badly named tag, you can delete it via
-    ```bash
-    git tag -d badly-named-tag
-    ```
-1. Push the new tag up:
-
-   a. If you're using a fork of the repo:
-
-    ```bash
-    git push upstream refs/tags/v1.3.6
-    ```
-
-   b. If you're working straight in this repo:
-
-    ```bash
-    git push origin refs/tags/v1.3.6
-    ```
-
-   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
-
-    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
-    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
-1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
-    * DO NOT approve the "publish" stages of the workflow yet.
-1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
-1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
-   or look at the source if there's any doubt the right code is being shipped.
-1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
-1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
-    - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
-1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
-1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
-1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
-1. If documentation changes need to be published, notify documentation team that release has been made.
-1. Review and merge the version bump PR that is automatically created by Actions.
-
-## Secrets and authentication for publishing
-
-Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
-
-To regenerate the Open VSX token:
-
-1. Log in to the [user settings page on Open VSX](https://open-vsx.org/user-settings/namespaces).
-1. Make sure you are a member of the GitHub namespace.
-1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
-1. Update the secret in the `publish-open-vsx` environment in the project settings.
-
-To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.
+[Information about testing can be found here](./docs/testing.md).
 
 ## Resources
 

LICENSE.md

@@ -17,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -7,7 +7,7 @@ The extension is released. You can download it from the [Visual Studio Marketpla
 To see what has changed in the last few versions of the extension, see the [Changelog](https://github.com/github/vscode-codeql/blob/main/extensions/ql-vscode/CHANGELOG.md).
 
 [![CI status badge](https://github.com/github/vscode-codeql/workflows/Build%20Extension/badge.svg)](https://github.com/github/vscode-codeql/actions?query=workflow%3A%22Build+Extension%22+branch%3Amaster)
-[![VS Marketplace badge](https://vsmarketplacebadge.apphb.com/version/github.vscode-codeql.svg)](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql)
+[![VS Marketplace badge](https://vsmarketplacebadges.dev/version/github.vscode-codeql.svg)](https://marketplace.visualstudio.com/items?itemName=github.vscode-codeql)
 
 ## Features
 
@@ -15,6 +15,7 @@ To see what has changed in the last few versions of the extension, see the [Chan
 * Shows the flow of data through the results of path queries, which is essential for triaging security results.
 * Provides an easy way to run queries from the large, open source repository of [CodeQL security queries](https://github.com/github/codeql).
 * Adds IntelliSense to support you writing and editing your own CodeQL query and library files.
+* Supports you running CodeQL queries against thousands of repositories on GitHub using multi-repository variant analysis.
 
 ## Project goals and scope
 
@@ -24,8 +25,8 @@ This project will track new feature development in CodeQL and, whenever appropri
 
 This extension depends on the following two extensions for required functionality. They will be installed automatically when you install VS Code CodeQL.
 
-- [Test Adapter Converter](https://marketplace.visualstudio.com/items?itemName=ms-vscode.test-adapter-converter)
-- [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer)
+* [Test Adapter Converter](https://marketplace.visualstudio.com/items?itemName=ms-vscode.test-adapter-converter)
+* [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer)
 
 ## Contributing
 

docs/releasing.md

@@ -0,0 +1,92 @@
+# Releasing (write access required)
+
+1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
+    * Go through all recent PRs and make sure they are properly accounted for.
+    * Make sure all changelog entries have links back to their PR(s) if appropriate.
+    * For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
+      * Making substantial new features available to all users. This can include lifting a feature flag.
+      * Breakage in compatibility with recent versions of the CLI.
+      * Minimum required version of VS Code is increased.
+      * New telemetry events are added.
+      * Deprecation or removal of commands.
+      * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
+1. Double-check that the node version we're using matches the one used for VS Code. You can find this info by seleting "About Visual Studio Code" from the top menu. If it doesn't match, you will then need to update the node version in the following files:
+    * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
+    * `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
+    * `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting
+1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
+1. Create a PR for this release:
+    * This PR will contain any missing bits from steps 1, 2 and 3. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
+    * Create a new branch for the release named after the new version. For example: `v1.3.6`
+    * Create a new commit with a message the same as the branch name.
+    * Create a PR for this branch.
+    * Wait for the PR to be merged into `main`
+1. Switch to `main` branch and pull latest changes
+1. Lock the `main` branch.
+    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
+    * Select "Lock branch"
+    * Click "Save changes"
+1. Ensure that no PRs have been merged since the release PR that you merged. If there were, you might need to unlock `main` temporarily and update the CHANGELOG again.
+1. Build the extension `npm run build` and install it on your VS Code using "Install from VSIX".
+1. Go through [our test plan](./test-plan.md) to ensure that the extension is working as expected.
+1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+
+    ```bash
+    git checkout main
+    git tag v1.3.6
+    ```
+
+   If you've accidentally created a badly named tag, you can delete it via
+
+   ```bash
+   git tag -d badly-named-tag
+   ```
+
+1. Unlock the main branch
+    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
+    * Deselect "Lock branch"
+    * Click "Save changes"
+1. Push the new tag up:
+
+   a. If you're using a fork of the repo:
+
+    ```bash
+    git push upstream refs/tags/v1.3.6
+    ```
+
+   b. If you're working straight in this repo:
+
+    ```bash
+    git push origin refs/tags/v1.3.6
+    ```
+
+   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
+
+    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
+    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
+1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
+    * DO NOT approve the "publish" stages of the workflow yet.
+1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
+1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
+   or look at the source if there's any doubt the right code is being shipped.
+1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
+1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+    * If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
+1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
+1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
+1. If documentation changes need to be published, notify documentation team that release has been made.
+1. Review and merge the version bump PR that is automatically created by Actions.
+
+## Secrets and authentication for publishing
+
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+
+To regenerate the Open VSX token:
+
+1. Log in to the [user settings page on Open VSX](https://open-vsx.org/user-settings/namespaces).
+1. Make sure you are a member of the GitHub namespace.
+1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
+1. Update the secret in the `publish-open-vsx` environment in the project settings.
+
+To regenerate the VSCode Marketplace token, please see our internal documentation. Note that Azure DevOps PATs expire every 90 days and must be regenerated.

docs/test-plan.md

@@ -2,96 +2,152 @@
 
 This document describes the manual test plan for the QL extension for Visual Studio Code.
 
-The plan will be executed manually to start with but the goal is to eventually automate parts of the process (based on 
+The plan will be executed manually to start with but the goal is to eventually automate parts of the process (based on
 effort vs value basis).
 
-#### What this doesn't cover
+## What this doesn't cover
+
 We don't need to test features (and permutations of features) that are covered by automated tests.
 
-### Before releasing the VS Code extension
-- Go through the required test cases listed below
-- Check major PRs since the previous release for specific one-off things to test. Based on that, you might want to 
+## Before releasing the VS Code extension
+
+- Run at least one local query and MRVA using the existing version of the extension (to generate "old" query history items).
+- Go through the required test cases listed below.
+- Check major PRs since the previous release for specific one-off things to test. Based on that, you might want to
 choose to go through some of the Optional Test Cases.
-- Run a query using the existing version of the extension (to generate an "old" query history item)
 
 ## Required Test Cases
 
-### Pre-requisites
+### Local databases
 
-- Flip the `codeQL.canary` flag. This will enable MRVA in the extension.
+#### Test case 1: Download a database from GitHub
 
-### Test Case 1: MRVA - Running a problem path query and viewing results
+1. Click "Download Database from GitHub" and enter `angular-cn/ng-nice` and select the javascript language if prompted
 
-1. Open the [UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+#### Test case 2: Import a database from an archive
+
+1. Obtain a javascript database for `babel/babel`
+   - You can do `gh api "/repos/babel/babel/code-scanning/codeql/databases/javascript" -H "Accept: application/zip" > babel.zip` to fetch a database from GitHub.
+2. Click "Choose Database from Archive" and select the file you just downloaded above.
+
+### Local queries
+
+#### Test case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+2. Select the `angular-cn/ng-nice` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that the paths can be opened and paths are rendered correctly
+   - Check that alert locations can be clicked on
+
+#### Test case 2: Running a problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+2. Select the `babel/babel` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that alert locations can be clicked on
+
+#### Test case 3: Running a non-probem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+2. Select the `google/brotli` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the results table is rendered
+   - Check that alert locations can be clicked on
+
+#### Test case 3: Can use AST viewer
+
+1. Click on any code location from a previous query to open a source file from a database
+2. Open the AST viewing panel and click "View AST"
+3. Once the AST is computed:
+   - Check that it can be navigated
+
+### MRVA
+
+#### Test Case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
 2. Run a MRVA against the following repo list:
-```
-{
-   "name": "test-repo-list",
-   "repositories": [
-      "angular-cn/ng-nice",
-      "apache/hadoop",
-      "apache/hive"
-   ]
-}
-```
+
+   ```json
+   {
+      "name": "test-repo-list",
+      "repositories": [
+         "angular-cn/ng-nice",
+         "apache/hadoop",
+         "apache/hive"
+      ]
+   }
+   ```
+
 3. Check that a notification message pops up and the results view is opened.
 4. Check the query history. It should:
    - Show that an item has been added to the query history
    - The item should be marked as "in progress".
 5. Once the query starts:
-   - Check the results view 
-   - Check the code paths view, including the code paths drop down menu. 
+   - Check the results view
+   - Check the code paths view, including the code paths drop down menu.
    - Check that the repository filter box works
    - Click links to files/locations on GitHub
    - Check that the query history item is updated to show the number of results
 6. Once the query completes:
    - Check that the query history item is updated to show the query status as "complete"
 
-### Test Case 2: MRVA - Running a problem query and viewing results
+#### Test Case 2: Running a problem query and viewing results
 
-1. Open the [ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
 2. Run a MRVA against the "Top 10" repositories.
-3. Check the notification message. It should:
-   - Show the number of repos that are going to be queried
-   - Provide a link to the actions workflow
+3. Check that a notification message pops up and the results view is opened.
 4. Check the query history. It should:
    - Show that an item has been added to the query history
    - The item should be marked as "in progress".
-5. Once the query starts:
-   - Check that a notification is shown with a link to the results view
+5. Once the query completes:
    - Check that the results are rendered with an alert message and a highlighted code snippet:
-![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-### Test Case 3: MRVA - Running a non-problem query and viewing results
+   ![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-1. Open the [FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+#### Test Case 3: Running a non-problem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
 2. Run a MRVA against a single repository (e.g. `google/brotli`).
-3. Once the query starts:
-   - Open the query results
+3. Check that a notification message pops up and the results view is opened.
+4. Check the query history. It should:
+   - Show that an item has been added to the query history
+   - The item should be marked as "in progress".
+5. Once the query completes:
    - Check that the results show up in a table:
-![results-table](images/results-table.png)
 
-### Test Case 4: MRVA - Interacting with query history
+   ![results-table](images/results-table.png)
+
+#### Test Case 4: Interacting with query history
 
 1. Click a history item (for MRVA):
    - Check that exporting results works
    - Check that sorting results works
-   - Check that copying repo lists works 
-2. Open the query directory (containing results):
+   - Check that copying repo lists works
+2. Click "Open Results Directory":
    - Check that the correct directory is opened and there are results in it
-3. Open variant analysis on GitHub
+3. Click "View Logs":
    - Check that the correct workflow is opened
 
-### Test Case 5: MRVA - Canceling a variant analysis run
+#### Test Case 5: Canceling a variant analysis run
 
 Run one of the above MRVAs, but cancel it from within VS Code:
+
 - Check that the query is canceled and the query history item is updated.
-- Check that the workflow run is also canceled. 
+- Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
-### Test Case 6: MRVA - Change to a different colour theme 
+### General
 
-Open one of the above MRVAs, try changing to a different colour theme and check that everything looks sensible. 
+#### Test case 1: Change to a different colour theme
+
+Open at least one of the above MRVAs and at least one local query, then try changing to a different colour theme and check that everything looks sensible.
 Are there any components that are not showing up?
 
 ## Optional Test Cases
@@ -101,9 +157,10 @@ These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA
 ### Selecting repositories to run on
 
 #### Test case 1: Running a query on a single repository
-1. When the repository exists and is public 
-   1. Has a CodeQL database for the correct language 
-   2. Has a CodeQL database for another language 
+
+1. When the repository exists and is public
+   1. Has a CodeQL database for the correct language
+   2. Has a CodeQL database for another language
    3. Does not have any CodeQL databases
 2. When the repository exists and is private
    1. Is accessible and has a CodeQL database
@@ -111,14 +168,16 @@ These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA
 3. When the repository does not exist
 
 #### Test case 2: Running a query on a custom repository list
+
 1. The repository list is non-empty
-   1. All repositories in the list have a CodeQL database 
+   1. All repositories in the list have a CodeQL database
    2. Some but not all repositories in the list have a CodeQL database
    3. No repositories in the list have a CodeQL database
 2. The repository list is empty
 
 #### Test case 3: Running a query on all repositories in an organization
-1. The org exists 
+
+1. The org exists
    1. The org contains repositories that have CodeQL databases
    2. The org contains repositories of the right language but without CodeQL databases
    3. The org contains repositories not of the right language
@@ -128,20 +187,25 @@ These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA
 ### Using different types of controller repos
 
 #### Test case 1: Running a query when the controller repository is public
+
 1. Can run queries on public repositories
 2. Can not run queries on private repositories
 
 #### Test case 2: Running a query when the controller repository is private
+
 1. Can run queries on public repositories
 2. Can run queries on private repositories
 
 #### Test case 3: Running a query when the controller repo exists but you do not have write access
+
 1. Cannot run queries
 
 #### Test case 4: Running a query when the controller repo doesn’t exist
+
 1. Cannot run queries
 
 #### Test case 5: Running a query when the "config field" for the controller repo is not set
+
 1. Cannot run queries
 
 ### Query History
@@ -152,6 +216,7 @@ The first test case specifies actions that you can do when the query is first ru
 with this since it has quite a limited number of actions you can do.
 
 #### Test case 1: When variant analysis state is "pending"
+
 1. Starts monitoring variant analysis
 2. Cannot open query history item
 3. Can delete a query history item
@@ -162,10 +227,10 @@ with this since it has quite a limited number of actions you can do.
    2. By query date
    3. By result count
 5. Cannot open query directory
-6. Can open query that produced these results 
-   1. When the file still exists and has not moved 
+6. Can open query that produced these results
+   1. When the file still exists and has not moved
    2. When the file does not exist
-7. Cannot open variant analysis on github
+7. Cannot view logs
 8. Cannot copy repository list
 9. Cannot export results
 10. Cannot select to create a gist
@@ -173,6 +238,7 @@ with this since it has quite a limited number of actions you can do.
 12. Cannot cancel analysis
 
 #### Test case 2: When the variant analysis state is not "pending"
+
 1. Query history is loaded when VSCode starts
 2. Handles when action workflow was canceled while VSCode was closed
 3. Can open query history item
@@ -189,7 +255,7 @@ with this since it has quite a limited number of actions you can do.
 7. Can open query that produced these results
     1. When the file still exists and has not moved
     2. When the file does not exist
-8. Can open variant analysis on github
+8. Can view logs
 9. Can copy repository list
    1. Text is copied to clipboard
    2. Text is a valid repository list
@@ -206,12 +272,14 @@ with this since it has quite a limited number of actions you can do.
     4. A popup allows you to open the directory
 
 #### Test case 3: When variant analysis state is "in_progress"
+
 1. Starts monitoring variant analysis
-   1. Ready results are downloaded 
-2. Can cancel analysis 
+   1. Ready results are downloaded
+2. Can cancel analysis
    1. Causes the actions run to be canceled
 
 #### Test case 4: When variant analysis state is in final state ("succeeded"/"failed"/"canceled")
+
 1. Stops monitoring variant analysis
    1. All results are downloaded if state is succeeded
    2. Otherwise, ready results are downloaded, if any are available
@@ -221,7 +289,9 @@ with this since it has quite a limited number of actions you can do.
 
 This requires running a MRVA query and seeing the results view.
 
+<!-- markdownlint-disable-next-line MD024 -->
 #### Test case 1: When variant analysis state is "pending"
+
 1. Can open a results view
 2. Results view opens automatically
    - When starting variant analysis run
@@ -229,9 +299,10 @@ This requires running a MRVA query and seeing the results view.
 3. Results view is empty
 
 #### Test case 2: When variant analysis state is not "pending"
+
 1. Can open a results view
 2. Results view opens automatically
-   1. When starting variant analysis run 
+   1. When starting variant analysis run
    2. When VSCode opens (if view was open when VSCode was closed)
 3. Can copy repository list
    1. Text is copied to clipboard
@@ -242,43 +313,45 @@ This requires running a MRVA query and seeing the results view.
 6. Can open query file
    1. When the file still exists and has not moved
    2. When the file does not exist
-7. Can open query text 
-8. Can sort repos 
-   1. By name 
-   2. By results 
-   3. By stars 
-   4. By last updated
+7. Can open query text
+8. Can sort repos
+   1. Alphabetically
+   2. By number of results
+   3. By popularity
+   4. By most recent commit
 9. Can filter repos
-10. Shows correct statistics 
-    1. Total number of results 
-    2. Total number of repositories 
+10. Shows correct statistics
+    1. Total number of results
+    2. Total number of repositories
     3. Duration
-11. Can see live results 
+11. Can see live results
     1. Results appear in extension as soon as each query is completed
 12. Can view interpreted results (i.e. for a "problem" query)
-    1. Can view non-path results 
+    1. Can view non-path results
     2. Can view code paths for "path-problem" queries
 13. Can view raw results (i.e. for a non "problem" query)
     1. Renders a table
-14. Can see skipped repositories 
-    1. Can see repos with no db in a tab 
-       1. Shown warning that explains the tab 
+14. Can see skipped repositories
+    1. Can see repos with no db in a tab
+       1. Shown warning that explains the tab
     2. Can see repos with no access in a tab
-       1. Shown warning that explains the tab 
+       1. Shown warning that explains the tab
     3. Only shows tab when there are skipped repos
-15. Result downloads 
-    1. All results are downloaded automatically 
+15. Result downloads
+    1. All results are downloaded automatically
     2. Download status is indicated by a spinner (Not currently any indication of progress beyond "downloading" and "not downloading")
-    3. Only 3 items are downloaded at a time 
-    4. Results for completed queries are still downloaded when 
-       1. Some but not all queries failed 
+    3. Only 3 items are downloaded at a time
+    4. Results for completed queries are still downloaded when
+       1. Some but not all queries failed
        2. The variant analysis was canceled after some queries completed
 
 #### Test case 3: When variant analysis state is in "succeeded" state
+
 1. Can view logs
-2. All results are downloaded 
+2. All results are downloaded
 
 #### Test case 4: When variant analysis is in "failed" or "canceled" state
+
 1. Can view logs
 1. Results for finished queries are still downloaded.
 
@@ -307,14 +380,17 @@ This requires running a MRVA query and seeing the results view.
 1. Collapse/expand tree nodes
 
 Error cases that trigger an error notification:
-1. Try to add a list with a name that already exists 
+
+1. Try to add a list with a name that already exists
 1. Try to add a top-level database that already exists
 1. Try to add a database in a list that already exists in the list
 
 Error cases that show an error in the panel (and only the edit button should be visible):
+
 1. Edit the db config file directly and save invalid JSON
 1. Edit the db config file directly and save valid JSON but invalid config (e.g. add an unknown property)
-1. Edit the db config file directly and save two lists with the same name 
+1. Edit the db config file directly and save two lists with the same name
 
 Cases where there the welcome view is shown:
-1. No controller repo is set in the user's settings JSON. 
+
+1. No controller repo is set in the user's settings JSON.

docs/testing.md

@@ -0,0 +1,136 @@
+# Testing
+
+We have several types of tests:
+
+* Unit tests: these live in the `tests/unit-tests/` directory
+* View tests: these live in `src/view/variant-analysis/__tests__/`
+* VSCode integration tests:
+  * `test/vscode-tests/activated-extension` tests: These are intended to cover functionality that require the full extension to be activated but don't require the CLI. This suite is not run against multiple versions of the CLI in CI.
+  * `test/vscode-tests/no-workspace` tests: These are intended to cover functionality around not having a workspace. The extension is not activated in these tests.
+  * `test/vscode-tests/minimal-workspace` tests: These are intended to cover functionality that need a workspace but don't require the full extension to be activated.
+* CLI integration tests: these live in `test/vscode-tests/cli-integration`
+  * These tests are intended to cover functionality that is related to the integration between the CodeQL CLI and the extension. These tests are run against each supported versions of the CLI in CI.
+
+The CLI integration tests require an instance of the CodeQL CLI to run so they will require some extra setup steps. When adding new tests to our test suite, please be mindful of whether they need to be in the cli-integration folder. If the tests don't depend on the CLI, they are better suited to being a VSCode integration test.
+
+Any test data you're using (sample projects, config files, etc.) must go in a `test/vscode-tests/*/data` directory. When you run the tests, the test runner will copy the data directory to `out/vscode-tests/*/data`.
+
+## Running the tests
+
+Pre-requisites:
+
+1. Run `npm run build`.
+2. You will need to have `npm run watch` running in the background.
+
+### 1. From the terminal
+
+Then, from the `extensions/ql-vscode` directory, use the appropriate command to run the tests:
+
+* Unit tests: `npm run test:unit`
+* View Tests: `npm test:view`
+* VSCode integration tests: `npm run test:vscode-integration`
+
+#### Running CLI integration tests from the terminal
+
+The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
+
+1. Set the `TEST_CODEQL_PATH` environment variable: running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+2. Run your test command:
+
+```shell
+cd extensions/ql-vscode && npm run test:cli-integration
+```
+
+### 2. From VSCode
+
+Alternatively, you can run the tests inside of VSCode. There are several VSCode launch configurations defined that run the unit and integration tests.
+
+You will need to run tests using a task from inside of VS Code, under the "Run and Debug" view:
+
+* Unit tests: run the _Launch Unit Tests - React_ task
+* View Tests: run the _Launch Unit Tests_ task
+* VSCode integration tests: run the _Launch Unit Tests - No Workspace_ and _Launch Unit Tests - Minimal Workspace_ tasks
+
+#### Running CLI integration tests from VSCode
+
+The CLI integration tests require the CodeQL standard libraries in order to run so you will need to clone a local copy of the `github/codeql` repository.
+
+1. Set the `TEST_CODEQL_PATH` environment variable: running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+2. Set the codeql path in VSCode's launch configuration: open `launch.json` and under the _Launch Integration Tests - With CLI_ section, uncomment the `"${workspaceRoot}/../codeql"` line. If you've cloned the `github/codeql` repo to a different path, replace the value with the correct path.
+
+3. Run the VSCode task from the "Run and Debug" view called _Launch Integration Tests - With CLI_.
+
+## Running a single test
+
+### 1. Running a single test from the terminal
+
+The easiest way to run a single test is to change the `it` of the test to `it.only` and then run the test command with some additional options
+to only run tests for this specific file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
+
+```shell
+npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts
+```
+
+You can also use the `--testNamePattern` option to run a specific test within a file. For example, to run the test `test/vscode-tests/cli-integration/run-queries.test.ts`:
+
+```shell
+npm run test:cli-integration -- --runTestsByPath test/vscode-tests/cli-integration/run-queries.test.ts --testNamePattern "should create a QueryEvaluationInfo"
+```
+
+### 2. Running a single test from VSCode
+
+Alternatively, you can run a single test inside VSCode. To do so, install the [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) extension. Then,
+you will have quicklinks to run a single test from within test files. To run a single unit or integration test, click the "Run" button. Debugging a single test is currently only supported
+for unit tests by default. To debug integration tests, open the `.vscode/settings.json` file and uncomment the `jestrunner.debugOptions` lines. This will allow you to debug integration tests.
+Please make sure to revert this change before committing; with this setting enabled, it is not possible to debug unit tests.
+
+Without the Jest Runner extension, you can also use the "Launch Selected Unit Test (vscode-codeql)" launch configuration to run a single unit test.
+
+## Using a mock GitHub API server
+
+Multi-Repo Variant Analyses (MRVA) rely on the GitHub API. In order to make development and testing easy, we have functionality that allows us to intercept requests to the GitHub API and provide mock responses.
+
+### Using a pre-recorded test scenario
+
+To run a mock MRVA scenario, follow these steps:
+
+1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
+
+```json
+"codeQL.mockGitHubApiServer": {
+  "enabled": true
+}
+```
+
+1. Run the `CodeQL: Mock GitHub API Server: Load Scenario` command from the command pallet, and choose one of the scenarios to load.
+1. Execute a normal MRVA. At this point you should see the scenario being played out, rather than an actual MRVA running.
+1. Once you're done, you can stop using the mock scenario with `CodeQL: Mock GitHub API Server: Unload Scenario`
+
+If you want to replay the same scenario you should unload and reload it so requests are replayed from the start.
+
+### Recording a new test scenario
+
+To record a new mock MRVA scenario, follow these steps:
+
+1. Enable the mock GitHub API server by adding the following in your VS Code user settings (which can be found by running the `Preferences: Open User Settings (JSON)` VS Code command):
+
+```json
+"codeQL.mockGitHubApiServer": {
+  "enabled": true
+}
+```
+
+1. Run the `CodeQL: Mock GitHub API Server: Start Scenario Recording` VS Code command from the command pallet.
+1. Execute a normal MRVA.
+1. Once what you wanted to record is done (e.g. the MRVA has finished), then run the `CodeQL: Mock GitHub API Server: Save Scenario` command from the command pallet.
+1. The scenario should then be available for replaying.
+
+If you want to cancel recording, run the `CodeQL: Mock GitHub API Server: Cancel Scenario Recording` command.
+
+Once the scenario has been recorded, it's often useful to remove some of the requests to speed up the replay, particularly ones that fetch the variant analysis status. Once some of the request files have manually been removed, the [fix-scenario-file-numbering script](../extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts) can be used to update the number of the files. See the script file for details on how to use.
+
+### Scenario data location
+
+Pre-recorded scenarios are stored in `./src/variant-analysis/github-api/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.

extensions/ql-vscode/.babelrc.json

@@ -0,0 +1,16 @@
+{
+  "sourceType": "unambiguous",
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "chrome": 100
+        }
+      }
+    ],
+    "@babel/preset-typescript",
+    "@babel/preset-react"
+  ],
+  "plugins": []
+}

extensions/ql-vscode/.markdownlint-cli2.cjs

@@ -0,0 +1,16 @@
+// Having the base options in a top-level config file means
+// that the VS Code markdownlint extension can pick them up
+// too, since that only considers _this_ file when looking
+// at files in this directory or below.
+base_options = require('../../.markdownlint.json')
+
+const options = require('@github/markdownlint-github').init(
+  base_options
+)
+module.exports = {
+    config: options,
+    customRules: ["@github/markdownlint-github"],
+    outputFormatters: [
+      [ "markdownlint-cli2-formatter-pretty", { "appendLink": true } ] // ensures the error message includes a link to the rule documentation
+    ]
+} 

extensions/ql-vscode/.nvmrc

@@ -1 +1 @@
-v16.14.2
+v16.17.1

extensions/ql-vscode/.storybook/main.ts

@@ -12,6 +12,9 @@ const config: StorybookConfig = {
   core: {
     builder: "@storybook/builder-webpack5",
   },
+  features: {
+    babelModeV7: true,
+  },
 };
 
 module.exports = config;

extensions/ql-vscode/CHANGELOG.md

@@ -1,8 +1,50 @@
 # CodeQL for Visual Studio Code: Changelog
 
-## [UNRELEASED]
+## 1.8.5 - 6 June 2023
 
-# 1.7.10 - 23 February 2023
+- Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Fix bug to ensure error messages have complete stack trace in message logs. [#2425](https://github.com/github/vscode-codeql/pull/2425)
+- Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
+- Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
+- Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
+- Retry MRVA results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
+
+## 1.8.4 - 3 May 2023
+
+- Avoid repeated error messages when unable to monitor a variant analysis. [#2396](https://github.com/github/vscode-codeql/pull/2396)
+- Fix bug where a variant analysis didn't display the `#select` results set correctly when the [query metadata](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#query-metadata) didn't exactly match the query results. [#2395](https://github.com/github/vscode-codeql/pull/2395)
+- On the variant analysis results page, show the count of successful analyses instead of completed analyses, and indicate the reason why analyses were not successful. [#2349](https://github.com/github/vscode-codeql/pull/2349)
+- Fix bug where the "CodeQL: Set Current Database" command didn't always select the database. [#2384](https://github.com/github/vscode-codeql/pull/2384)
+
+## 1.8.3 - 26 April 2023
+
+- Added ability to filter repositories for a variant analysis to only those that have results [#2343](https://github.com/github/vscode-codeql/pull/2343)
+- Add new configuration option to allow downloading databases from http, non-secure servers. [#2332](https://github.com/github/vscode-codeql/pull/2332)
+- Remove title actions from the query history panel that depended on history items being selected. [#2350](https://github.com/github/vscode-codeql/pull/2350)
+
+## 1.8.2 - 12 April 2023
+
+- Fix bug where users could end up with the managed CodeQL CLI getting uninstalled during upgrades and not reinstalled. [#2294](https://github.com/github/vscode-codeql/pull/2294)
+- Fix bug that was causing code flows to not get updated when switching between results. [#2288](https://github.com/github/vscode-codeql/pull/2288)
+- Restart the CodeQL language server whenever the _CodeQL: Restart Query Server_ command is invoked. This avoids bugs where the CLI version changes to support new language features, but the language server is not updated. [#2238](https://github.com/github/vscode-codeql/pull/2238)
+- Avoid requiring a manual restart of the query server when the [external CLI config file](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file) changes. [#2289](https://github.com/github/vscode-codeql/pull/2289)
+
+## 1.8.1 - 23 March 2023
+
+- Show data flow paths of a variant analysis in a new tab. [#2172](https://github.com/github/vscode-codeql/pull/2172) & [#2182](https://github.com/github/vscode-codeql/pull/2182)
+- Show labels of entities in exported CSV results. [#2170](https://github.com/github/vscode-codeql/pull/2170)
+
+## 1.8.0 - 9 March 2023
+
+- Send telemetry about unhandled errors happening within the extension. [#2125](https://github.com/github/vscode-codeql/pull/2125)
+- Enable multi-repository variant analysis. [#2144](https://github.com/github/vscode-codeql/pull/2144)
+
+## 1.7.11 - 1 March 2023
+
+- Enable collection of telemetry concerning interactions with UI elements, including buttons, links, and other inputs. [#2114](https://github.com/github/vscode-codeql/pull/2114)
+- Prevent the automatic installation of CodeQL CLI version 2.12.3 to avoid a bug in the language server. CodeQL CLI 2.12.2 will be used instead. [#2126](https://github.com/github/vscode-codeql/pull/2126)
+
+## 1.7.10 - 23 February 2023
 
 - Fix bug that was causing unwanted error notifications.
 

extensions/ql-vscode/README.md

@@ -28,7 +28,7 @@ For information about other configurations, see the separate [CodeQL help](https
 1. [Run a query](#running-a-query).
 
 ---
-
+<!-- markdownlint-disable-next-line MD024 -->
 ## Quick start: Installing and configuring the extension
 
 ### Installing the extension
@@ -71,6 +71,7 @@ in the starter workspace directory.
 
 If you're using your own clone of the CodeQL standard libraries, you can do a `git pull` from where you have the libraries checked out.
 
+<!-- markdownlint-disable-next-line MD024 -->
 ## Quick start: Using CodeQL
 
 You can find all the commands contributed by the extension in the Command Palette (**Ctrl+Shift+P** or **Cmd+Shift+P**) by typing `CodeQL`, many of them are also accessible through the interface, and via keyboard shortcuts.

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -93,12 +93,6 @@ export async function deployPackage(
     );
     await copyPackage(sourcePath, distPath);
 
-    // This is necessary for vsce to know the dependencies
-    await copyDirectory(
-      resolve(sourcePath, "node_modules"),
-      resolve(distPath, "node_modules"),
-    );
-
     return {
       distPath,
       name: packageJson.name,

extensions/ql-vscode/gulpfile.ts/package.ts

@@ -17,6 +17,7 @@ export async function packageExtension(): Promise<void> {
       "..",
       `${deployedPackage.name}-${deployedPackage.version}.vsix`,
     ),
+    "--no-dependencies",
   ];
   const proc = spawn(resolve(__dirname, "../node_modules/.bin/vsce"), args, {
     cwd: deployedPackage.distPath,

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.7.10",
+  "version": "1.8.5",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -36,6 +36,7 @@
   "activationEvents": [
     "onLanguage:ql",
     "onLanguage:ql-summary",
+    "onView:codeQLQueries",
     "onView:codeQLDatabases",
     "onView:codeQLVariantAnalysisRepositories",
     "onView:codeQLQueryHistory",
@@ -44,11 +45,6 @@
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
     "onCommand:codeQL.authenticateToGitHub",
-    "onCommand:codeQLDatabases.chooseDatabaseFolder",
-    "onCommand:codeQLDatabases.chooseDatabaseArchive",
-    "onCommand:codeQLDatabases.chooseDatabaseInternet",
-    "onCommand:codeQLDatabases.chooseDatabaseGithub",
-    "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.viewAst",
     "onCommand:codeQL.viewCfg",
     "onCommand:codeQL.openReferencedFile",
@@ -57,20 +53,11 @@
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
     "onCommand:codeQL.chooseDatabaseGithub",
-    "onCommand:codeQLDatabases.chooseDatabase",
-    "onCommand:codeQLDatabases.setCurrentDatabase",
-    "onCommand:codeQLVariantAnalysisRepositories.openConfigFile",
-    "onCommand:codeQLVariantAnalysisRepositories.addNewDatabase",
-    "onCommand:codeQLVariantAnalysisRepositories.addNewList",
-    "onCommand:codeQLVariantAnalysisRepositories.setSelectedItem",
-    "onCommand:codeQLVariantAnalysisRepositories.setSelectedItemContextMenu",
-    "onCommand:codeQLVariantAnalysisRepositories.renameItemContextMenu",
-    "onCommand:codeQLVariantAnalysisRepositories.openOnGitHubContextMenu",
-    "onCommand:codeQLVariantAnalysisRepositories.removeItemContextMenu",
     "onCommand:codeQL.quickQuery",
     "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
     "onWebviewPanel:codeQL.variantAnalysis",
+    "onWebviewPanel:codeQL.dataFlowPaths",
     "onFileSystem:codeql-zip-archive"
   ],
   "main": "./out/extension",
@@ -90,6 +77,48 @@
         "editor.wordBasedSuggestions": false
       }
     },
+    "debuggers": [
+      {
+        "type": "codeql",
+        "label": "CodeQL Debugger",
+        "languages": [
+          "ql"
+        ],
+        "configurationAttributes": {
+          "launch": {
+            "properties": {
+              "query": {
+                "type": "string",
+                "description": "Path to query file (.ql)",
+                "default": "${file}"
+              },
+              "database": {
+                "type": "string",
+                "description": "Path to the target database"
+              },
+              "additionalPacks": {
+                "type": [
+                  "array",
+                  "string"
+                ],
+                "description": "Additional folders to search for library packs. Defaults to searching all workspace folders."
+              },
+              "extensionPacks": {
+                "type": [
+                  "array",
+                  "string"
+                ],
+                "description": "Names of extension packs to include in the evaluation. These are resolved from the locations specified in `additionalPacks`."
+              }
+            }
+          }
+        },
+        "variables": {
+          "currentDatabase": "codeQL.getCurrentDatabase",
+          "currentQuery": "codeQL.getCurrentQuery"
+        }
+      }
+    ],
     "jsonValidation": [
       {
         "fileMatch": "GitHub.vscode-codeql/databases.json",
@@ -239,6 +268,19 @@
           "default": true,
           "description": "Enable the 'Quick Evaluation' CodeLens."
         },
+        "codeQL.runningQueries.useExtensionPacks": {
+          "type": "string",
+          "default": "none",
+          "enum": [
+            "none",
+            "all"
+          ],
+          "enumDescriptions": [
+            "Do not use extension packs.",
+            "Use all extension packs found in the workspace."
+          ],
+          "description": "Choose whether or not to run queries using extension packs. Requires CodeQL CLI v2.12.3 or later."
+        },
         "codeQL.resultsDisplay.pageSize": {
           "type": "integer",
           "default": 200,
@@ -281,22 +323,6 @@
           "scope": "application",
           "description": "Specifies whether or not to write telemetry events to the extension log."
         },
-        "codeQL.variantAnalysis.repositoryLists": {
-          "type": [
-            "object",
-            null
-          ],
-          "patternProperties": {
-            ".*": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          },
-          "default": null,
-          "markdownDescription": "[For internal use only] Lists of GitHub repositories that you want to run variant analysis against. This should be a JSON object where each key is a user-specified name for this repository list, and the value is an array of GitHub repositories (of the form `<owner>/<repo>`)."
-        },
         "codeQL.variantAnalysis.controllerRepo": {
           "type": "string",
           "default": "",
@@ -304,12 +330,65 @@
           "patternErrorMessage": "Please enter a valid GitHub repository",
           "markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
         },
+        "codeQL.variantAnalysis.defaultResultsFilter": {
+          "type": "string",
+          "default": "all",
+          "enum": [
+            "all",
+            "withResults"
+          ],
+          "enumDescriptions": [
+            "Show all repositories in the results view.",
+            "Show only repositories with results in the results view."
+          ],
+          "description": "The default filter to apply to the variant analysis results view."
+        },
+        "codeQL.variantAnalysis.defaultResultsSort": {
+          "type": "string",
+          "default": "numberOfResults",
+          "enum": [
+            "alphabetically",
+            "popularity",
+            "mostRecentCommit",
+            "numberOfResults"
+          ],
+          "enumDescriptions": [
+            "Sort repositories alphabetically in the results view.",
+            "Sort repositories by popularity in the results view.",
+            "Sort repositories by most recent commit in the results view.",
+            "Sort repositories by number of results in the results view."
+          ],
+          "description": "The default sorting order for repositories in the variant analysis results view."
+        },
         "codeQL.logInsights.joinOrderWarningThreshold": {
           "type": "number",
           "default": 50,
           "scope": "window",
           "minimum": 0,
           "description": "Report a warning for any join order whose metric exceeds this value."
+        },
+        "codeQL.databaseDownload.allowHttp": {
+          "type": "boolean",
+          "default": false,
+          "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+        },
+        "codeQL.createQuery.qlPackLocation": {
+          "type": "string",
+          "patternErrorMessage": "Please enter a valid folder",
+          "markdownDescription": "The name of the folder where we want to create queries and QL packs via the \"CodeQL: Create Query\" command. The folder should exist."
+        },
+        "codeQL.createQuery.autogenerateQlPacks": {
+          "type": "string",
+          "default": "ask",
+          "enum": [
+            "ask",
+            "never"
+          ],
+          "enumDescriptions": [
+            "Ask to create a QL pack when a new CodeQL database is added.",
+            "Never create a QL pack when a new CodeQL database is added."
+          ],
+          "description": "Ask the user to generate a QL pack when a new CodeQL database is downloaded."
         }
       }
     },
@@ -322,14 +401,50 @@
         "command": "codeQL.runQuery",
         "title": "CodeQL: Run Query on Selected Database"
       },
+      {
+        "command": "codeQL.runQueryContextEditor",
+        "title": "CodeQL: Run Query on Selected Database"
+      },
+      {
+        "command": "codeQL.debugQuery",
+        "title": "CodeQL: Debug Query"
+      },
+      {
+        "command": "codeQL.debugQueryContextEditor",
+        "title": "CodeQL: Debug Query"
+      },
+      {
+        "command": "codeQL.startDebuggingSelection",
+        "title": "CodeQL: Debug Selection"
+      },
+      {
+        "command": "codeQL.startDebuggingSelectionContextEditor",
+        "title": "CodeQL: Debug Selection"
+      },
+      {
+        "command": "codeQL.continueDebuggingSelection",
+        "title": "CodeQL: Debug Selection"
+      },
+      {
+        "command": "codeQL.continueDebuggingSelectionContextEditor",
+        "title": "CodeQL: Debug Selection"
+      },
       {
         "command": "codeQL.runQueryOnMultipleDatabases",
         "title": "CodeQL: Run Query on Multiple Databases"
       },
+      {
+        "command": "codeQL.runQueryOnMultipleDatabasesContextEditor",
+        "title": "CodeQL: Run Query on Multiple Databases"
+      },
       {
         "command": "codeQL.runVariantAnalysis",
         "title": "CodeQL: Run Variant Analysis"
       },
+      {
+        "command": "codeQL.runVariantAnalysisContextEditor",
+        "title": "CodeQL: Run Variant Analysis"
+      },
       {
         "command": "codeQL.exportSelectedVariantAnalysisResults",
         "title": "CodeQL: Export Variant Analysis Results"
@@ -342,18 +457,42 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.quickEvalContextEditor",
+        "title": "CodeQL: Quick Evaluation"
+      },
       {
         "command": "codeQL.openReferencedFile",
         "title": "CodeQL: Open Referenced File"
       },
+      {
+        "command": "codeQL.openReferencedFileContextEditor",
+        "title": "CodeQL: Open Referenced File"
+      },
+      {
+        "command": "codeQL.openReferencedFileContextExplorer",
+        "title": "CodeQL: Open Referenced File"
+      },
       {
         "command": "codeQL.previewQueryHelp",
         "title": "CodeQL: Preview Query Help"
       },
+      {
+        "command": "codeQL.previewQueryHelpContextExplorer",
+        "title": "CodeQL: Preview Query Help"
+      },
+      {
+        "command": "codeQL.previewQueryHelpContextEditor",
+        "title": "CodeQL: Preview Query Help"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
       },
+      {
+        "command": "codeQL.createQuery",
+        "title": "CodeQL: Create Query"
+      },
       {
         "command": "codeQL.openDocumentation",
         "title": "CodeQL: Open Documentation"
@@ -377,6 +516,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+        "title": "Add repositories with GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -437,14 +580,38 @@
         "command": "codeQL.setCurrentDatabase",
         "title": "CodeQL: Set Current Database"
       },
+      {
+        "command": "codeQL.getCurrentDatabase",
+        "title": "CodeQL: Get Current Database"
+      },
+      {
+        "command": "codeQL.getCurrentQuery",
+        "title": "CodeQL: Get Current Query"
+      },
       {
         "command": "codeQL.viewAst",
         "title": "CodeQL: View AST"
       },
+      {
+        "command": "codeQL.viewAstContextExplorer",
+        "title": "CodeQL: View AST"
+      },
+      {
+        "command": "codeQL.viewAstContextEditor",
+        "title": "CodeQL: View AST"
+      },
       {
         "command": "codeQL.viewCfg",
         "title": "CodeQL: View CFG"
       },
+      {
+        "command": "codeQL.viewCfgContextExplorer",
+        "title": "CodeQL: View CFG"
+      },
+      {
+        "command": "codeQL.viewCfgContextEditor",
+        "title": "CodeQL: View CFG"
+      },
       {
         "command": "codeQL.upgradeCurrentDatabase",
         "title": "CodeQL: Upgrade Current Database"
@@ -522,7 +689,7 @@
         "title": "CodeQL: Check for CLI Updates"
       },
       {
-        "command": "codeQLQueryHistory.openQuery",
+        "command": "codeQLQueryHistory.openQueryContextMenu",
         "title": "View Query",
         "icon": "$(edit)"
       },
@@ -532,7 +699,12 @@
         "icon": "$(preview)"
       },
       {
-        "command": "codeQLQueryHistory.removeHistoryItem",
+        "command": "codeQLQueryHistory.removeHistoryItemContextMenu",
+        "title": "Delete",
+        "icon": "$(trash)"
+      },
+      {
+        "command": "codeQLQueryHistory.removeHistoryItemContextInline",
         "title": "Delete",
         "icon": "$(trash)"
       },
@@ -640,6 +812,10 @@
         "command": "codeQLTests.acceptOutput",
         "title": "Accept Test Output"
       },
+      {
+        "command": "codeQLTests.acceptOutputContextTestItem",
+        "title": "Accept Test Output"
+      },
       {
         "command": "codeQLAstViewer.gotoCode",
         "title": "Go To Code"
@@ -665,6 +841,15 @@
         "title": "CodeQL: Go to QL Code",
         "enablement": "codeql.hasQLSource"
       },
+      {
+        "command": "codeQL.gotoQLContextEditor",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
+      },
+      {
+        "command": "codeQL.openDataExtensionsEditor",
+        "title": "CodeQL: Open Data Extensions Editor"
+      },
       {
         "command": "codeQL.mockGitHubApiServer.startRecording",
         "title": "CodeQL: Mock GitHub API Server: Start Scenario Recording"
@@ -718,21 +903,6 @@
           "when": "view == codeQLDatabases",
           "group": "navigation"
         },
-        {
-          "command": "codeQLQueryHistory.openQuery",
-          "when": "view == codeQLQueryHistory",
-          "group": "navigation"
-        },
-        {
-          "command": "codeQLQueryHistory.itemClicked",
-          "when": "view == codeQLQueryHistory",
-          "group": "navigation"
-        },
-        {
-          "command": "codeQLQueryHistory.removeHistoryItem",
-          "when": "view == codeQLQueryHistory",
-          "group": "navigation"
-        },
         {
           "command": "codeQLQueryHistory.sortByName",
           "when": "view == codeQLQueryHistory",
@@ -795,6 +965,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/ && config.codeQL.codeSearch",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -826,19 +1001,19 @@
           "group": "inline"
         },
         {
-          "command": "codeQLQueryHistory.openQuery",
+          "command": "codeQLQueryHistory.openQueryContextMenu",
           "group": "2_queryHistory@0",
           "when": "view == codeQLQueryHistory"
         },
         {
-          "command": "codeQLQueryHistory.removeHistoryItem",
+          "command": "codeQLQueryHistory.removeHistoryItemContextMenu",
           "group": "7_queryHistory@0",
-          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
+          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledRemoteResultsItemWithoutLogs || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
         },
         {
-          "command": "codeQLQueryHistory.removeHistoryItem",
+          "command": "codeQLQueryHistory.removeHistoryItemContextInline",
           "group": "inline",
-          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
+          "when": "viewItem == interpretedResultsItem || viewItem == rawResultsItem || viewItem == remoteResultsItem || viewItem == cancelledRemoteResultsItemWithoutLogs || viewItem == cancelledResultsItem || viewItem == cancelledRemoteResultsItem"
         },
         {
           "command": "codeQLQueryHistory.renameItem",
@@ -931,6 +1106,13 @@
           "when": "viewItem == testWithSource"
         }
       ],
+      "testing/item/context": [
+        {
+          "command": "codeQLTests.acceptOutputContextTestItem",
+          "group": "qltest@1",
+          "when": "controllerId == codeql && testId =~ /^test /"
+        }
+      ],
       "explorer/context": [
         {
           "command": "codeQL.setCurrentDatabase",
@@ -938,12 +1120,12 @@
           "when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
         },
         {
-          "command": "codeQL.viewAst",
+          "command": "codeQL.viewAstContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceScheme == codeql-zip-archive && !explorerResourceIsFolder && !listMultiSelection"
         },
         {
-          "command": "codeQL.viewCfg",
+          "command": "codeQL.viewCfgContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
         },
@@ -953,12 +1135,12 @@
           "when": "resourceScheme != codeql-zip-archive"
         },
         {
-          "command": "codeQL.openReferencedFile",
+          "command": "codeQL.openReferencedFileContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
@@ -972,17 +1154,49 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.runQueryContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.debugQuery",
+          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql && !inDebugMode"
+        },
+        {
+          "command": "codeQL.debugQueryContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.startDebuggingSelection",
+          "when": "config.codeQL.canary && editorLangId == ql && debugState == inactive && debugConfigurationType == codeql"
+        },
+        {
+          "command": "codeQL.startDebuggingSelectionContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.continueDebuggingSelection",
+          "when": "config.codeQL.canary && editorLangId == ql && debugState == stopped && debugType == codeql"
+        },
+        {
+          "command": "codeQL.continueDebuggingSelectionContextEditor",
+          "when": "false"
+        },
         {
           "command": "codeQL.runQueryOnMultipleDatabases",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
         {
-          "command": "codeQL.runVariantAnalysis",
-          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
+          "command": "codeQL.runQueryOnMultipleDatabasesContextEditor",
+          "when": "false"
         },
         {
-          "command": "codeQL.exportSelectedVariantAnalysisResults",
-          "when": "config.codeQL.canary"
+          "command": "codeQL.runVariantAnalysis",
+          "when": "editorLangId == ql && resourceExtname == .ql"
+        },
+        {
+          "command": "codeQL.runVariantAnalysisContextEditor",
+          "when": "false"
         },
         {
           "command": "codeQL.runQueries",
@@ -992,26 +1206,74 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.quickEvalContextEditor",
+          "when": "false"
+        },
         {
           "command": "codeQL.openReferencedFile",
           "when": "resourceExtname == .qlref"
         },
+        {
+          "command": "codeQL.openReferencedFileContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.openReferencedFileContextExplorer",
+          "when": "false"
+        },
         {
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
+        {
+          "command": "codeQL.previewQueryHelpContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.previewQueryHelpContextExplorer",
+          "when": "false"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQL.getCurrentDatabase",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.getCurrentQuery",
+          "when": "false"
+        },
         {
           "command": "codeQL.viewAst",
           "when": "resourceScheme == codeql-zip-archive"
         },
+        {
+          "command": "codeQL.viewAstContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.viewAstContextExplorer",
+          "when": "false"
+        },
         {
           "command": "codeQL.viewCfg",
           "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
         },
+        {
+          "command": "codeQL.viewCfgContextExplorer",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.viewCfgContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.openDataExtensionsEditor",
+          "when": "config.codeQL.canary && config.codeQL.dataExtensions.editor"
+        },
         {
           "command": "codeQLVariantAnalysisRepositories.openConfigFile",
           "when": "false"
@@ -1044,6 +1306,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -1097,11 +1363,15 @@
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.openQuery",
+          "command": "codeQLQueryHistory.openQueryContextMenu",
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.removeHistoryItem",
+          "command": "codeQLQueryHistory.removeHistoryItemContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.removeHistoryItemContextInline",
           "when": "false"
         },
         {
@@ -1223,43 +1493,67 @@
         {
           "command": "codeQL.mockGitHubApiServer.unloadScenario",
           "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.scenarioLoaded"
+        },
+        {
+          "command": "codeQL.createQuery",
+          "when": "config.codeQL.codespacesTemplate"
+        },
+        {
+          "command": "codeQLTests.acceptOutputContextTestItem",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.gotoQLContextEditor",
+          "when": "false"
         }
       ],
       "editor/context": [
         {
-          "command": "codeQL.runQuery",
+          "command": "codeQL.runQueryContextEditor",
+          "when": "editorLangId == ql && resourceExtname == .ql && !inDebugMode"
+        },
+        {
+          "command": "codeQL.runQueryOnMultipleDatabasesContextEditor",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
         {
-          "command": "codeQL.runQueryOnMultipleDatabases",
+          "command": "codeQL.runVariantAnalysisContextEditor",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
         {
-          "command": "codeQL.runVariantAnalysis",
-          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql"
-        },
-        {
-          "command": "codeQL.viewAst",
+          "command": "codeQL.viewAstContextEditor",
           "when": "resourceScheme == codeql-zip-archive"
         },
         {
-          "command": "codeQL.viewCfg",
+          "command": "codeQL.viewCfgContextEditor",
           "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
         },
         {
-          "command": "codeQL.quickEval",
-          "when": "editorLangId == ql"
+          "command": "codeQL.quickEvalContextEditor",
+          "when": "editorLangId == ql && debugState == inactive"
         },
         {
-          "command": "codeQL.openReferencedFile",
+          "command": "codeQL.debugQueryContextEditor",
+          "when": "config.codeQL.canary && editorLangId == ql && resourceExtname == .ql && !inDebugMode"
+        },
+        {
+          "command": "codeQL.startDebuggingSelectionContextEditor",
+          "when": "config.codeQL.canary && editorLangId == ql && debugState == inactive && debugConfigurationType == codeql"
+        },
+        {
+          "command": "codeQL.continueDebuggingSelectionContextEditor",
+          "when": "config.codeQL.canary && editorLangId == ql && debugState == stopped && debugType == codeql"
+        },
+        {
+          "command": "codeQL.openReferencedFileContextEditor",
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextEditor",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
         {
-          "command": "codeQL.gotoQL",
+          "command": "codeQL.gotoQLContextEditor",
           "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
@@ -1275,14 +1569,18 @@
     },
     "views": {
       "ql-container": [
+        {
+          "id": "codeQLQueries",
+          "name": "Queries",
+          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
+        },
         {
           "id": "codeQLDatabases",
           "name": "Databases"
         },
         {
           "id": "codeQLVariantAnalysisRepositories",
-          "name": "Variant Analysis Repositories",
-          "when": "config.codeQL.canary"
+          "name": "Variant Analysis Repositories"
         },
         {
           "id": "codeQLQueryHistory",
@@ -1308,6 +1606,10 @@
         "view": "codeQLQueryHistory",
         "contents": "You have no query history items at the moment.\n\nSelect a database to run a CodeQL query and get your first results."
       },
+      {
+        "view": "codeQLQueries",
+        "contents": "This workspace doesn't contain any CodeQL queries at the moment."
+      },
       {
         "view": "codeQLDatabases",
         "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)"
@@ -1318,7 +1620,7 @@
       },
       {
         "view": "codeQLVariantAnalysisRepositories",
-        "contents": "Set up a controller repository to start using variant analysis.\n[Set up controller repository](command:codeQLVariantAnalysisRepositories.setupControllerRepository)",
+        "contents": "Set up a controller repository to start using variant analysis. [Learn more](https://codeql.github.com/docs/codeql-for-visual-studio-code/running-codeql-queries-at-scale-with-mrva#controller-repository) about controller repositories. \n[Set up controller repository](command:codeQLVariantAnalysisRepositories.setupControllerRepository)",
         "when": "!config.codeQL.variantAnalysis.controllerRepo"
       }
     ]
@@ -1337,19 +1639,21 @@
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",
+    "lint:markdown": "markdownlint-cli2 \"../../**/*.{md,mdx}\" \"!**/node_modules/**\" \"!**/.vscode-test/**\" \"!**/build/cli/v*/**\"",
     "format-staged": "lint-staged",
     "storybook": "start-storybook -p 6006",
     "build-storybook": "build-storybook",
     "lint:scenarios": "ts-node scripts/lint-scenarios.ts",
     "check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
-    "postinstall": "patch-package"
+    "postinstall": "patch-package",
+    "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
     "@octokit/plugin-retry": "^3.0.9",
     "@octokit/rest": "^19.0.4",
-    "@primer/octicons-react": "^17.6.0",
-    "@primer/react": "^35.0.0",
     "@vscode/codicons": "^0.0.31",
+    "@vscode/debugadapter": "^1.59.0",
+    "@vscode/debugprotocol": "^1.59.0",
     "@vscode/webview-ui-toolkit": "^1.0.1",
     "ajv": "^8.11.0",
     "child-process-promise": "^2.2.1",
@@ -1357,18 +1661,18 @@
     "classnames": "~2.2.6",
     "d3": "^7.6.1",
     "d3-graphviz": "^5.0.2",
-    "fs-extra": "^10.0.1",
-    "glob-promise": "^4.2.2",
+    "fs-extra": "^11.1.1",
     "immutable": "^4.0.0",
     "js-yaml": "^4.1.0",
+    "minimatch": "^9.0.0",
     "minimist": "~1.2.6",
-    "msw": "^0.49.0",
+    "msw": "^1.2.0",
     "nanoid": "^3.2.0",
     "node-fetch": "~2.6.7",
     "p-queue": "^6.0.0",
     "path-browserify": "^1.0.1",
-    "react": "^17.0.2",
-    "react-dom": "^17.0.2",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
     "semver": "~7.3.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
@@ -1391,25 +1695,25 @@
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
     "@faker-js/faker": "^7.5.0",
-    "@octokit/plugin-throttling": "^4.3.2",
-    "@storybook/addon-actions": "^6.5.10",
-    "@storybook/addon-essentials": "^6.5.10",
-    "@storybook/addon-interactions": "^6.5.10",
-    "@storybook/addon-links": "^6.5.10",
-    "@storybook/builder-webpack5": "^6.5.10",
-    "@storybook/manager-webpack5": "^6.5.10",
-    "@storybook/react": "^6.5.10",
+    "@github/markdownlint-github": "^0.3.0",
+    "@octokit/plugin-throttling": "^5.0.1",
+    "@storybook/addon-actions": "^6.5.17-alpha.0",
+    "@storybook/addon-essentials": "^6.5.17-alpha.0",
+    "@storybook/addon-interactions": "^6.5.17-alpha.0",
+    "@storybook/addon-links": "^6.5.17-alpha.0",
+    "@storybook/builder-webpack5": "^6.5.17-alpha.0",
+    "@storybook/manager-webpack5": "^6.5.17-alpha.0",
+    "@storybook/react": "^6.5.17-alpha.0",
     "@storybook/testing-library": "^0.0.13",
     "@testing-library/jest-dom": "^5.16.5",
-    "@testing-library/react": "^12.1.5",
+    "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
     "@types/d3": "^7.4.0",
     "@types/d3-graphviz": "^2.6.6",
     "@types/del": "^4.0.0",
-    "@types/fs-extra": "^9.0.6",
-    "@types/glob": "^7.1.1",
+    "@types/fs-extra": "^11.0.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.9",
     "@types/gulp-replace": "^1.1.0",
@@ -1420,17 +1724,18 @@
     "@types/nanoid": "^3.0.0",
     "@types/node": "^16.11.25",
     "@types/node-fetch": "~2.5.2",
-    "@types/react": "^17.0.2",
-    "@types/react-dom": "^17.0.2",
+    "@types/react": "^18.0.28",
+    "@types/react-dom": "^18.0.11",
     "@types/sarif": "~2.1.2",
     "@types/semver": "~7.2.0",
     "@types/stream-chain": "~2.0.1",
     "@types/stream-json": "~1.7.1",
+    "@types/styled-components": "^5.1.11",
     "@types/tar-stream": "^2.2.2",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",
-    "@types/vscode": "^1.59.0",
+    "@types/vscode": "^1.67.0",
     "@types/webpack": "^5.28.0",
     "@types/webpack-env": "^1.18.0",
     "@types/xml2js": "~0.4.4",
@@ -1440,7 +1745,6 @@
     "@vscode/vsce": "^2.15.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
-    "babel-loader": "^8.2.5",
     "cross-env": "^7.0.3",
     "css-loader": "~3.1.0",
     "del": "^6.0.0",
@@ -1455,37 +1759,33 @@
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-storybook": "^0.6.4",
     "file-loader": "^6.2.0",
-    "glob": "^7.1.4",
+    "glob": "^10.0.0",
     "gulp": "^4.0.2",
     "gulp-esbuild": "^0.10.5",
     "gulp-replace": "^1.1.3",
     "gulp-sourcemaps": "^3.0.0",
     "gulp-typescript": "^5.0.1",
-    "husky": "~4.3.8",
+    "husky": "^8.0.0",
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "~10.2.2",
+    "lint-staged": "~13.2.0",
+    "markdownlint-cli2": "^0.6.0",
+    "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "patch-package": "^6.5.0",
+    "patch-package": "^7.0.0",
     "prettier": "^2.7.1",
-    "tar-stream": "^2.2.0",
+    "tar-stream": "^3.0.0",
     "through2": "^4.0.2",
     "ts-jest": "^29.0.1",
     "ts-json-schema-generator": "^1.1.2",
-    "ts-loader": "^8.1.0",
+    "ts-loader": "^9.4.2",
     "ts-node": "^10.7.0",
     "ts-protoc-gen": "^0.9.0",
-    "typescript": "^4.5.5",
-    "webpack": "^5.62.2",
-    "webpack-cli": "^4.6.0"
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "npm run format-staged",
-      "pre-push": "scripts/forbid-test-only"
-    }
+    "typescript": "^5.0.2",
+    "webpack": "^5.76.0",
+    "webpack-cli": "^5.0.1"
   },
   "lint-staged": {
     "./**/*.{json,css,scss}": [
@@ -1495,8 +1795,5 @@
       "prettier --write",
       "eslint --fix"
     ]
-  },
-  "resolutions": {
-    "glob-parent": "6.0.0"
   }
 }

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -18,13 +18,16 @@ import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
 
 import { getFiles } from "./util/files";
-import type { GitHubApiRequest } from "../src/mocks/gh-api-request";
-import { isGetVariantAnalysisRequest } from "../src/mocks/gh-api-request";
+import type { GitHubApiRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
+import { isGetVariantAnalysisRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
 import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
 import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 
 // Make sure we don't run into rate limits by automatically waiting until we can
 // make another request.

extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts

@@ -20,7 +20,10 @@ if (process.argv.length !== 3) {
 const scenarioName = process.argv[2];
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 const scenarioDirectory = resolve(scenariosDirectory, scenarioName);
 
 async function fixScenarioFiles() {

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -8,13 +8,19 @@ import { getFiles } from "./util/files";
 
 const extensionDirectory = resolve(__dirname, "..");
 const rootDirectory = resolve(extensionDirectory, "../..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 
 const debug = process.env.RUNNER_DEBUG || process.argv.includes("--debug");
 
 async function lintScenarios() {
   const schema = createGenerator({
-    path: resolve(extensionDirectory, "src/mocks/gh-api-request.ts"),
+    path: resolve(
+      extensionDirectory,
+      "src/variant-analysis/gh-api/mocks/gh-api-request.ts",
+    ),
     tsconfig: resolve(extensionDirectory, "tsconfig.json"),
     type: "GitHubApiRequest",
     skipTypeCheck: true,

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,7 +1,7 @@
 import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { Logger } from "../common";
+import { getErrorMessage } from "../pure/helpers-pure";
 
 /**
  * Get the version of a CodeQL CLI.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -9,10 +9,10 @@ import { Readable } from "stream";
 import { StringDecoder } from "string_decoder";
 import tk from "tree-kill";
 import { promisify } from "util";
-import { CancellationToken, commands, Disposable, Uri } from "vscode";
+import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "./pure/bqrs-cli-types";
-import { allowCanaryQueryServer, CliConfig } from "./config";
+import { BQRSInfo, DecodedBqrsChunk } from "../pure/bqrs-cli-types";
+import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
   FindDistributionResultKind,
@@ -21,14 +21,14 @@ import {
   assertNever,
   getErrorMessage,
   getErrorStack,
-} from "./pure/helpers-pure";
-import { QueryMetadata, SortDirection } from "./pure/interface-types";
-import { Logger, ProgressReporter } from "./common";
-import { CompilationMessage } from "./pure/legacy-messages";
-import { sarifParser } from "./sarif-parser";
-import { walkDirectory } from "./helpers";
-import { App } from "./common/app";
-import { QueryLanguage } from "./common/query-language";
+} from "../pure/helpers-pure";
+import { QueryMetadata, SortDirection } from "../pure/interface-types";
+import { BaseLogger, Logger, ProgressReporter } from "../common";
+import { CompilationMessage } from "../pure/legacy-messages";
+import { sarifParser } from "../common/sarif-parser";
+import { walkDirectory } from "../helpers";
+import { App } from "../common/app";
+import { QueryLanguage } from "../common/query-language";
 
 /**
  * The version of the SARIF format that we are using.
@@ -107,6 +107,21 @@ export type MlModelInfo = {
 /** The expected output of `codeql resolve ml-models`. */
 export type MlModelsInfo = { models: MlModelInfo[] };
 
+/** Information about a data extension predicate, as resolved by `codeql resolve extensions`. */
+export type DataExtensionResult = {
+  predicate: string;
+  file: string;
+  index: number;
+};
+
+/** The expected output of `codeql resolve extensions`. */
+export type ResolveExtensionsResult = {
+  models: MlModelInfo[];
+  data: {
+    [path: string]: DataExtensionResult[];
+  };
+};
+
 /**
  * The expected output of `codeql resolve qlref`.
  */
@@ -119,6 +134,11 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries`.
+ */
+export type ResolvedQueries = string[];
+
 /**
  * The expected output of `codeql resolve tests`.
  */
@@ -134,6 +154,7 @@ export interface TestCompleted {
   compilationMs: number;
   evaluationMs: number;
   expected: string;
+  actual?: string;
   diff: string[] | undefined;
   failureDescription?: string;
   failureStage?: string;
@@ -155,6 +176,8 @@ export type OnLineCallback = (
   line: string,
 ) => Promise<string | undefined> | string | undefined;
 
+type VersionChangedListener = (newVersion: SemVer | undefined) => void;
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -172,7 +195,9 @@ export class CodeQLCliServer implements Disposable {
   nullBuffer: Buffer;
 
   /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: Promise<SemVer> | undefined;
+  private _version: SemVer | undefined;
+
+  private _versionChangedListeners: VersionChangedListener[] = [];
 
   /**
    * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
@@ -193,7 +218,7 @@ export class CodeQLCliServer implements Disposable {
     private readonly app: App,
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    public readonly logger: Logger,
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -305,6 +330,7 @@ export class CodeQLCliServer implements Disposable {
     commandArgs: string[],
     description: string,
     onLine?: OnLineCallback,
+    silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
@@ -324,7 +350,12 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(" ");
-      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      // If we are running silently, we don't want to print anything to the console.
+      if (!silent) {
+        void this.logger.log(
+          `${description} using CodeQL CLI: ${argsString}...`,
+        );
+      }
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -370,24 +401,30 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        void this.logger.log("CLI command succeeded.");
+        if (!silent) {
+          void this.logger.log("CLI command succeeded.");
+        }
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
+        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
         const newError =
           stderrBuffers.length === 0
-            ? new Error(`${description} failed: ${err}`)
+            ? new Error(
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
+              )
             : new Error(
-                `${description} failed: ${Buffer.concat(stderrBuffers).toString(
-                  "utf8",
-                )}`,
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
+                  stderrBuffers,
+                ).toString("utf8")}`,
               );
         newError.stack += getErrorStack(err);
         throw newError;
       } finally {
-        void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent) {
+          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
         process.stderr.removeAllListeners("data");
@@ -424,7 +461,7 @@ export class CodeQLCliServer implements Disposable {
     command: string[],
     commandArgs: string[],
     cancellationToken?: CancellationToken,
-    logger?: Logger,
+    logger?: BaseLogger,
   ): AsyncGenerator<string, void, unknown> {
     // Add format argument first, in case commandArgs contains positional parameters.
     const args = [...command, "--format", "jsonz", ...commandArgs];
@@ -432,6 +469,11 @@ export class CodeQLCliServer implements Disposable {
     // Spawn the CodeQL process
     const codeqlPath = await this.getCodeQlPath();
     const childPromise = spawn(codeqlPath, args);
+    // Avoid a runtime message about unhandled rejection.
+    childPromise.catch(() => {
+      /**/
+    });
+
     const child = childPromise.childProcess;
 
     let cancellationRegistration: Disposable | undefined = undefined;
@@ -482,7 +524,7 @@ export class CodeQLCliServer implements Disposable {
       logger,
     }: {
       cancellationToken?: CancellationToken;
-      logger?: Logger;
+      logger?: BaseLogger;
     } = {},
   ): AsyncGenerator<EventType, void, unknown> {
     for await (const event of this.runAsyncCodeQlCliCommandInternal(
@@ -519,9 +561,11 @@ export class CodeQLCliServer implements Disposable {
     {
       progressReporter,
       onLine,
+      silent = false,
     }: {
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<string> {
     if (progressReporter) {
@@ -537,6 +581,7 @@ export class CodeQLCliServer implements Disposable {
             commandArgs,
             description,
             onLine,
+            silent,
           ).then(resolve, reject);
         } catch (err) {
           reject(err);
@@ -570,10 +615,12 @@ export class CodeQLCliServer implements Disposable {
       addFormat = true,
       progressReporter,
       onLine,
+      silent = false,
     }: {
       addFormat?: boolean;
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
@@ -584,6 +631,7 @@ export class CodeQLCliServer implements Disposable {
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
       onLine,
+      silent,
     });
     try {
       return JSON.parse(result) as OutputType;
@@ -706,6 +754,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Finds all available queries in a given directory.
+   * @param queryDir Root of directory tree to search for queries.
+   * @param silent If true, don't print logs to the CodeQL extension log.
+   * @returns The list of queries that were found.
+   */
+  public async resolveQueries(
+    queryDir: string,
+    silent?: boolean,
+  ): Promise<ResolvedQueries> {
+    const subcommandArgs = [queryDir];
+    return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
+      ["resolve", "queries"],
+      subcommandArgs,
+      "Resolving queries",
+      { silent },
+    );
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.
@@ -761,7 +828,7 @@ export class CodeQLCliServer implements Disposable {
       logger,
     }: {
       cancellationToken?: CancellationToken;
-      logger?: Logger;
+      logger?: BaseLogger;
     },
   ): AsyncGenerator<TestCompleted, void, unknown> {
     const subcommandArgs = this.cliConfig.additionalTestArguments.concat([
@@ -1006,6 +1073,7 @@ export class CodeQLCliServer implements Disposable {
     resultsPath: string,
     interpretedResultsPath: string,
     sourceInfo?: SourceInfo,
+    args?: string[],
   ): Promise<sarif.Log> {
     const additionalArgs = [
       // TODO: This flag means that we don't group interpreted results
@@ -1013,6 +1081,7 @@ export class CodeQLCliServer implements Disposable {
       // interpretation with and without this flag, or do some
       // grouping client-side.
       "--no-group-results",
+      ...(args ?? []),
     ];
 
     await this.runInterpretCommand(
@@ -1163,24 +1232,55 @@ export class CodeQLCliServer implements Disposable {
 
   /**
    * Gets information about available qlpacks
-   * @param additionalPacks A list of directories to search for qlpacks before searching in `searchPath`.
-   * @param searchPath A list of directories to search for packs not found in `additionalPacks`. If undefined,
-   *   the default CLI search path is used.
+   * @param additionalPacks A list of directories to search for qlpacks.
+   * @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
+   *    be returned. If false, all packs will be returned.
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
-  resolveQlpacks(
+  async resolveQlpacks(
     additionalPacks: string[],
-    searchPath?: string[],
+    extensionPacksOnly = false,
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
-    if (searchPath?.length) {
-      args.push("--search-path", join(...searchPath));
+    if (extensionPacksOnly) {
+      if (!(await this.cliConstraints.supportsQlpacksKind())) {
+        void this.logger.log(
+          "Warning: Running with extension packs is only supported by CodeQL CLI v2.12.3 or later.",
+        );
+        return {};
+      }
+      args.push("--kind", "extension", "--no-recursive");
     }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(
       ["resolve", "qlpacks"],
       args,
-      "Resolving qlpack information",
+      `Resolving qlpack information${
+        extensionPacksOnly ? " (extension packs only)" : ""
+      }`,
+    );
+  }
+
+  /**
+   * Gets information about available extensions
+   * @param suite The suite to resolve.
+   * @param additionalPacks A list of directories to search for qlpacks.
+   * @returns An object containing the list of models and extensions
+   */
+  async resolveExtensions(
+    suite: string,
+    additionalPacks: string[],
+  ): Promise<ResolveExtensionsResult> {
+    const args = this.getAdditionalPacksArg(additionalPacks);
+    args.push(suite);
+
+    return this.runJsonCodeQlCliCommand<ResolveExtensionsResult>(
+      ["resolve", "extensions"],
+      args,
+      "Resolving extensions",
+      {
+        addFormat: false,
+      },
     );
   }
 
@@ -1276,11 +1376,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  async packInstall(dir: string, forceUpdate = false) {
+  async packInstall(
+    dir: string,
+    { forceUpdate = false, workspaceFolders = [] as string[] } = {},
+  ) {
     const args = [dir];
     if (forceUpdate) {
       args.push("--mode", "update");
     }
+    if (workspaceFolders?.length > 0) {
+      if (await this.cliConstraints.supportsAdditionalPacksInstall()) {
+        args.push(
+          // Allow prerelease packs from the ql submodule.
+          "--allow-prerelease",
+          // Allow the use of --additional-packs argument without issueing a warning
+          "--no-strict-mode",
+          ...this.getAdditionalPacksArg(workspaceFolders),
+        );
+      }
+    }
     return this.runJsonCodeQlCliCommandWithAuthentication(
       ["pack", "install"],
       args,
@@ -1351,15 +1465,36 @@ export class CodeQLCliServer implements Disposable {
 
   public async getVersion() {
     if (!this._version) {
-      this._version = this.refreshVersion();
-      // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
-      await commands.executeCommand(
-        "setContext",
-        "codeql.supportsEvalLog",
-        await this.cliConstraints.supportsPerQueryEvalLog(),
-      );
+      try {
+        const newVersion = await this.refreshVersion();
+        this._version = newVersion;
+        this._versionChangedListeners.forEach((listener) =>
+          listener(newVersion),
+        );
+
+        // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
+        await this.app.commands.execute(
+          "setContext",
+          "codeql.supportsEvalLog",
+          newVersion.compare(
+            CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
+          ) >= 0,
+        );
+      } catch (e) {
+        this._versionChangedListeners.forEach((listener) =>
+          listener(undefined),
+        );
+        throw e;
+      }
     }
-    return await this._version;
+    return this._version;
+  }
+
+  public addVersionChangedListener(listener: VersionChangedListener) {
+    if (this._version) {
+      listener(this._version);
+    }
+    this._versionChangedListeners.push(listener);
   }
 
   private async refreshVersion() {
@@ -1380,6 +1515,17 @@ export class CodeQLCliServer implements Disposable {
   private getAdditionalPacksArg(paths: string[]): string[] {
     return paths.length ? ["--additional-packs", paths.join(delimiter)] : [];
   }
+
+  public async useExtensionPacks(): Promise<boolean> {
+    return (
+      this.cliConfig.useExtensionPacks &&
+      (await this.cliConstraints.supportsQlpacksKind())
+    );
+  }
+
+  public async setUseExtensionPacks(useExtensionPacks: boolean) {
+    await this.cliConfig.setUseExtensionPacks(useExtensionPacks);
+  }
 }
 
 /**
@@ -1423,10 +1569,23 @@ export function spawnServer(
     );
   }
 
+  let lastStdout: any = undefined;
+  child.stdout!.on("data", (data) => {
+    lastStdout = data;
+  });
   // Set up event listeners.
-  child.on("close", (code) =>
-    logger.log(`Child process exited with code ${code}`),
-  );
+  child.on("close", async (code, signal) => {
+    if (code !== null)
+      void logger.log(`Child process exited with code ${code}`);
+    if (signal)
+      void logger.log(
+        `Child process exited due to receipt of signal ${signal}`,
+      );
+    // If the process exited abnormally, log the last stdout message,
+    // It may be from the jvm.
+    if (code !== 0 && lastStdout !== undefined)
+      void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+  });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
     child.stdout!.on("data", stdoutListener);
@@ -1590,7 +1749,7 @@ const lineEndings = ["\r\n", "\r", "\n"];
  * @param stream The stream to log.
  * @param logger The logger that will consume the stream output.
  */
-async function logStream(stream: Readable, logger: Logger): Promise<void> {
+async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
   for await (const line of splitStreamAtSeparators(stream, lineEndings)) {
     // Await the result of log here in order to ensure the logs are written in the correct order.
     await logger.log(line);
@@ -1622,6 +1781,10 @@ export function shouldDebugCliServer() {
 }
 
 export class CliVersionConstraint {
+  // The oldest version of the CLI that we support. This is used to determine
+  // whether to show a warning about the CLI being too old on startup.
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.7.6");
+
   /**
    * CLI version where building QLX packs for remote queries is supported.
    * (The options were _accepted_ by a few earlier versions, but only from
@@ -1668,6 +1831,20 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_WORKSPACE_RFERENCES = new SemVer("2.11.3");
 
+  /**
+   * CLI version that supports the `--kind` option for the `resolve qlpacks` command.
+   */
+  public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
+
+  /**
+   * CLI version that supports the `--additional-packs` option for the `pack install` command.
+   */
+  public static CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL = new SemVer(
+    "2.12.4",
+  );
+
+  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1725,4 +1902,20 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_WORKSPACE_RFERENCES,
     );
   }
+
+  async supportsQlpacksKind() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
+    );
+  }
+
+  async supportsAdditionalPacksInstall() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
+    );
+  }
+
+  async usesGlobalCompilationCache() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -3,24 +3,26 @@ import { pathExists, mkdtemp, createWriteStream, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
 import * as semver from "semver";
-import { parse } from "url";
+import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
-import { DistributionConfig } from "./config";
-import {
-  InvocationRateLimiter,
-  InvocationRateLimiterResultKind,
-  showAndLogErrorMessage,
-  showAndLogWarningMessage,
-} from "./helpers";
-import { extLogger } from "./common";
+import { DistributionConfig } from "../config";
+import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers";
+import { extLogger } from "../common";
 import { getCodeQlCliVersion } from "./cli-version";
-import { ProgressCallback, reportStreamProgress } from "./commandRunner";
+import {
+  ProgressCallback,
+  reportStreamProgress,
+} from "../common/vscode/progress";
 import {
   codeQlLauncherName,
   deprecatedCodeQlLauncherName,
   extractZipArchive,
   getRequiredAssetName,
-} from "./pure/distribution";
+} from "../pure/distribution";
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+} from "../common/invocation-rate-limiter";
 
 /**
  * distribution.ts
@@ -73,7 +75,7 @@ export class DistributionManager implements DistributionProvider {
         extensionContext,
       );
     this.updateCheckRateLimiter = new InvocationRateLimiter(
-      extensionContext,
+      extensionContext.globalState,
       "extensionSpecificDistributionUpdateCheck",
       () =>
         this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),
@@ -215,6 +217,9 @@ export class DistributionManager implements DistributionProvider {
     minSecondsSinceLastUpdateCheck: number,
   ): Promise<DistributionUpdateCheckResult> {
     const distribution = await this.getDistributionWithoutVersionCheck();
+    if (distribution === undefined) {
+      minSecondsSinceLastUpdateCheck = 0;
+    }
     const extensionManagedCodeQlPath =
       await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
@@ -315,6 +320,15 @@ class ExtensionSpecificDistributionManager {
     const extensionSpecificRelease = this.getInstalledRelease();
     const latestRelease = await this.getLatestRelease();
 
+    // v2.12.3 was released with a bug that causes the extension to fail
+    // so we force the extension to ignore it.
+    if (
+      extensionSpecificRelease &&
+      extensionSpecificRelease.name === "v2.12.3"
+    ) {
+      return createUpdateAvailableResult(latestRelease);
+    }
+
     if (
       extensionSpecificRelease !== undefined &&
       codeQlPath !== undefined &&
@@ -430,6 +444,12 @@ class ExtensionSpecificDistributionManager {
       this.versionRange,
       this.config.includePrerelease,
       (release) => {
+        // v2.12.3 was released with a bug that causes the extension to fail
+        // so we force the extension to ignore it.
+        if (release.name === "v2.12.3") {
+          return false;
+        }
+
         const matchingAssets = release.assets.filter(
           (asset) => asset.name === requiredAssetName,
         );
@@ -484,7 +504,7 @@ class ExtensionSpecificDistributionManager {
         0,
       ) || "";
     return join(
-      this.extensionContext.globalStoragePath,
+      this.extensionContext.globalStorageUri.fsPath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
         distributionFolderIndex,
     );
@@ -649,7 +669,7 @@ export class ReleasesApiConsumer {
       redirectUrl &&
       redirectCount < ReleasesApiConsumer._maxRedirects
     ) {
-      const parsedRedirectUrl = parse(redirectUrl);
+      const parsedRedirectUrl = new URL(redirectUrl);
       if (parsedRedirectUrl.protocol !== "https:") {
         throw new Error("Encountered a non-https redirect, rejecting");
       }

extensions/ql-vscode/src/commandRunner.ts

@@ -1,268 +0,0 @@
-import {
-  CancellationToken,
-  ProgressOptions,
-  window as Window,
-  commands,
-  Disposable,
-  ProgressLocation,
-} from "vscode";
-import {
-  showAndLogExceptionWithTelemetry,
-  showAndLogWarningMessage,
-} from "./helpers";
-import { extLogger } from "./common";
-import { asError, getErrorMessage, getErrorStack } from "./pure/helpers-pure";
-import { telemetryListener } from "./telemetry";
-import { redactableError } from "./pure/errors";
-
-export class UserCancellationException extends Error {
-  /**
-   * @param message The error message
-   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
-   */
-  constructor(message?: string, public readonly silent = false) {
-    super(message);
-  }
-}
-
-export interface ProgressUpdate {
-  /**
-   * The current step
-   */
-  step: number;
-  /**
-   * The maximum step. This *should* be constant for a single job.
-   */
-  maxStep: number;
-  /**
-   * The current progress message
-   */
-  message: string;
-}
-
-export type ProgressCallback = (p: ProgressUpdate) => void;
-
-/**
- * A task that handles command invocations from `commandRunner`
- * and includes a progress monitor.
- *
- *
- * Arguments passed to the command handler are passed along,
- * untouched to this `ProgressTask` instance.
- *
- * @param progress a progress handler function. Call this
- * function with a `ProgressUpdate` instance in order to
- * denote some progress being achieved on this task.
- * @param token a cencellation token
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-export type ProgressTask<R> = (
-  progress: ProgressCallback,
-  token: CancellationToken,
-  ...args: any[]
-) => Thenable<R>;
-
-/**
- * A task that handles command invocations from `commandRunner`.
- * Arguments passed to the command handler are passed along,
- * untouched to this `NoProgressTask` instance.
- *
- * @param args arguments passed to this task passed on from
- * `commands.registerCommand`.
- */
-type NoProgressTask = (...args: any[]) => Promise<any>;
-
-/**
- * This mediates between the kind of progress callbacks we want to
- * write (where we *set* current progress position and give
- * `maxSteps`) and the kind vscode progress api expects us to write
- * (which increment progress by a certain amount out of 100%).
- *
- * Where possible, the `commandRunner` function below should be used
- * instead of this function. The commandRunner is meant for wrapping
- * top-level commands and provides error handling and other support
- * automatically.
- *
- * Only use this function if you need a progress monitor and the
- * control flow does not always come from a command (eg- during
- * extension activation, or from an internal language server
- * request).
- */
-export function withProgress<R>(
-  options: ProgressOptions,
-  task: ProgressTask<R>,
-  ...args: any[]
-): Thenable<R> {
-  let progressAchieved = 0;
-  return Window.withProgress(options, (progress, token) => {
-    return task(
-      (p) => {
-        const { message, step, maxStep } = p;
-        const increment = (100 * (step - progressAchieved)) / maxStep;
-        progressAchieved = step;
-        progress.report({ message, increment });
-      },
-      token,
-      ...args,
-    );
-  });
-}
-
-/**
- * A generic wrapper for command registration. This wrapper adds uniform error handling for commands.
- *
- * In this variant of the command runner, no progress monitor is used.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task.
- */
-export function commandRunner(
-  commandId: string,
-  task: NoProgressTask,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const startTime = Date.now();
-    let error: Error | undefined;
-
-    try {
-      return await task(...args);
-    } catch (e) {
-      error = asError(e);
-      const errorMessage = redactableError(error)`${
-        getErrorMessage(e) || e
-      } (${commandId})`;
-      const errorStack = getErrorStack(e);
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          void extLogger.log(errorMessage.fullMessage);
-        } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage);
-        }
-      } else {
-        // Include the full stack in the error log only.
-        const fullMessage = errorStack
-          ? `${errorMessage.fullMessage}\n${errorStack}`
-          : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      }
-      return undefined;
-    } finally {
-      const executionTime = Date.now() - startTime;
-      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
-    }
-  });
-}
-
-/**
- * A generic wrapper for command registration.  This wrapper adds uniform error handling,
- * progress monitoring, and cancellation for commands.
- *
- * @param commandId The ID of the command to register.
- * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
- * arguments to the command handler are passed on to the task after the progress callback
- * and cancellation token.
- * @param progressOptions Progress options to be sent to the progress monitor.
- */
-export function commandRunnerWithProgress<R>(
-  commandId: string,
-  task: ProgressTask<R>,
-  progressOptions: Partial<ProgressOptions>,
-  outputLogger = extLogger,
-): Disposable {
-  return commands.registerCommand(commandId, async (...args: any[]) => {
-    const startTime = Date.now();
-    let error: Error | undefined;
-    const progressOptionsWithDefaults = {
-      location: ProgressLocation.Notification,
-      ...progressOptions,
-    };
-    try {
-      return await withProgress(progressOptionsWithDefaults, task, ...args);
-    } catch (e) {
-      error = asError(e);
-      const errorMessage = redactableError`${
-        getErrorMessage(e) || e
-      } (${commandId})`;
-      const errorStack = getErrorStack(e);
-      if (e instanceof UserCancellationException) {
-        // User has cancelled this action manually
-        if (e.silent) {
-          void outputLogger.log(errorMessage.fullMessage);
-        } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage, {
-            outputLogger,
-          });
-        }
-      } else {
-        // Include the full stack in the error log only.
-        const fullMessage = errorStack
-          ? `${errorMessage.fullMessage}\n${errorStack}`
-          : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          outputLogger,
-          fullMessage,
-          extraTelemetryProperties: {
-            command: commandId,
-          },
-        });
-      }
-      return undefined;
-    } finally {
-      const executionTime = Date.now() - startTime;
-      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
-    }
-  });
-}
-
-/**
- * Displays a progress monitor that indicates how much progess has been made
- * reading from a stream.
- *
- * @param readable The stream to read progress from
- * @param messagePrefix A prefix for displaying the message
- * @param totalNumBytes Total number of bytes in this stream
- * @param progress The progress callback used to set messages
- */
-export function reportStreamProgress(
-  readable: NodeJS.ReadableStream,
-  messagePrefix: string,
-  totalNumBytes?: number,
-  progress?: ProgressCallback,
-) {
-  if (progress && totalNumBytes) {
-    let numBytesDownloaded = 0;
-    const bytesToDisplayMB = (numBytes: number): string =>
-      `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
-    const updateProgress = () => {
-      progress({
-        step: numBytesDownloaded,
-        maxStep: totalNumBytes,
-        message: `${messagePrefix} [${bytesToDisplayMB(
-          numBytesDownloaded,
-        )} of ${bytesToDisplayMB(totalNumBytes)}]`,
-      });
-    };
-
-    // Display the progress straight away rather than waiting for the first chunk.
-    updateProgress();
-
-    readable.on("data", (data) => {
-      numBytesDownloaded += data.length;
-      updateProgress();
-    });
-  } else if (progress) {
-    progress({
-      step: 1,
-      maxStep: 2,
-      message: `${messagePrefix} (Size unknown)`,
-    });
-  }
-}

extensions/ql-vscode/src/common/app.ts

@@ -3,10 +3,15 @@ import { Disposable } from "../pure/disposable-object";
 import { AppEventEmitter } from "./events";
 import { Logger } from "./logging";
 import { Memento } from "./memento";
+import { AppCommandManager } from "./commands";
+import type {
+  WorkspaceFolder,
+  Event,
+  WorkspaceFoldersChangeEvent,
+} from "vscode";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
-  executeCommand(command: string, ...args: any): Thenable<void>;
   readonly mode: AppMode;
   readonly logger: Logger;
   readonly subscriptions: Disposable[];
@@ -14,7 +19,11 @@ export interface App {
   readonly globalStoragePath: string;
   readonly workspaceStoragePath?: string;
   readonly workspaceState: Memento;
+  readonly workspaceFolders: readonly WorkspaceFolder[] | undefined;
+  readonly onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>;
   readonly credentials: Credentials;
+  readonly commands: AppCommandManager;
+  readonly environment: EnvironmentContext;
 }
 
 export enum AppMode {
@@ -22,3 +31,7 @@ export enum AppMode {
   Development = 2,
   Test = 3,
 }
+
+export interface EnvironmentContext {
+  language: string;
+}

extensions/ql-vscode/src/common/commands.ts

@@ -0,0 +1,355 @@
+import type { CommandManager } from "../packages/commands";
+import type { Uri, Range, TextDocumentShowOptions } from "vscode";
+import type { AstItem } from "../language-support";
+import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
+import type { DatabaseItem } from "../databases/local-databases";
+import type { QueryHistoryInfo } from "../query-history/query-history-info";
+import type { RepositoriesFilterSortStateWithIds } from "../pure/variant-analysis-filter-sort";
+import type { TestTreeNode } from "../query-testing/test-tree-node";
+import type {
+  VariantAnalysis,
+  VariantAnalysisScannedRepository,
+  VariantAnalysisScannedRepositoryResult,
+} from "../variant-analysis/shared/variant-analysis";
+import type { QLDebugConfiguration } from "../debugger/debug-configuration";
+
+// A command function matching the signature that VS Code calls when
+// a command is invoked from a context menu on a TreeView with
+// canSelectMany set to true.
+//
+// singleItem will always be defined and corresponds to the item that
+// was hovered or right-clicked. If precisely one item was selected then
+// multiSelect will be undefined. If more than one item is selected then
+// multiSelect will contain all selected items, including singleItem.
+export type TreeViewContextMultiSelectionCommandFunction<Item> = (
+  singleItem: Item,
+  multiSelect: Item[] | undefined,
+) => Promise<void>;
+
+// A command function matching the signature that VS Code calls when
+// a command is invoked from a context menu on a TreeView with
+// canSelectMany set to false.
+//
+// It is guaranteed that precisely one item will be selected.
+export type TreeViewContextSingleSelectionCommandFunction<Item> = (
+  singleItem: Item,
+) => Promise<void>;
+
+// A command function matching the signature that VS Code calls when
+// a command is invoked from a context menu on the file explorer.
+//
+// singleItem corresponds to the item that was right-clicked.
+// multiSelect will always been defined and non-empty and contains
+// all selected items, including singleItem.
+export type ExplorerSelectionCommandFunction<Item> = (
+  singleItem: Item,
+  multiSelect: Item[],
+) => Promise<void>;
+
+/**
+ * Contains type definitions for all commands used by the extension.
+ *
+ * To add a new command first define its type here, then provide
+ * the implementation in the corresponding `getCommands` function.
+ */
+
+// Builtin commands where the implementation is provided by VS Code and not by this extension.
+// See https://code.visualstudio.com/api/references/commands
+export type BuiltInVsCodeCommands = {
+  // The codeQLDatabases.focus command is provided by VS Code because we've registered the custom view
+  "codeQLDatabases.focus": () => Promise<void>;
+  "markdown.showPreviewToSide": (uri: Uri) => Promise<void>;
+  revealFileInOS: (uri: Uri) => Promise<void>;
+  setContext: (
+    key: `${"codeql" | "codeQL"}${string}`,
+    value: unknown,
+  ) => Promise<void>;
+  "workbench.action.reloadWindow": () => Promise<void>;
+  "vscode.diff": (
+    leftSideResource: Uri,
+    rightSideResource: Uri,
+    title?: string,
+    columnOrOptions?: TextDocumentShowOptions,
+  ) => Promise<void>;
+  "vscode.open": (uri: Uri) => Promise<void>;
+  "vscode.openFolder": (uri: Uri) => Promise<void>;
+  revealInExplorer: (uri: Uri) => Promise<void>;
+  // We type the `config` property specifically as a CodeQL debug configuration, since that's the
+  // only kinds we specify anyway.
+  "workbench.action.debug.start": (options?: {
+    config?: Partial<QLDebugConfiguration>;
+    noDebug?: boolean;
+  }) => Promise<void>;
+  "workbench.action.debug.stepInto": () => Promise<void>;
+  "workbench.action.debug.stepOver": () => Promise<void>;
+  "workbench.action.debug.stepOut": () => Promise<void>;
+};
+
+// Commands that are available before the extension is fully activated.
+// These commands are *not* registered using the command manager, but can
+// be invoked using the command manager.
+export type PreActivationCommands = {
+  "codeQL.checkForUpdatesToCLI": () => Promise<void>;
+};
+
+// Base commands not tied directly to a module like e.g. variant analysis.
+export type BaseCommands = {
+  "codeQL.openDocumentation": () => Promise<void>;
+  "codeQL.showLogs": () => Promise<void>;
+  "codeQL.authenticateToGitHub": () => Promise<void>;
+
+  "codeQL.copyVersion": () => Promise<void>;
+  "codeQL.restartQueryServer": () => Promise<void>;
+  "codeQL.restartQueryServerOnConfigChange": () => Promise<void>;
+  "codeQL.restartLegacyQueryServerOnConfigChange": () => Promise<void>;
+  "codeQL.restartQueryServerOnExternalConfigChange": () => Promise<void>;
+};
+
+// Commands used when working with queries in the editor
+export type QueryEditorCommands = {
+  "codeQL.openReferencedFile": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.openReferencedFileContextEditor": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
+  "codeQL.openReferencedFileContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
+  "codeQL.previewQueryHelp": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextEditor": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
+};
+
+// Commands used for running local queries
+export type LocalQueryCommands = {
+  "codeQL.runQuery": (uri?: Uri) => Promise<void>;
+  "codeQL.runQueryContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQL.runQueryOnMultipleDatabases": (uri?: Uri) => Promise<void>;
+  "codeQL.runQueryOnMultipleDatabasesContextEditor": (
+    uri?: Uri,
+  ) => Promise<void>;
+  "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
+  "codeQL.quickEval": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
+  "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
+  "codeQL.quickQuery": () => Promise<void>;
+  "codeQL.getCurrentQuery": () => Promise<string>;
+  "codeQL.createQuery": () => Promise<void>;
+};
+
+// Debugger commands
+export type DebuggerCommands = {
+  "codeQL.debugQuery": (uri: Uri | undefined) => Promise<void>;
+  "codeQL.debugQueryContextEditor": (uri: Uri) => Promise<void>;
+  "codeQL.startDebuggingSelection": () => Promise<void>;
+  "codeQL.startDebuggingSelectionContextEditor": () => Promise<void>;
+  "codeQL.continueDebuggingSelection": () => Promise<void>;
+  "codeQL.continueDebuggingSelectionContextEditor": () => Promise<void>;
+};
+
+export type ResultsViewCommands = {
+  "codeQLQueryResults.up": () => Promise<void>;
+  "codeQLQueryResults.down": () => Promise<void>;
+  "codeQLQueryResults.left": () => Promise<void>;
+  "codeQLQueryResults.right": () => Promise<void>;
+  "codeQLQueryResults.nextPathStep": () => Promise<void>;
+  "codeQLQueryResults.previousPathStep": () => Promise<void>;
+};
+
+// Commands used for the query history panel
+export type QueryHistoryCommands = {
+  // Commands in the "navigation" group
+  "codeQLQueryHistory.sortByName": () => Promise<void>;
+  "codeQLQueryHistory.sortByDate": () => Promise<void>;
+  "codeQLQueryHistory.sortByCount": () => Promise<void>;
+
+  // Commands in the context menu or in the hover menu
+  "codeQLQueryHistory.openQueryContextMenu": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.removeHistoryItemContextMenu": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showQueryLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.showQueryText": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.openQueryDirectory": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.cancel": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.exportResults": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewCsvResults": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewCsvAlerts": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewSarifAlerts": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.viewDil": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.itemClicked": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.openOnGithub": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.copyRepoList": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+
+  // Commands in the command palette
+  "codeQL.exportSelectedVariantAnalysisResults": () => Promise<void>;
+};
+
+// Commands used for the local databases panel
+export type LocalDatabasesCommands = {
+  // Command palette commands
+  "codeQL.chooseDatabaseFolder": () => Promise<void>;
+  "codeQL.chooseDatabaseArchive": () => Promise<void>;
+  "codeQL.chooseDatabaseInternet": () => Promise<void>;
+  "codeQL.chooseDatabaseGithub": () => Promise<void>;
+  "codeQL.upgradeCurrentDatabase": () => Promise<void>;
+  "codeQL.clearCache": () => Promise<void>;
+
+  // Explorer context menu
+  "codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
+
+  // Database panel view title commands
+  "codeQLDatabases.chooseDatabaseFolder": () => Promise<void>;
+  "codeQLDatabases.chooseDatabaseArchive": () => Promise<void>;
+  "codeQLDatabases.chooseDatabaseInternet": () => Promise<void>;
+  "codeQLDatabases.chooseDatabaseGithub": () => Promise<void>;
+  "codeQLDatabases.sortByName": () => Promise<void>;
+  "codeQLDatabases.sortByDateAdded": () => Promise<void>;
+
+  // Database panel context menu
+  "codeQLDatabases.setCurrentDatabase": (
+    databaseItem: DatabaseItem,
+  ) => Promise<void>;
+
+  // Database panel selection commands
+  "codeQLDatabases.removeDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.upgradeDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.renameDatabase": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.openDatabaseFolder": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+  "codeQLDatabases.addDatabaseSource": TreeViewContextMultiSelectionCommandFunction<DatabaseItem>;
+
+  // Codespace template commands
+  "codeQL.setDefaultTourDatabase": () => Promise<void>;
+
+  // Internal commands
+  "codeQLDatabases.removeOrphanedDatabases": () => Promise<void>;
+  "codeQL.getCurrentDatabase": () => Promise<string | undefined>;
+};
+
+// Commands tied to variant analysis
+export type VariantAnalysisCommands = {
+  "codeQL.autoDownloadVariantAnalysisResult": (
+    scannedRepo: VariantAnalysisScannedRepository,
+    variantAnalysisSummary: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.copyVariantAnalysisRepoList": (
+    variantAnalysisId: number,
+    filterSort?: RepositoriesFilterSortStateWithIds,
+  ) => Promise<void>;
+  "codeQL.loadVariantAnalysisRepoResults": (
+    variantAnalysisId: number,
+    repositoryFullName: string,
+  ) => Promise<VariantAnalysisScannedRepositoryResult>;
+  "codeQL.monitorNewVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.monitorRehydratedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.monitorReauthenticatedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
+  "codeQL.openVariantAnalysisLogs": (
+    variantAnalysisId: number,
+  ) => Promise<void>;
+  "codeQL.openVariantAnalysisView": (
+    variantAnalysisId: number,
+  ) => Promise<void>;
+  "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
+  "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+};
+
+export type DatabasePanelCommands = {
+  "codeQLVariantAnalysisRepositories.openConfigFile": () => Promise<void>;
+  "codeQLVariantAnalysisRepositories.addNewDatabase": () => Promise<void>;
+  "codeQLVariantAnalysisRepositories.addNewList": () => Promise<void>;
+  "codeQLVariantAnalysisRepositories.setupControllerRepository": () => Promise<void>;
+
+  "codeQLVariantAnalysisRepositories.setSelectedItem": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+};
+
+export type AstCfgCommands = {
+  "codeQL.viewAst": (selectedFile: Uri) => Promise<void>;
+  "codeQL.viewAstContextExplorer": (selectedFile: Uri) => Promise<void>;
+  "codeQL.viewAstContextEditor": (selectedFile: Uri) => Promise<void>;
+  "codeQL.viewCfg": () => Promise<void>;
+  "codeQL.viewCfgContextExplorer": () => Promise<void>;
+  "codeQL.viewCfgContextEditor": () => Promise<void>;
+};
+
+export type AstViewerCommands = {
+  "codeQLAstViewer.clear": () => Promise<void>;
+  "codeQLAstViewer.gotoCode": (item: AstItem) => Promise<void>;
+};
+
+export type PackagingCommands = {
+  "codeQL.installPackDependencies": () => Promise<void>;
+  "codeQL.downloadPacks": () => Promise<void>;
+};
+
+export type DataExtensionsEditorCommands = {
+  "codeQL.openDataExtensionsEditor": () => Promise<void>;
+};
+
+export type EvalLogViewerCommands = {
+  "codeQLEvalLogViewer.clear": () => Promise<void>;
+};
+
+export type SummaryLanguageSupportCommands = {
+  "codeQL.gotoQL": () => Promise<void>;
+  "codeQL.gotoQLContextEditor": () => Promise<void>;
+};
+
+export type TestUICommands = {
+  "codeQLTests.showOutputDifferences": (node: TestTreeNode) => Promise<void>;
+  "codeQLTests.acceptOutput": (node: TestTreeNode) => Promise<void>;
+  "codeQLTests.acceptOutputContextTestItem": (
+    node: TestTreeNode,
+  ) => Promise<void>;
+};
+
+export type MockGitHubApiServerCommands = {
+  "codeQL.mockGitHubApiServer.startRecording": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.saveScenario": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.cancelRecording": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.loadScenario": () => Promise<void>;
+  "codeQL.mockGitHubApiServer.unloadScenario": () => Promise<void>;
+};
+
+// All commands where the implementation is provided by this activated extension.
+export type AllExtensionCommands = BaseCommands &
+  QueryEditorCommands &
+  ResultsViewCommands &
+  QueryHistoryCommands &
+  LocalDatabasesCommands &
+  DebuggerCommands &
+  VariantAnalysisCommands &
+  DatabasePanelCommands &
+  AstCfgCommands &
+  AstViewerCommands &
+  PackagingCommands &
+  DataExtensionsEditorCommands &
+  EvalLogViewerCommands &
+  SummaryLanguageSupportCommands &
+  Partial<TestUICommands> &
+  MockGitHubApiServerCommands;
+
+export type AllCommands = AllExtensionCommands &
+  PreActivationCommands &
+  BuiltInVsCodeCommands;
+
+export type AppCommandManager = CommandManager<AllCommands>;
+
+// Separate command manager because it uses a different logger
+export type QueryServerCommands = LocalQueryCommands;
+export type QueryServerCommandManager = CommandManager<QueryServerCommands>;

extensions/ql-vscode/src/common/discovery.ts

@@ -1,6 +1,6 @@
-import { DisposableObject } from "./pure/disposable-object";
-import { extLogger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { DisposableObject } from "../pure/disposable-object";
+import { getErrorMessage } from "../pure/helpers-pure";
+import { Logger } from "./logging";
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
@@ -8,18 +8,28 @@ import { getErrorMessage } from "./pure/helpers-pure";
  * same time.
  */
 export abstract class Discovery<T> extends DisposableObject {
-  private retry = false;
-  private discoveryInProgress = false;
+  private restartWhenFinished = false;
+  private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string) {
+  constructor(private readonly name: string, private readonly logger: Logger) {
     super();
   }
 
+  /**
+   * Returns the promise of the currently running refresh operation, if one is in progress.
+   * Otherwise returns a promise that resolves immediately.
+   */
+  public waitForCurrentRefresh(): Promise<void> {
+    return this.currentDiscoveryPromise ?? Promise.resolve();
+  }
+
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
+   *
+   * Returns a promise that resolves when the refresh is complete, including any retries.
    */
-  public refresh(): void {
+  public refresh(): Promise<void> {
     // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
     // got a storm of refresh requests due to, say, the copying or deletion of a large directory
     // tree, we could potentially spawn a separate simultaneous discovery operation for each
@@ -36,14 +46,16 @@ export abstract class Discovery<T> extends DisposableObject {
     // other change notifications that might be coming along. However, this would create more
     // latency in the common case, in order to save a bit of latency in the uncommon case.
 
-    if (this.discoveryInProgress) {
+    if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
-      this.retry = true;
+      this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
-      this.discoveryInProgress = true;
-      this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery().finally(() => {
+        this.currentDiscoveryPromise = undefined;
+      });
     }
+    return this.currentDiscoveryPromise;
   }
 
   /**
@@ -51,34 +63,31 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery operation completes, the `update` function will be invoked with the results of the
    * discovery.
    */
-  private launchDiscovery(): void {
-    const discoveryPromise = this.discover();
-    discoveryPromise
-      .then((results) => {
-        if (!this.retry) {
-          // Update any listeners with the results of the discovery.
-          this.discoveryInProgress = false;
-          this.update(results);
-        }
-      })
+  private async launchDiscovery(): Promise<void> {
+    let results: T | undefined;
+    try {
+      results = await this.discover();
+    } catch (err) {
+      void this.logger.log(
+        `${this.name} failed. Reason: ${getErrorMessage(err)}`,
+      );
+      results = undefined;
+    }
 
-      .catch((err: unknown) => {
-        void extLogger.log(
-          `${this.name} failed. Reason: ${getErrorMessage(err)}`,
-        );
-      })
-
-      .finally(() => {
-        if (this.retry) {
-          // Another refresh request came in while we were still running a previous discovery
-          // operation. Since the discovery results we just computed are now stale, we'll launch
-          // another discovery operation instead of updating.
-          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-          // initial discovery operation failed.
-          this.retry = false;
-          this.launchDiscovery();
-        }
-      });
+    if (this.restartWhenFinished) {
+      // Another refresh request came in while we were still running a previous discovery
+      // operation. Since the discovery results we just computed are now stale, we'll launch
+      // another discovery operation instead of updating.
+      // We want to relaunch discovery regardless of if the initial discovery operation
+      // succeeded or failed.
+      this.restartWhenFinished = false;
+      await this.launchDiscovery();
+    } else {
+      // If the discovery was successful, then update any listeners with the results.
+      if (results !== undefined) {
+        this.update(results);
+      }
+    }
   }
 
   /**

extensions/ql-vscode/src/common/events.ts

@@ -4,7 +4,7 @@ export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;
 }
 
-export interface AppEventEmitter<T> {
+export interface AppEventEmitter<T> extends Disposable {
   event: AppEvent<T>;
   fire(data: T): void;
 }

extensions/ql-vscode/src/common/file-tree-nodes.ts

@@ -0,0 +1,122 @@
+import { basename, dirname, join } from "path";
+import { EnvironmentContext } from "./app";
+
+/**
+ * A node in the tree of files. This will be either a `FileTreeDirectory` or a `FileTreeLeaf`.
+ */
+export abstract class FileTreeNode<T = undefined> {
+  constructor(
+    private _path: string,
+    private _name: string,
+    private _data?: T,
+  ) {}
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public get data(): T | undefined {
+    return this._data;
+  }
+
+  public abstract get children(): ReadonlyArray<FileTreeNode<T>>;
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more files or other directories.
+ */
+export class FileTreeDirectory<T = undefined> extends FileTreeNode<T> {
+  constructor(
+    _path: string,
+    _name: string,
+    protected readonly env: EnvironmentContext,
+    private _children: Array<FileTreeNode<T>> = [],
+  ) {
+    super(_path, _name);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return this._children;
+  }
+
+  public addChild(child: FileTreeNode<T>): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): FileTreeDirectory<T> {
+    if (relativePath === ".") {
+      return this;
+    }
+    const dirName = dirname(relativePath);
+    if (dirName === ".") {
+      return this.createChildDirectory(relativePath);
+    } else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    // remove empty directories
+    this._children.filter(
+      (child) => child instanceof FileTreeLeaf || child.children.length > 0,
+    );
+    this._children.sort((a, b) =>
+      a.name.localeCompare(b.name, this.env.language),
+    );
+    this._children.forEach((child, i) => {
+      child.finish();
+      if (
+        child.children?.length === 1 &&
+        child.children[0] instanceof FileTreeDirectory
+      ) {
+        // collapse children
+        const replacement = new FileTreeDirectory<T>(
+          child.children[0].path,
+          `${child.name} / ${child.children[0].name}`,
+          this.env,
+          Array.from(child.children[0].children),
+        );
+        this._children[i] = replacement;
+      }
+    });
+  }
+
+  private createChildDirectory(name: string): FileTreeDirectory<T> {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return existingChild as FileTreeDirectory<T>;
+    } else {
+      const newChild = new FileTreeDirectory<T>(
+        join(this.path, name),
+        name,
+        this.env,
+      );
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single file.
+ */
+export class FileTreeLeaf<T = undefined> extends FileTreeNode<T> {
+  constructor(_path: string, _name: string, _data?: T) {
+    super(_path, _name, _data);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return [];
+  }
+
+  public finish(): void {
+    /**/
+  }
+}

extensions/ql-vscode/src/common/invocation-rate-limiter.ts

@@ -0,0 +1,89 @@
+import { Memento } from "./memento";
+
+/**
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
+ */
+export class InvocationRateLimiter<T> {
+  constructor(
+    private readonly globalState: Memento,
+    private readonly funcIdentifier: string,
+    private readonly func: () => Promise<T>,
+    private readonly createDate: (dateString?: string) => Date = (s) =>
+      s ? new Date(s) : new Date(),
+  ) {}
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(
+    minSecondsSinceLastInvocation: number,
+  ): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this.createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
+        updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this.func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined = this.globalState.get(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+    );
+    return maybeDateString ? this.createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this.globalState.update(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+      date,
+    );
+  }
+
+  private static readonly _invocationRateLimiterPrefix =
+    "invocationRateLimiter_lastInvocationDate_";
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited,
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result,
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited,
+  };
+}

extensions/ql-vscode/src/common/logging/index.ts

@@ -1,3 +1,4 @@
 export * from "./logger";
+export * from "./tee-logger";
 export * from "./vscode/loggers";
 export * from "./vscode/output-channel-logger";

extensions/ql-vscode/src/common/logging/logger.ts

@@ -1,12 +1,10 @@
 export interface LogOptions {
   // If false, don't output a trailing newline for the log entry. Default true.
   trailingNewline?: boolean;
-
-  // If specified, add this log entry to the log file at the specified location.
-  additionalLogLocation?: string;
 }
 
-export interface Logger {
+/** Minimal logger interface. */
+export interface BaseLogger {
   /**
    * Writes the given log message, optionally followed by a newline.
    * This function is asynchronous and will only resolve once the message is written
@@ -18,18 +16,14 @@ export interface Logger {
    * @param options Optional settings.
    */
   log(message: string, options?: LogOptions): Promise<void>;
+}
 
+/** Full logger interface, including a function to show the log in the UI. */
+export interface Logger extends BaseLogger {
   /**
    * Reveal the logger channel in the UI.
    *
    * @param preserveFocus When `true` the channel will not take focus.
    */
   show(preserveFocus?: boolean): void;
-
-  /**
-   * Remove the log at the specified location.
-   *
-   * @param location log to remove
-   */
-  removeAdditionalLogLocation(location: string | undefined): void;
 }

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -0,0 +1,68 @@
+import { appendFile, ensureFile } from "fs-extra";
+import { isAbsolute } from "path";
+import { getErrorMessage } from "../../pure/helpers-pure";
+import { Logger, LogOptions } from "./logger";
+
+/**
+ * An implementation of {@link Logger} that sends the output both to another {@link Logger}
+ * and to a file.
+ *
+ * The first time a message is written, an additional banner is written to the underlying logger
+ * pointing the user to the "side log" file.
+ */
+export class TeeLogger implements Logger {
+  private emittedRedirectMessage = false;
+  private error = false;
+
+  public constructor(
+    private readonly logger: Logger,
+    private readonly location: string,
+  ) {
+    if (!isAbsolute(location)) {
+      throw new Error(
+        `Additional Log Location must be an absolute path: ${location}`,
+      );
+    }
+  }
+
+  async log(message: string, options = {} as LogOptions): Promise<void> {
+    if (!this.emittedRedirectMessage) {
+      this.emittedRedirectMessage = true;
+      const msg = `| Log being saved to ${this.location} |`;
+      const separator = new Array(msg.length).fill("-").join("");
+      await this.logger.log(separator);
+      await this.logger.log(msg);
+      await this.logger.log(separator);
+    }
+
+    if (!this.error) {
+      try {
+        const trailingNewline = options.trailingNewline ?? true;
+        await ensureFile(this.location);
+
+        await appendFile(
+          this.location,
+          message + (trailingNewline ? "\n" : ""),
+          {
+            encoding: "utf8",
+          },
+        );
+      } catch (e) {
+        // Write an error message to the primary log, and stop trying to write to the side log.
+        this.error = true;
+        const errorMessage = getErrorMessage(e);
+        await this.logger.log(
+          `Error writing to additional log file: ${errorMessage}`,
+        );
+      }
+    }
+
+    if (!this.error) {
+      await this.logger.log(message, options);
+    }
+  }
+
+  show(preserveFocus?: boolean): void {
+    this.logger.show(preserveFocus);
+  }
+}

extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts

@@ -1,6 +1,4 @@
 import { window as Window, OutputChannel, Progress } from "vscode";
-import { ensureFile, appendFile } from "fs-extra";
-import { isAbsolute } from "path";
 import { Logger, LogOptions } from "../logger";
 import { DisposableObject } from "../../../pure/disposable-object";
 
@@ -9,10 +7,6 @@ import { DisposableObject } from "../../../pure/disposable-object";
  */
 export class OutputChannelLogger extends DisposableObject implements Logger {
   public readonly outputChannel: OutputChannel;
-  private readonly additionalLocations = new Map<
-    string,
-    AdditionalLogLocation
-  >();
   isCustomLogDirectory: boolean;
 
   constructor(title: string) {
@@ -32,27 +26,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
       } else {
         this.outputChannel.append(message);
       }
-
-      if (options.additionalLogLocation) {
-        if (!isAbsolute(options.additionalLogLocation)) {
-          throw new Error(
-            `Additional Log Location must be an absolute path: ${options.additionalLogLocation}`,
-          );
-        }
-        const logPath = options.additionalLogLocation;
-        let additional = this.additionalLocations.get(logPath);
-        if (!additional) {
-          const msg = `| Log being saved to ${logPath} |`;
-          const separator = new Array(msg.length).fill("-").join("");
-          this.outputChannel.appendLine(separator);
-          this.outputChannel.appendLine(msg);
-          this.outputChannel.appendLine(separator);
-          additional = new AdditionalLogLocation(logPath);
-          this.additionalLocations.set(logPath, additional);
-        }
-
-        await additional.log(message, options);
-      }
     } catch (e) {
       if (e instanceof Error && e.message === "Channel has been closed") {
         // Output channel is closed logging to console instead
@@ -69,31 +42,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   show(preserveFocus?: boolean): void {
     this.outputChannel.show(preserveFocus);
   }
-
-  removeAdditionalLogLocation(location: string | undefined): void {
-    if (location) {
-      this.additionalLocations.delete(location);
-    }
-  }
-}
-
-class AdditionalLogLocation {
-  constructor(private location: string) {}
-
-  async log(message: string, options = {} as LogOptions): Promise<void> {
-    if (options.trailingNewline === undefined) {
-      options.trailingNewline = true;
-    }
-    await ensureFile(this.location);
-
-    await appendFile(
-      this.location,
-      message + (options.trailingNewline ? "\n" : ""),
-      {
-        encoding: "utf8",
-      },
-    );
-  }
 }
 
 export type ProgressReporter = Progress<{ message: string }>;

extensions/ql-vscode/src/common/sarif-parser.ts

@@ -1,7 +1,7 @@
 import * as Sarif from "sarif";
 import { createReadStream } from "fs-extra";
 import { connectTo } from "stream-json/Assembler";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { getErrorMessage } from "../pure/helpers-pure";
 import { withParser } from "stream-json/filters/Pick";
 
 const DUMMY_TOOL: Sarif.Tool = { driver: { name: "" } };

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -9,13 +9,13 @@ import {
 } from "vscode";
 import { join } from "path";
 
-import { DisposableObject, DisposeHandler } from "./pure/disposable-object";
-import { tmpDir } from "./helpers";
+import { DisposableObject, DisposeHandler } from "../../pure/disposable-object";
+import { tmpDir } from "../../helpers";
 import {
   getHtmlForWebview,
   WebviewMessage,
   WebviewView,
-} from "./interface-utils";
+} from "../../interface-utils";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -24,6 +24,7 @@ export type WebviewPanelConfig = {
   view: WebviewView;
   preserveFocus?: boolean;
   additionalOptions?: WebviewPanelOptions & WebviewOptions;
+  allowWasmEval?: boolean;
 };
 
 export abstract class AbstractWebview<
@@ -116,6 +117,7 @@ export abstract class AbstractWebview<
       config.view,
       {
         allowInlineStyles: true,
+        allowWasmEval: !!config.allowWasmEval,
       },
     );
     this.push(

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -1,7 +1,7 @@
 import { pathExists } from "fs-extra";
 import * as unzipper from "unzipper";
 import * as vscode from "vscode";
-import { extLogger } from "./common";
+import { extLogger } from "..";
 
 // All path operations in this file must be on paths *within* the zip
 // archive.

extensions/ql-vscode/src/common/vscode/authentication.ts

@@ -1,9 +1,9 @@
 import * as vscode from "vscode";
 import * as Octokit from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
-import { Credentials } from "./common/authentication";
+import { Credentials } from "../authentication";
 
-const GITHUB_AUTH_PROVIDER_ID = "github";
+export const GITHUB_AUTH_PROVIDER_ID = "github";
 
 // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist,
 // and 'read:packages' for reading private CodeQL packages.

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -0,0 +1,100 @@
+import { commands, Disposable } from "vscode";
+import { CommandFunction, CommandManager } from "../../packages/commands";
+import { extLogger, OutputChannelLogger } from "../logging";
+import {
+  asError,
+  getErrorMessage,
+  getErrorStack,
+} from "../../pure/helpers-pure";
+import { redactableError } from "../../pure/errors";
+import { UserCancellationException } from "./progress";
+import {
+  showAndLogExceptionWithTelemetry,
+  showAndLogWarningMessage,
+} from "../../helpers";
+import { telemetryListener } from "../../telemetry";
+
+/**
+ * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
+ * and vscode.executeCommand.
+ */
+export function createVSCodeCommandManager<
+  Commands extends Record<string, CommandFunction>,
+>(outputLogger?: OutputChannelLogger): CommandManager<Commands> {
+  return new CommandManager((commandId, task) => {
+    return registerCommandWithErrorHandling(commandId, task, outputLogger);
+  }, wrapExecuteCommand);
+}
+
+/**
+ * A wrapper for command registration. This wrapper adds uniform error handling for commands.
+ *
+ * @param commandId The ID of the command to register.
+ * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
+ * arguments to the command handler are passed on to the task.
+ */
+export function registerCommandWithErrorHandling(
+  commandId: string,
+  task: (...args: any[]) => Promise<any>,
+  outputLogger = extLogger,
+): Disposable {
+  return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTime = Date.now();
+    let error: Error | undefined;
+
+    try {
+      return await task(...args);
+    } catch (e) {
+      error = asError(e);
+      const errorMessage = redactableError(error)`${
+        getErrorMessage(e) || e
+      } (${commandId})`;
+      if (e instanceof UserCancellationException) {
+        // User has cancelled this action manually
+        if (e.silent) {
+          void outputLogger.log(errorMessage.fullMessage);
+        } else {
+          void showAndLogWarningMessage(errorMessage.fullMessage, {
+            outputLogger,
+          });
+        }
+      } else {
+        // Include the full stack in the error log only.
+        const errorStack = getErrorStack(e);
+        const fullMessage = errorStack
+          ? `${errorMessage.fullMessage}\n${errorStack}`
+          : errorMessage.fullMessage;
+        void showAndLogExceptionWithTelemetry(errorMessage, {
+          outputLogger,
+          fullMessage,
+          extraTelemetryProperties: {
+            command: commandId,
+          },
+        });
+      }
+      return undefined;
+    } finally {
+      const executionTime = Date.now() - startTime;
+      telemetryListener?.sendCommandUsage(commandId, executionTime, error);
+    }
+  });
+}
+
+/**
+ * wrapExecuteCommand wraps commands.executeCommand to satisfy that the
+ * type is a Promise. Type script does not seem to be smart enough
+ * to figure out that `ReturnType<Commands[CommandName]>` is actually
+ * a Promise, so we need to add a second layer of wrapping and unwrapping
+ * (The `Promise<Awaited<` part) to get the right types.
+ */
+async function wrapExecuteCommand<
+  Commands extends Record<string, CommandFunction>,
+  CommandName extends keyof Commands & string = keyof Commands & string,
+>(
+  commandName: CommandName,
+  ...args: Parameters<Commands[CommandName]>
+): Promise<Awaited<ReturnType<Commands[CommandName]>>> {
+  return await commands.executeCommand<
+    Awaited<ReturnType<Commands[CommandName]>>
+  >(commandName, ...args);
+}

extensions/ql-vscode/src/common/vscode/environment-context.ts

@@ -0,0 +1,8 @@
+import { env } from "vscode";
+import { EnvironmentContext } from "../app";
+
+export class AppEnvironmentContext implements EnvironmentContext {
+  public get language(): string {
+    return env.language;
+  }
+}

extensions/ql-vscode/src/common/vscode/external-files.ts

@@ -0,0 +1,54 @@
+import { Uri, window } from "vscode";
+import { AppCommandManager } from "../commands";
+import {
+  showAndLogExceptionWithTelemetry,
+  showBinaryChoiceDialog,
+} from "../../helpers";
+import { redactableError } from "../../pure/errors";
+import {
+  asError,
+  getErrorMessage,
+  getErrorStack,
+} from "../../pure/helpers-pure";
+
+export async function tryOpenExternalFile(
+  commandManager: AppCommandManager,
+  fileLocation: string,
+) {
+  const uri = Uri.file(fileLocation);
+  try {
+    await window.showTextDocument(uri, { preview: false });
+  } catch (e) {
+    const msg = getErrorMessage(e);
+    if (
+      msg.includes("Files above 50MB cannot be synchronized with extensions") ||
+      msg.includes("too large to open")
+    ) {
+      const res = await showBinaryChoiceDialog(
+        `VS Code does not allow extensions to open files >50MB. This file
+exceeds that limit. Do you want to open it outside of VS Code?
+
+You can also try manually opening it inside VS Code by selecting
+the file in the file explorer and dragging it into the workspace.`,
+      );
+      if (res) {
+        try {
+          await commandManager.execute("revealFileInOS", uri);
+        } catch (e) {
+          void showAndLogExceptionWithTelemetry(
+            redactableError(
+              asError(e),
+            )`Failed to reveal file in OS: ${getErrorMessage(e)}`,
+          );
+        }
+      }
+    } else {
+      void showAndLogExceptionWithTelemetry(
+        redactableError(asError(e))`Could not open file ${fileLocation}`,
+        {
+          fullMessage: `${getErrorMessage(e)}\n${getErrorStack(e)}`,
+        },
+      );
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../../pure/disposable-object";
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from "vscode";
 
 /**

extensions/ql-vscode/src/common/vscode/progress.ts

@@ -0,0 +1,151 @@
+import {
+  CancellationToken,
+  ProgressLocation,
+  ProgressOptions as VSCodeProgressOptions,
+  window as Window,
+} from "vscode";
+
+export class UserCancellationException extends Error {
+  /**
+   * @param message The error message
+   * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
+   */
+  constructor(message?: string, public readonly silent = false) {
+    super(message);
+  }
+}
+
+export interface ProgressUpdate {
+  /**
+   * The current step
+   */
+  step: number;
+  /**
+   * The maximum step. This *should* be constant for a single job.
+   */
+  maxStep: number;
+  /**
+   * The current progress message
+   */
+  message: string;
+}
+
+export type ProgressCallback = (p: ProgressUpdate) => void;
+
+// Make certain properties within a type optional
+type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
+
+export type ProgressOptions = Optional<VSCodeProgressOptions, "location">;
+
+/**
+ * A task that reports progress.
+ *
+ * @param progress a progress handler function. Call this
+ * function with a `ProgressUpdate` instance in order to
+ * denote some progress being achieved on this task.
+ * @param token a cancellation token
+ */
+export type ProgressTask<R> = (
+  progress: ProgressCallback,
+  token: CancellationToken,
+) => Thenable<R>;
+
+/**
+ * This mediates between the kind of progress callbacks we want to
+ * write (where we *set* current progress position and give
+ * `maxSteps`) and the kind vscode progress api expects us to write
+ * (which increment progress by a certain amount out of 100%).
+ */
+export function withProgress<R>(
+  task: ProgressTask<R>,
+  {
+    location = ProgressLocation.Notification,
+    title,
+    cancellable,
+  }: ProgressOptions = {},
+): Thenable<R> {
+  let progressAchieved = 0;
+  return Window.withProgress(
+    {
+      location,
+      title,
+      cancellable,
+    },
+    (progress, token) => {
+      return task((p) => {
+        const { message, step, maxStep } = p;
+        const increment = (100 * (step - progressAchieved)) / maxStep;
+        progressAchieved = step;
+        progress.report({ message, increment });
+      }, token);
+    },
+  );
+}
+
+export interface ProgressContext {
+  progress: ProgressCallback;
+  token: CancellationToken;
+}
+
+/**
+ * Like `withProgress()`, except that the caller is not required to provide a progress context. If
+ * the caller does provide one, any long-running operations performed by `task` will use the
+ * supplied progress context. Otherwise, this function wraps `task` in a new progress context with
+ * the supplied options.
+ */
+export function withInheritedProgress<R>(
+  parent: ProgressContext | undefined,
+  task: ProgressTask<R>,
+  options: ProgressOptions,
+): Thenable<R> {
+  if (parent !== undefined) {
+    return task(parent.progress, parent.token);
+  } else {
+    return withProgress(task, options);
+  }
+}
+
+/**
+ * Displays a progress monitor that indicates how much progess has been made
+ * reading from a stream.
+ *
+ * @param readable The stream to read progress from
+ * @param messagePrefix A prefix for displaying the message
+ * @param totalNumBytes Total number of bytes in this stream
+ * @param progress The progress callback used to set messages
+ */
+export function reportStreamProgress(
+  readable: NodeJS.ReadableStream,
+  messagePrefix: string,
+  totalNumBytes?: number,
+  progress?: ProgressCallback,
+) {
+  if (progress && totalNumBytes) {
+    let numBytesDownloaded = 0;
+    const bytesToDisplayMB = (numBytes: number): string =>
+      `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+    const updateProgress = () => {
+      progress({
+        step: numBytesDownloaded,
+        maxStep: totalNumBytes,
+        message: `${messagePrefix} [${bytesToDisplayMB(
+          numBytesDownloaded,
+        )} of ${bytesToDisplayMB(totalNumBytes)}]`,
+      });
+    };
+
+    // Display the progress straight away rather than waiting for the first chunk.
+    updateProgress();
+
+    readable.on("data", (data) => {
+      numBytesDownloaded += data.length;
+      updateProgress();
+    });
+  } else if (progress) {
+    progress({
+      step: 1,
+      maxStep: 2,
+      message: `${messagePrefix} (Size unknown)`,
+    });
+  }
+}

extensions/ql-vscode/src/common/vscode/selection-commands.ts

@@ -0,0 +1,51 @@
+import { showAndLogErrorMessage } from "../../helpers";
+import {
+  ExplorerSelectionCommandFunction,
+  TreeViewContextMultiSelectionCommandFunction,
+  TreeViewContextSingleSelectionCommandFunction,
+} from "../commands";
+
+// A hack to match types that are not an array, which is useful to help avoid
+// misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
+// as DatabaseItem[] instead of DatabaseItem.
+type NotArray = object & { length?: never };
+
+// A way to get the type system to help assert that one type is a supertype of another.
+type CreateSupertypeOf<Super, Sub extends Super> = Sub;
+
+// This asserts that SelectionCommand is assignable to all of the different types of
+// SelectionCommand defined in commands.ts. The intention is the output from the helpers
+// in this file can be used with any of the select command types and can handle any of
+// the inputs.
+type SelectionCommand<T extends NotArray> = CreateSupertypeOf<
+  TreeViewContextMultiSelectionCommandFunction<T> &
+    TreeViewContextSingleSelectionCommandFunction<T> &
+    ExplorerSelectionCommandFunction<T>,
+  (singleItem: T, multiSelect?: T[] | undefined) => Promise<void>
+>;
+
+export function createSingleSelectionCommand<T extends NotArray>(
+  f: (argument: T) => Promise<void>,
+  itemName: string,
+): SelectionCommand<T> {
+  return async (singleItem, multiSelect) => {
+    if (multiSelect === undefined || multiSelect.length === 1) {
+      return f(singleItem);
+    } else {
+      void showAndLogErrorMessage(`Please select a single ${itemName}.`);
+      return;
+    }
+  };
+}
+
+export function createMultiSelectionCommand<T extends NotArray>(
+  f: (argument: T[]) => Promise<void>,
+): SelectionCommand<T> {
+  return async (singleItem, multiSelect) => {
+    if (multiSelect !== undefined && multiSelect.length > 0) {
+      return f(multiSelect);
+    } else {
+      return f([singleItem]);
+    }
+  };
+}

extensions/ql-vscode/src/common/vscode/vscode-app.ts

@@ -1,19 +1,27 @@
 import * as vscode from "vscode";
-import { VSCodeCredentials } from "../../authentication";
+import { VSCodeCredentials } from "./authentication";
 import { Disposable } from "../../pure/disposable-object";
-import { App, AppMode } from "../app";
+import { App, AppMode, EnvironmentContext } from "../app";
 import { AppEventEmitter } from "../events";
-import { extLogger, Logger } from "../logging";
+import { extLogger, Logger, queryServerLogger } from "../logging";
 import { Memento } from "../memento";
 import { VSCodeAppEventEmitter } from "./events";
+import { AppCommandManager, QueryServerCommandManager } from "../commands";
+import { createVSCodeCommandManager } from "./commands";
+import { AppEnvironmentContext } from "./environment-context";
 
 export class ExtensionApp implements App {
   public readonly credentials: VSCodeCredentials;
+  public readonly commands: AppCommandManager;
+  public readonly queryServerCommands: QueryServerCommandManager;
 
   public constructor(
     public readonly extensionContext: vscode.ExtensionContext,
   ) {
     this.credentials = new VSCodeCredentials();
+    this.commands = createVSCodeCommandManager();
+    this.queryServerCommands = createVSCodeCommandManager(queryServerLogger);
+    extensionContext.subscriptions.push(this.commands);
   }
 
   public get extensionPath(): string {
@@ -32,6 +40,14 @@ export class ExtensionApp implements App {
     return this.extensionContext.workspaceState;
   }
 
+  public get workspaceFolders(): readonly vscode.WorkspaceFolder[] | undefined {
+    return vscode.workspace.workspaceFolders;
+  }
+
+  public get onDidChangeWorkspaceFolders(): vscode.Event<vscode.WorkspaceFoldersChangeEvent> {
+    return vscode.workspace.onDidChangeWorkspaceFolders;
+  }
+
   public get subscriptions(): Disposable[] {
     return this.extensionContext.subscriptions;
   }
@@ -55,7 +71,7 @@ export class ExtensionApp implements App {
     return new VSCodeAppEventEmitter<T>();
   }
 
-  public executeCommand(command: string, ...args: any): Thenable<void> {
-    return vscode.commands.executeCommand(command, ...args);
+  public get environment(): EnvironmentContext {
+    return new AppEnvironmentContext();
   }
 }

extensions/ql-vscode/src/compare/compare-view.ts

@@ -6,8 +6,8 @@ import {
   QueryCompareResult,
 } from "../pure/interface-types";
 import { Logger } from "../common";
-import { CodeQLCliServer } from "../cli";
-import { DatabaseManager } from "../local-databases";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { DatabaseManager } from "../databases/local-databases";
 import { jumpToLocation } from "../interface-utils";
 import {
   transformBqrsResultSet,
@@ -18,8 +18,13 @@ import resultsDiff from "./resultsDiff";
 import { CompletedLocalQueryInfo } from "../query-results";
 import { assertNever, getErrorMessage } from "../pure/helpers-pure";
 import { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
-import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
+import {
+  AbstractWebview,
+  WebviewPanelConfig,
+} from "../common/vscode/abstract-webview";
 import { telemetryListener } from "../telemetry";
+import { redactableError } from "../pure/errors";
+import { showAndLogExceptionWithTelemetry } from "../helpers";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
@@ -139,6 +144,14 @@ export class CompareView extends AbstractWebview<
         telemetryListener?.sendUIInteraction(msg.action);
         break;
 
+      case "unhandledError":
+        void showAndLogExceptionWithTelemetry(
+          redactableError(
+            msg.error,
+          )`Unhandled error in result comparison view: ${msg.error.message}`,
+        );
+        break;
+
       default:
         assertNever(msg);
     }
@@ -162,21 +175,40 @@ export class CompareView extends AbstractWebview<
     const commonResultSetNames = fromSchemaNames.filter((name) =>
       toSchemaNames.includes(name),
     );
+
+    // Fall back on the default result set names if there are no common ones.
+    const defaultFromResultSetName = fromSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+    const defaultToResultSetName = toSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+
+    if (
+      commonResultSetNames.length === 0 &&
+      !(defaultFromResultSetName || defaultToResultSetName)
+    ) {
+      throw new Error(
+        "No common result sets found between the two queries. Please check that the queries are compatible.",
+      );
+    }
+
     const currentResultSetName =
       selectedResultSetName || commonResultSetNames[0];
     const fromResultSet = await this.getResultSet(
       fromSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultFromResultSetName!,
       from.completedQuery.query.resultsPaths.resultsPath,
     );
     const toResultSet = await this.getResultSet(
       toSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultToResultSetName!,
       to.completedQuery.query.resultsPaths.resultsPath,
     );
     return [
       commonResultSetNames,
-      currentResultSetName,
+      currentResultSetName ||
+        `${defaultFromResultSetName} <-> ${defaultToResultSetName}`,
       fromResultSet,
       toResultSet,
     ];

extensions/ql-vscode/src/config.ts

@@ -6,9 +6,14 @@ import {
   ConfigurationChangeEvent,
   ConfigurationTarget,
 } from "vscode";
-import { DistributionManager } from "./distribution";
+import { DistributionManager } from "./codeql-cli/distribution";
 import { extLogger } from "./common";
 import { ONE_DAY_IN_MS } from "./pure/time";
+import {
+  FilterKey,
+  SortKey,
+  defaultFilterSortState,
+} from "./pure/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -69,6 +74,10 @@ const ROOT_SETTING = new Setting("codeQL");
 // Global configuration
 const TELEMETRY_SETTING = new Setting("telemetry", ROOT_SETTING);
 const AST_VIEWER_SETTING = new Setting("astViewer", ROOT_SETTING);
+const CONTEXTUAL_QUERIES_SETTINGS = new Setting(
+  "contextualQueries",
+  ROOT_SETTING,
+);
 const GLOBAL_TELEMETRY_SETTING = new Setting("telemetry");
 const LOG_INSIGHTS_SETTING = new Setting("logInsights", ROOT_SETTING);
 
@@ -83,15 +92,6 @@ export const GLOBAL_ENABLE_TELEMETRY = new Setting(
   GLOBAL_TELEMETRY_SETTING,
 );
 
-const ENABLE_NEW_TELEMETRY = new Setting(
-  "enableNewTelemetry",
-  TELEMETRY_SETTING,
-);
-
-export function newTelemetryEnabled(): boolean {
-  return ENABLE_NEW_TELEMETRY.getValue<boolean>();
-}
-
 // Distribution configuration
 const DISTRIBUTION_SETTING = new Setting("cli", ROOT_SETTING);
 export const CUSTOM_CODEQL_PATH_SETTING = new Setting(
@@ -146,6 +146,10 @@ const DEBUG_SETTING = new Setting("debug", RUNNING_QUERIES_SETTING);
 const MAX_PATHS = new Setting("maxPaths", RUNNING_QUERIES_SETTING);
 const RUNNING_TESTS_SETTING = new Setting("runningTests", ROOT_SETTING);
 const RESULTS_DISPLAY_SETTING = new Setting("resultsDisplay", ROOT_SETTING);
+const USE_EXTENSION_PACKS = new Setting(
+  "useExtensionPacks",
+  RUNNING_QUERIES_SETTING,
+);
 
 export const ADDITIONAL_TEST_ARGUMENTS_SETTING = new Setting(
   "additionalTestArguments",
@@ -205,6 +209,7 @@ const CLI_SETTINGS = [
   NUMBER_OF_TEST_THREADS_SETTING,
   NUMBER_OF_THREADS_SETTING,
   MAX_PATHS,
+  USE_EXTENSION_PACKS,
 ];
 
 export interface CliConfig {
@@ -212,7 +217,9 @@ export interface CliConfig {
   numberTestThreads: number;
   numberThreads: number;
   maxPaths: number;
+  useExtensionPacks: boolean;
   onDidChangeConfiguration?: Event<void>;
+  setUseExtensionPacks: (useExtensionPacks: boolean) => Promise<void>;
 }
 
 export abstract class ConfigListener extends DisposableObject {
@@ -409,6 +416,19 @@ export class CliConfigListener extends ConfigListener implements CliConfig {
     return MAX_PATHS.getValue<number>();
   }
 
+  public get useExtensionPacks(): boolean {
+    // currently, we are restricting the values of this setting to 'all' or 'none'.
+    return USE_EXTENSION_PACKS.getValue() === "all";
+  }
+
+  // Exposed for testing only
+  public async setUseExtensionPacks(newUseExtensionPacks: boolean) {
+    await USE_EXTENSION_PACKS.updateValue(
+      newUseExtensionPacks ? "all" : "none",
+      ConfigurationTarget.Global,
+    );
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(CLI_SETTINGS, e);
   }
@@ -468,13 +488,21 @@ export function joinOrderWarningThreshold(): number {
 }
 
 /**
- * Avoids caching in the AST viewer if the user is also a canary user.
+ * Hidden setting: Avoids caching in the AST viewer if the user is also a canary user.
  */
 export const NO_CACHE_AST_VIEWER = new Setting(
   "disableCache",
   AST_VIEWER_SETTING,
 );
 
+/**
+ * Hidden setting: Avoids caching in jump to def and find refs contextual queries if the user is also a canary user.
+ */
+export const NO_CACHE_CONTEXTUAL_QUERIES = new Setting(
+  "disableCache",
+  CONTEXTUAL_QUERIES_SETTINGS,
+);
+
 // Settings for variant analysis
 const VARIANT_ANALYSIS_SETTING = new Setting("variantAnalysis", ROOT_SETTING);
 
@@ -518,6 +546,34 @@ export class VariantAnalysisConfigListener
   }
 }
 
+const VARIANT_ANALYSIS_FILTER_RESULTS = new Setting(
+  "defaultResultsFilter",
+  VARIANT_ANALYSIS_SETTING,
+);
+
+export function getVariantAnalysisDefaultResultsFilter(): FilterKey {
+  const value = VARIANT_ANALYSIS_FILTER_RESULTS.getValue<string>();
+  if (Object.values(FilterKey).includes(value as FilterKey)) {
+    return value as FilterKey;
+  } else {
+    return defaultFilterSortState.filterKey;
+  }
+}
+
+const VARIANT_ANALYSIS_SORT_RESULTS = new Setting(
+  "defaultResultsSort",
+  VARIANT_ANALYSIS_SETTING,
+);
+
+export function getVariantAnalysisDefaultResultsSort(): SortKey {
+  const value = VARIANT_ANALYSIS_SORT_RESULTS.getValue<string>();
+  if (Object.values(SortKey).includes(value as SortKey)) {
+    return value as SortKey;
+  } else {
+    return defaultFilterSortState.sortKey;
+  }
+}
+
 /**
  * The branch of "github/codeql-variant-analysis-action" to use with the "Run Variant Analysis" command.
  * Default value is "main".
@@ -597,3 +653,68 @@ export const CODESPACES_TEMPLATE = new Setting(
 export function isCodespacesTemplate() {
   return !!CODESPACES_TEMPLATE.getValue<boolean>();
 }
+
+const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING);
+
+export const ALLOW_HTTP_SETTING = new Setting(
+  "allowHttp",
+  DATABASE_DOWNLOAD_SETTING,
+);
+
+export function allowHttp(): boolean {
+  return ALLOW_HTTP_SETTING.getValue<boolean>() || false;
+}
+
+/**
+ * Parent setting for all settings related to the "Create Query" command.
+ */
+const CREATE_QUERY_COMMAND = new Setting("createQuery", ROOT_SETTING);
+
+/**
+ * The name of the folder where we want to create QL packs.
+ **/
+const QL_PACK_LOCATION = new Setting("qlPackLocation", CREATE_QUERY_COMMAND);
+
+export function getQlPackLocation(): string | undefined {
+  return QL_PACK_LOCATION.getValue<string>() || undefined;
+}
+
+export async function setQlPackLocation(folder: string | undefined) {
+  await QL_PACK_LOCATION.updateValue(folder, ConfigurationTarget.Global);
+}
+
+/**
+ * Whether to ask the user to autogenerate a QL pack. The options are "ask" and "never".
+ **/
+const AUTOGENERATE_QL_PACKS = new Setting(
+  "autogenerateQlPacks",
+  CREATE_QUERY_COMMAND,
+);
+
+const AutogenerateQLPacksValues = ["ask", "never"] as const;
+type AutogenerateQLPacks = typeof AutogenerateQLPacksValues[number];
+
+export function getAutogenerateQlPacks(): AutogenerateQLPacks {
+  const value = AUTOGENERATE_QL_PACKS.getValue<AutogenerateQLPacks>();
+  return AutogenerateQLPacksValues.includes(value) ? value : "ask";
+}
+
+export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
+  await AUTOGENERATE_QL_PACKS.updateValue(choice, ConfigurationTarget.Global);
+}
+
+/**
+ * A flag indicating whether to show the queries panel in the QL view container.
+ */
+const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
+
+export function showQueriesPanel(): boolean {
+  return !!QUERIES_PANEL.getValue<boolean>();
+}
+
+const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
+const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
+
+export function showLlmGeneration(): boolean {
+  return !!LLM_GENERATION.getValue<boolean>();
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts

@@ -0,0 +1,54 @@
+import { Credentials } from "../common/authentication";
+import { OctokitResponse } from "@octokit/types";
+
+export enum ClassificationType {
+  Unknown = "CLASSIFICATION_TYPE_UNKNOWN",
+  Neutral = "CLASSIFICATION_TYPE_NEUTRAL",
+  Source = "CLASSIFICATION_TYPE_SOURCE",
+  Sink = "CLASSIFICATION_TYPE_SINK",
+  Summary = "CLASSIFICATION_TYPE_SUMMARY",
+}
+
+export interface Classification {
+  type: ClassificationType;
+  kind: string;
+  explanation: string;
+}
+
+export interface Method {
+  package: string;
+  type: string;
+  name: string;
+  signature: string;
+  usages: string[];
+  classification?: Classification;
+  input?: string;
+  output?: string;
+}
+
+export interface ModelRequest {
+  language: string;
+  candidates: Method[];
+  samples: Method[];
+}
+
+export interface ModelResponse {
+  language: string;
+  predicted: Method[];
+}
+
+export async function autoModel(
+  credentials: Credentials,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<ModelResponse> = await octokit.request(
+    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+    {
+      data: request,
+    },
+  );
+
+  return response.data;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

@@ -0,0 +1,136 @@
+import { CancellationTokenSource } from "vscode";
+import { join } from "path";
+import { runQuery } from "./external-api-usage-query";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { interpretResultsSarif } from "../query-results";
+import { ProgressCallback } from "../common/vscode/progress";
+
+type Options = {
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+};
+
+export type UsageSnippetsBySignature = Record<string, string[]>;
+
+export async function getAutoModelUsages({
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+}: Options): Promise<UsageSnippetsBySignature> {
+  const maxStep = 1500;
+
+  const cancellationTokenSource = new CancellationTokenSource();
+
+  // This will re-run the query that was already run when opening the data extensions editor. This
+  // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
+  // need to interpret the results.
+  const queryResult = await runQuery({
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    progress: (update) =>
+      progress({
+        maxStep,
+        step: update.step,
+        message: update.message,
+      }),
+    token: cancellationTokenSource.token,
+  });
+  if (!queryResult) {
+    throw new Error("Query failed");
+  }
+
+  progress({
+    maxStep,
+    step: 1100,
+    message: "Retrieving source location prefix",
+  });
+
+  // CodeQL needs to have access to the database to be able to retrieve the
+  // snippets from it. The source location prefix is used to determine the
+  // base path of the database.
+  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
+    cliServer,
+  );
+  const sourceArchiveUri = databaseItem.sourceArchive;
+  const sourceInfo =
+    sourceArchiveUri === undefined
+      ? undefined
+      : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix,
+        };
+
+  progress({
+    maxStep,
+    step: 1200,
+    message: "Interpreting results",
+  });
+
+  // Convert the results to SARIF so that Codeql will retrieve the snippets
+  // from the datababe. This means we don't need to do that in the extension
+  // and everything is handled by the CodeQL CLI.
+  const sarif = await interpretResultsSarif(
+    cliServer,
+    {
+      // To interpret the results we need to provide metadata about the query. We could do this using
+      // `resolveMetadata` but that would be an extra call to the CodeQL CLI server and would require
+      // us to know the path to the query on the filesystem. Since we know what the metadata should
+      // look like and the only metadata that the CodeQL CLI requires is an ID and the kind, we can
+      // simply use constants here.
+      kind: "problem",
+      id: "usage",
+    },
+    {
+      resultsPath: queryResult.outputDir.bqrsPath,
+      interpretedResultsPath: join(
+        queryStorageDir,
+        "interpreted-results.sarif",
+      ),
+    },
+    sourceInfo,
+    ["--sarif-add-snippets"],
+  );
+
+  progress({
+    maxStep,
+    step: 1400,
+    message: "Parsing results",
+  });
+
+  const snippets: UsageSnippetsBySignature = {};
+
+  const results = sarif.runs[0]?.results;
+  if (!results) {
+    throw new Error("No results");
+  }
+
+  // This will group the snippets by the method signature.
+  for (const result of results) {
+    const signature = result.message.text;
+
+    const snippet =
+      result.locations?.[0]?.physicalLocation?.contextRegion?.snippet?.text;
+
+    if (!signature || !snippet) {
+      continue;
+    }
+
+    if (!(signature in snippets)) {
+      snippets[signature] = [];
+    }
+
+    snippets[signature].push(snippet);
+  }
+
+  return snippets;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model.ts

@@ -0,0 +1,221 @@
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+import {
+  Classification,
+  ClassificationType,
+  Method,
+  ModelRequest,
+} from "./auto-model-api";
+import type { UsageSnippetsBySignature } from "./auto-model-usages-query";
+
+export function createAutoModelRequest(
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+  usages: UsageSnippetsBySignature,
+): ModelRequest {
+  const request: ModelRequest = {
+    language,
+    samples: [],
+    candidates: [],
+  };
+
+  // Sort by number of usages so we always send the most used methods first
+  externalApiUsages = [...externalApiUsages];
+  externalApiUsages.sort((a, b) => b.usages.length - a.usages.length);
+
+  for (const externalApiUsage of externalApiUsages) {
+    const modeledMethod: ModeledMethod = modeledMethods[
+      externalApiUsage.signature
+    ] ?? {
+      type: "none",
+    };
+
+    const usagesForMethod =
+      usages[externalApiUsage.signature] ??
+      externalApiUsage.usages.map((usage) => usage.label);
+
+    const numberOfArguments =
+      externalApiUsage.methodParameters === "()"
+        ? 0
+        : externalApiUsage.methodParameters.split(",").length;
+
+    for (
+      let argumentIndex = 0;
+      argumentIndex < numberOfArguments;
+      argumentIndex++
+    ) {
+      const method: Method = {
+        package: externalApiUsage.packageName,
+        type: externalApiUsage.typeName,
+        name: externalApiUsage.methodName,
+        signature: externalApiUsage.methodParameters,
+        classification:
+          modeledMethod.type === "none"
+            ? undefined
+            : toMethodClassification(modeledMethod),
+        usages: usagesForMethod.slice(0, 10),
+        input: `Argument[${argumentIndex}]`,
+      };
+
+      if (modeledMethod.type === "none") {
+        request.candidates.push(method);
+      } else {
+        request.samples.push(method);
+      }
+    }
+  }
+
+  request.candidates = request.candidates.slice(0, 20);
+  request.samples = request.samples.slice(0, 100);
+
+  return request;
+}
+
+/**
+ * For now, we have a simplified model that only models methods as sinks. It does not model methods as neutral,
+ * so we aren't actually able to correctly determine that a method is neutral; it could still be a source or summary.
+ * However, to keep this method simple and give output to the user, we will model any method for which none of its
+ * arguments are modeled as sinks as neutral.
+ *
+ * If there are multiple arguments which are modeled as sinks, we will only model the first one.
+ */
+export function parsePredictedClassifications(
+  predicted: Method[],
+): Record<string, ModeledMethod> {
+  const predictedBySignature: Record<string, Method[]> = {};
+  for (const method of predicted) {
+    const signature = toFullMethodSignature(method);
+
+    if (!(signature in predictedBySignature)) {
+      predictedBySignature[signature] = [];
+    }
+
+    predictedBySignature[signature].push(method);
+  }
+
+  const modeledMethods: Record<string, ModeledMethod> = {};
+
+  for (const signature in predictedBySignature) {
+    const predictedMethods = predictedBySignature[signature];
+
+    const sinks = predictedMethods.filter(
+      (method) => method.classification?.type === ClassificationType.Sink,
+    );
+    if (sinks.length === 0) {
+      // For now, model any method for which none of its arguments are modeled as sinks as neutral
+      modeledMethods[signature] = {
+        type: "neutral",
+        kind: "summary",
+        input: "",
+        output: "",
+        provenance: "ai-generated",
+      };
+      continue;
+    }
+
+    // Order the sinks by the input alphabetically. This will ensure that the first argument is always
+    // first in the list of sinks, the second argument is always second, etc.
+    // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first
+    sinks.sort((a, b) => compareInputOutput(a.input ?? "", b.input ?? ""));
+
+    const sink = sinks[0];
+
+    modeledMethods[signature] = {
+      type: "sink",
+      kind: sink.classification?.kind ?? "",
+      input: sink.input ?? "",
+      output: sink.output ?? "",
+      provenance: "ai-generated",
+    };
+  }
+
+  return modeledMethods;
+}
+
+function toMethodClassificationType(
+  type: ModeledMethodType,
+): ClassificationType {
+  switch (type) {
+    case "source":
+      return ClassificationType.Source;
+    case "sink":
+      return ClassificationType.Sink;
+    case "summary":
+      return ClassificationType.Summary;
+    case "neutral":
+      return ClassificationType.Neutral;
+    default:
+      return ClassificationType.Unknown;
+  }
+}
+
+function toMethodClassification(modeledMethod: ModeledMethod): Classification {
+  return {
+    type: toMethodClassificationType(modeledMethod.type),
+    kind: modeledMethod.kind,
+    explanation: "",
+  };
+}
+
+function toFullMethodSignature(method: Method): string {
+  return `${method.package}.${method.type}#${method.name}${method.signature}`;
+}
+
+const argumentRegex = /^Argument\[(\d+)]$/;
+
+// Argument[this] is before ReturnValue
+const nonNumericArgumentOrder = ["Argument[this]", "ReturnValue"];
+
+/**
+ * Compare two inputs or outputs matching `Argument[<number>]`, `Argument[this]`, or `ReturnValue`.
+ * If they are the same, return 0. If a is less than b, returns a negative number.
+ * If a is greater than b, returns a positive number.
+ */
+export function compareInputOutput(a: string, b: string): number {
+  if (a === b) {
+    return 0;
+  }
+
+  const aMatch = a.match(argumentRegex);
+  const bMatch = b.match(argumentRegex);
+
+  // Numeric arguments are always first
+  if (aMatch && !bMatch) {
+    return -1;
+  }
+  if (!aMatch && bMatch) {
+    return 1;
+  }
+
+  // Neither is an argument
+  if (!aMatch && !bMatch) {
+    const aIndex = nonNumericArgumentOrder.indexOf(a);
+    const bIndex = nonNumericArgumentOrder.indexOf(b);
+
+    // If either one is unknown, it is sorted last
+    if (aIndex === -1 && bIndex === -1) {
+      // Use en-US because these are well-known strings that are not localized
+      return a.localeCompare(b, "en-US");
+    }
+    if (aIndex === -1) {
+      return 1;
+    }
+    if (bIndex === -1) {
+      return -1;
+    }
+
+    return aIndex - bIndex;
+  }
+
+  // This case shouldn't happen, but makes TypeScript happy
+  if (!aMatch || !bMatch) {
+    return 0;
+  }
+
+  // Both are arguments
+  const aIndex = parseInt(aMatch[1]);
+  const bIndex = parseInt(bMatch[1]);
+
+  return aIndex - bIndex;
+}

extensions/ql-vscode/src/data-extensions-editor/bqrs.ts

@@ -0,0 +1,61 @@
+import { DecodedBqrsChunk } from "../pure/bqrs-cli-types";
+import { Call, ExternalApiUsage } from "./external-api-usage";
+
+export function decodeBqrsToExternalApiUsages(
+  chunk: DecodedBqrsChunk,
+): ExternalApiUsage[] {
+  const methodsByApiName = new Map<string, ExternalApiUsage>();
+
+  chunk?.tuples.forEach((tuple) => {
+    const usage = tuple[0] as Call;
+    const signature = tuple[1] as string;
+    const supported = (tuple[2] as string) === "true";
+
+    const [packageWithType, methodDeclaration] = signature.split("#");
+
+    const packageName = packageWithType.substring(
+      0,
+      packageWithType.lastIndexOf("."),
+    );
+    const typeName = packageWithType.substring(
+      packageWithType.lastIndexOf(".") + 1,
+    );
+
+    const methodName = methodDeclaration.substring(
+      0,
+      methodDeclaration.indexOf("("),
+    );
+    const methodParameters = methodDeclaration.substring(
+      methodDeclaration.indexOf("("),
+    );
+
+    if (!methodsByApiName.has(signature)) {
+      methodsByApiName.set(signature, {
+        signature,
+        packageName,
+        typeName,
+        methodName,
+        methodParameters,
+        supported,
+        usages: [],
+      });
+    }
+
+    const method = methodsByApiName.get(signature)!;
+    method.usages.push(usage);
+  });
+
+  const externalApiUsages = Array.from(methodsByApiName.values());
+  externalApiUsages.sort((a, b) => {
+    // Sort first by supported, putting unmodeled methods first.
+    if (a.supported && !b.supported) {
+      return 1;
+    }
+    if (!a.supported && b.supported) {
+      return -1;
+    }
+    // Then sort by number of usages descending
+    return b.usages.length - a.usages.length;
+  });
+  return externalApiUsages;
+}

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -0,0 +1,112 @@
+import { ExtensionContext } from "vscode";
+import { DataExtensionsEditorView } from "./data-extensions-editor-view";
+import { DataExtensionsEditorCommands } from "../common/commands";
+import { CliVersionConstraint, CodeQLCliServer } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseManager } from "../databases/local-databases";
+import { ensureDir } from "fs-extra";
+import { join } from "path";
+import { App } from "../common/app";
+import { showAndLogErrorMessage } from "../helpers";
+import { withProgress } from "../common/vscode/progress";
+import { pickExtensionPackModelFile } from "./extension-pack-picker";
+
+const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
+
+export class DataExtensionsEditorModule {
+  private readonly queryStorageDir: string;
+
+  private constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly app: App,
+    private readonly databaseManager: DatabaseManager,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly queryRunner: QueryRunner,
+    baseQueryStorageDir: string,
+  ) {
+    this.queryStorageDir = join(
+      baseQueryStorageDir,
+      "data-extensions-editor-results",
+    );
+  }
+
+  public static async initialize(
+    ctx: ExtensionContext,
+    app: App,
+    databaseManager: DatabaseManager,
+    cliServer: CodeQLCliServer,
+    queryRunner: QueryRunner,
+    queryStorageDir: string,
+  ): Promise<DataExtensionsEditorModule> {
+    const dataExtensionsEditorModule = new DataExtensionsEditorModule(
+      ctx,
+      app,
+      databaseManager,
+      cliServer,
+      queryRunner,
+      queryStorageDir,
+    );
+
+    await dataExtensionsEditorModule.initialize();
+    return dataExtensionsEditorModule;
+  }
+
+  public getCommands(): DataExtensionsEditorCommands {
+    return {
+      "codeQL.openDataExtensionsEditor": async () => {
+        const db = this.databaseManager.currentDatabaseItem;
+        if (!db) {
+          void showAndLogErrorMessage("No database selected");
+          return;
+        }
+
+        if (!SUPPORTED_LANGUAGES.includes(db.language)) {
+          void showAndLogErrorMessage(
+            `The data extensions editor is not supported for ${db.language} databases.`,
+          );
+          return;
+        }
+
+        return withProgress(
+          async (progress, token) => {
+            if (!(await this.cliServer.cliConstraints.supportsQlpacksKind())) {
+              void showAndLogErrorMessage(
+                `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND.format()} or later.`,
+              );
+              return;
+            }
+
+            const modelFile = await pickExtensionPackModelFile(
+              this.cliServer,
+              db,
+              progress,
+              token,
+            );
+            if (!modelFile) {
+              return;
+            }
+
+            const view = new DataExtensionsEditorView(
+              this.ctx,
+              this.app,
+              this.databaseManager,
+              this.cliServer,
+              this.queryRunner,
+              this.queryStorageDir,
+              db,
+              modelFile,
+            );
+            await view.openView();
+          },
+          {
+            title: "Opening Data Extensions Editor",
+          },
+        );
+      },
+    };
+  }
+
+  private async initialize(): Promise<void> {
+    await ensureDir(this.queryStorageDir);
+  }
+}

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -0,0 +1,485 @@
+import {
+  CancellationTokenSource,
+  ExtensionContext,
+  Uri,
+  ViewColumn,
+  window,
+  workspace,
+} from "vscode";
+import { RequestError } from "@octokit/request-error";
+import {
+  AbstractWebview,
+  WebviewPanelConfig,
+} from "../common/vscode/abstract-webview";
+import {
+  FromDataExtensionsEditorMessage,
+  ToDataExtensionsEditorMessage,
+} from "../pure/interface-types";
+import { ProgressUpdate } from "../common/vscode/progress";
+import { QueryRunner } from "../query-server";
+import {
+  showAndLogErrorMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../helpers";
+import { extLogger } from "../common";
+import { outputFile, pathExists, readFile } from "fs-extra";
+import { load as loadYaml } from "js-yaml";
+import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { asError, assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { generateFlowModel } from "./generate-flow-model";
+import { promptImportGithubDatabase } from "../databases/database-fetcher";
+import { App } from "../common/app";
+import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
+import { showResolvableLocation } from "../interface-utils";
+import { decodeBqrsToExternalApiUsages } from "./bqrs";
+import { redactableError } from "../pure/errors";
+import { readQueryResults, runQuery } from "./external-api-usage-query";
+import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod } from "./modeled-method";
+import { ExtensionPackModelFile } from "./shared/extension-pack";
+import { autoModel, ModelRequest, ModelResponse } from "./auto-model-api";
+import {
+  createAutoModelRequest,
+  parsePredictedClassifications,
+} from "./auto-model";
+import { showLlmGeneration } from "../config";
+import { getAutoModelUsages } from "./auto-model-usages-query";
+
+export class DataExtensionsEditorView extends AbstractWebview<
+  ToDataExtensionsEditorMessage,
+  FromDataExtensionsEditorMessage
+> {
+  public constructor(
+    ctx: ExtensionContext,
+    private readonly app: App,
+    private readonly databaseManager: DatabaseManager,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly queryRunner: QueryRunner,
+    private readonly queryStorageDir: string,
+    private readonly databaseItem: DatabaseItem,
+    private readonly modelFile: ExtensionPackModelFile,
+  ) {
+    super(ctx);
+  }
+
+  public async openView() {
+    const panel = await this.getPanel();
+    panel.reveal(undefined, true);
+
+    await this.waitForPanelLoaded();
+  }
+
+  protected async getPanelConfig(): Promise<WebviewPanelConfig> {
+    return {
+      viewId: "data-extensions-editor",
+      title: "Data Extensions Editor",
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: "data-extensions-editor",
+    };
+  }
+
+  protected onPanelDispose(): void {
+    // Nothing to do here
+  }
+
+  protected async onMessage(
+    msg: FromDataExtensionsEditorMessage,
+  ): Promise<void> {
+    switch (msg.t) {
+      case "viewLoaded":
+        await this.onWebViewLoaded();
+
+        break;
+      case "openExtensionPack":
+        await this.app.commands.execute(
+          "revealInExplorer",
+          Uri.file(this.modelFile.extensionPack.path),
+        );
+
+        break;
+      case "openModelFile":
+        await window.showTextDocument(
+          await workspace.openTextDocument(this.modelFile.filename),
+        );
+
+        break;
+      case "jumpToUsage":
+        await this.jumpToUsage(msg.location);
+
+        break;
+      case "saveModeledMethods":
+        await this.saveModeledMethods(
+          msg.externalApiUsages,
+          msg.modeledMethods,
+        );
+        await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);
+
+        break;
+      case "generateExternalApi":
+        await this.generateModeledMethods();
+
+        break;
+      case "generateExternalApiFromLlm":
+        await this.generateModeledMethodsFromLlm(
+          msg.externalApiUsages,
+          msg.modeledMethods,
+        );
+
+        break;
+      default:
+        assertNever(msg);
+    }
+  }
+
+  protected async onWebViewLoaded() {
+    super.onWebViewLoaded();
+
+    await Promise.all([
+      this.setViewState(),
+      this.loadExternalApiUsages(),
+      this.loadExistingModeledMethods(),
+    ]);
+  }
+
+  private async setViewState(): Promise<void> {
+    await this.postMessage({
+      t: "setDataExtensionEditorViewState",
+      viewState: {
+        extensionPackModelFile: this.modelFile,
+        modelFileExists: await pathExists(this.modelFile.filename),
+        showLlmButton: showLlmGeneration(),
+      },
+    });
+  }
+
+  protected async jumpToUsage(
+    location: ResolvableLocationValue,
+  ): Promise<void> {
+    try {
+      await showResolvableLocation(location, this.databaseItem);
+    } catch (e) {
+      if (e instanceof Error) {
+        if (e.message.match(/File not found/)) {
+          void window.showErrorMessage(
+            "Original file of this result is not in the database's source archive.",
+          );
+        } else {
+          void extLogger.log(`Unable to handleMsgFromView: ${e.message}`);
+        }
+      } else {
+        void extLogger.log(`Unable to handleMsgFromView: ${e}`);
+      }
+    }
+  }
+
+  protected async saveModeledMethods(
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+  ): Promise<void> {
+    const yaml = createDataExtensionYaml(
+      this.databaseItem.language,
+      externalApiUsages,
+      modeledMethods,
+    );
+
+    await outputFile(this.modelFile.filename, yaml);
+
+    void extLogger.log(
+      `Saved data extension YAML to ${this.modelFile.filename}`,
+    );
+  }
+
+  protected async loadExistingModeledMethods(): Promise<void> {
+    try {
+      if (!(await pathExists(this.modelFile.filename))) {
+        return;
+      }
+
+      const yaml = await readFile(this.modelFile.filename, "utf8");
+
+      const data = loadYaml(yaml, {
+        filename: this.modelFile.filename,
+      });
+
+      const existingModeledMethods = loadDataExtensionYaml(data);
+
+      if (!existingModeledMethods) {
+        void showAndLogErrorMessage(
+          `Failed to parse data extension YAML ${this.modelFile.filename}.`,
+        );
+        return;
+      }
+
+      await this.postMessage({
+        t: "addModeledMethods",
+        modeledMethods: existingModeledMethods,
+      });
+    } catch (e: unknown) {
+      void showAndLogErrorMessage(
+        `Unable to read data extension YAML ${
+          this.modelFile.filename
+        }: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+
+  protected async loadExternalApiUsages(): Promise<void> {
+    const cancellationTokenSource = new CancellationTokenSource();
+
+    try {
+      const queryResult = await runQuery({
+        cliServer: this.cliServer,
+        queryRunner: this.queryRunner,
+        databaseItem: this.databaseItem,
+        queryStorageDir: this.queryStorageDir,
+        progress: (progressUpdate: ProgressUpdate) => {
+          void this.showProgress(progressUpdate, 1500);
+        },
+        token: cancellationTokenSource.token,
+      });
+      if (!queryResult) {
+        await this.clearProgress();
+        return;
+      }
+
+      await this.showProgress({
+        message: "Decoding results",
+        step: 1100,
+        maxStep: 1500,
+      });
+
+      const bqrsChunk = await readQueryResults({
+        cliServer: this.cliServer,
+        bqrsPath: queryResult.outputDir.bqrsPath,
+      });
+      if (!bqrsChunk) {
+        await this.clearProgress();
+        return;
+      }
+
+      await this.showProgress({
+        message: "Finalizing results",
+        step: 1450,
+        maxStep: 1500,
+      });
+
+      const externalApiUsages = decodeBqrsToExternalApiUsages(bqrsChunk);
+
+      await this.postMessage({
+        t: "setExternalApiUsages",
+        externalApiUsages,
+      });
+
+      await this.clearProgress();
+    } catch (err) {
+      void showAndLogExceptionWithTelemetry(
+        redactableError(
+          asError(err),
+        )`Failed to load external API usages: ${getErrorMessage(err)}`,
+      );
+    }
+  }
+
+  protected async generateModeledMethods(): Promise<void> {
+    const tokenSource = new CancellationTokenSource();
+
+    const selectedDatabase = this.databaseManager.currentDatabaseItem;
+
+    // The external API methods are in the library source code, so we need to ask
+    // the user to import the library database. We need to have the database
+    // imported to the query server, so we need to register it to our workspace.
+    const database = await promptImportGithubDatabase(
+      this.app.commands,
+      this.databaseManager,
+      this.app.workspaceStoragePath ?? this.app.globalStoragePath,
+      this.app.credentials,
+      (update) => this.showProgress(update),
+      tokenSource.token,
+      this.cliServer,
+    );
+    if (!database) {
+      await this.clearProgress();
+      void extLogger.log("No database chosen");
+
+      return;
+    }
+
+    // The library database was set as the current database by importing it,
+    // but we need to set it back to the originally selected database.
+    await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
+
+    await this.showProgress({
+      step: 0,
+      maxStep: 4000,
+      message: "Generating modeled methods for library",
+    });
+
+    try {
+      await generateFlowModel({
+        cliServer: this.cliServer,
+        queryRunner: this.queryRunner,
+        queryStorageDir: this.queryStorageDir,
+        databaseItem: database,
+        onResults: async (results) => {
+          const modeledMethodsByName: Record<string, ModeledMethod> = {};
+
+          for (const result of results) {
+            modeledMethodsByName[result.signature] = result.modeledMethod;
+          }
+
+          await this.postMessage({
+            t: "addModeledMethods",
+            modeledMethods: modeledMethodsByName,
+            overrideNone: true,
+          });
+        },
+        progress: (update) => this.showProgress(update),
+        token: tokenSource.token,
+      });
+    } catch (e: unknown) {
+      void showAndLogExceptionWithTelemetry(
+        redactableError(
+          asError(e),
+        )`Failed to generate flow model: ${getErrorMessage(e)}`,
+      );
+    }
+
+    // After the flow model has been generated, we can remove the temporary database
+    // which we used for generating the flow model.
+    await this.databaseManager.removeDatabaseItem(
+      () =>
+        this.showProgress({
+          step: 3900,
+          maxStep: 4000,
+          message: "Removing temporary database",
+        }),
+      tokenSource.token,
+      database,
+    );
+
+    await this.clearProgress();
+  }
+
+  private async generateModeledMethodsFromLlm(
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+  ): Promise<void> {
+    const maxStep = 3000;
+
+    await this.showProgress({
+      step: 0,
+      maxStep,
+      message: "Retrieving usages",
+    });
+
+    const usages = await getAutoModelUsages({
+      cliServer: this.cliServer,
+      queryRunner: this.queryRunner,
+      queryStorageDir: this.queryStorageDir,
+      databaseItem: this.databaseItem,
+      progress: (update) => this.showProgress(update, maxStep),
+    });
+
+    await this.showProgress({
+      step: 1800,
+      maxStep,
+      message: "Creating request",
+    });
+
+    const request = createAutoModelRequest(
+      this.databaseItem.language,
+      externalApiUsages,
+      modeledMethods,
+      usages,
+    );
+
+    await this.showProgress({
+      step: 2000,
+      maxStep,
+      message: "Sending request",
+    });
+
+    const response = await this.callAutoModelApi(request);
+    if (!response) {
+      return;
+    }
+
+    await this.showProgress({
+      step: 2500,
+      maxStep,
+      message: "Parsing response",
+    });
+
+    const predictedModeledMethods = parsePredictedClassifications(
+      response.predicted,
+    );
+
+    await this.showProgress({
+      step: 2800,
+      maxStep,
+      message: "Applying results",
+    });
+
+    await this.postMessage({
+      t: "addModeledMethods",
+      modeledMethods: predictedModeledMethods,
+      overrideNone: true,
+    });
+
+    await this.clearProgress();
+  }
+
+  /*
+   * Progress in this class is a bit weird. Most of the progress is based on running the query.
+   * Query progress is always between 0 and 1000. However, we still have some steps that need
+   * to be done after the query has finished. Therefore, the maximum step is 1500. This captures
+   * that there's 1000 steps of the query progress since that takes the most time, and then
+   * an additional 500 steps for the rest of the work. The progress doesn't need to be 100%
+   * accurate, so this is just a rough estimate.
+   *
+   * For generating the modeled methods for an external library, the max step is 4000. This is
+   * based on the following steps:
+   * - 1000 for the summary model
+   * - 1000 for the sink model
+   * - 1000 for the source model
+   * - 1000 for the neutral model
+   */
+  private async showProgress(update: ProgressUpdate, maxStep?: number) {
+    await this.postMessage({
+      t: "showProgress",
+      step: update.step,
+      maxStep: maxStep ?? update.maxStep,
+      message: update.message,
+    });
+  }
+
+  private async clearProgress() {
+    await this.showProgress({
+      step: 0,
+      maxStep: 0,
+      message: "",
+    });
+  }
+
+  private async callAutoModelApi(
+    request: ModelRequest,
+  ): Promise<ModelResponse | null> {
+    try {
+      return await autoModel(this.app.credentials, request);
+    } catch (e) {
+      await this.clearProgress();
+
+      if (e instanceof RequestError && e.status === 429) {
+        void showAndLogExceptionWithTelemetry(
+          redactableError(e)`Rate limit hit, please try again soon.`,
+        );
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
+}

extensions/ql-vscode/src/data-extensions-editor/data-schema.json

@@ -0,0 +1,45 @@
+{
+  "type": "object",
+  "properties": {
+    "extensions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["addsTo", "data"],
+        "properties": {
+          "addsTo": {
+            "type": "object",
+            "required": ["pack", "extensible"],
+            "properties": {
+              "pack": {
+                "type": "string"
+              },
+              "extensible": {
+                "type": "string"
+              }
+            }
+          },
+          "data": {
+            "type": "array",
+            "items": {
+              "type": "array",
+              "items": {
+                "oneOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "type": "number"
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -0,0 +1,417 @@
+import { join, relative, resolve, sep } from "path";
+import { outputFile, pathExists, readFile } from "fs-extra";
+import { dump as dumpYaml, load as loadYaml } from "js-yaml";
+import { minimatch } from "minimatch";
+import { CancellationToken, window } from "vscode";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import {
+  getOnDiskWorkspaceFolders,
+  getOnDiskWorkspaceFoldersObjects,
+  showAndLogErrorMessage,
+} from "../helpers";
+import { ProgressCallback } from "../common/vscode/progress";
+import { DatabaseItem } from "../databases/local-databases";
+import { getQlPackPath, QLPACK_FILENAMES } from "../pure/ql";
+import { getErrorMessage } from "../pure/helpers-pure";
+import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
+
+const maxStep = 3;
+
+const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
+const packNameRegex = new RegExp(
+  `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
+);
+const packNameLength = 128;
+
+export async function pickExtensionPackModelFile(
+  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
+  databaseItem: Pick<DatabaseItem, "name" | "language">,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<ExtensionPackModelFile | undefined> {
+  const extensionPack = await pickExtensionPack(
+    cliServer,
+    databaseItem,
+    progress,
+    token,
+  );
+  if (!extensionPack) {
+    return undefined;
+  }
+
+  const modelFile = await pickModelFile(
+    cliServer,
+    databaseItem,
+    extensionPack,
+    progress,
+    token,
+  );
+  if (!modelFile) {
+    return;
+  }
+
+  return {
+    filename: modelFile,
+    extensionPack,
+  };
+}
+
+async function pickExtensionPack(
+  cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
+  databaseItem: Pick<DatabaseItem, "name" | "language">,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<ExtensionPack | undefined> {
+  progress({
+    message: "Resolving extension packs...",
+    step: 1,
+    maxStep,
+  });
+
+  // Get all existing extension packs in the workspace
+  const additionalPacks = getOnDiskWorkspaceFolders();
+  const extensionPacksInfo = await cliServer.resolveQlpacks(
+    additionalPacks,
+    true,
+  );
+
+  if (Object.keys(extensionPacksInfo).length === 0) {
+    return pickNewExtensionPack(databaseItem, token);
+  }
+
+  const extensionPacks = (
+    await Promise.all(
+      Object.entries(extensionPacksInfo).map(async ([name, paths]) => {
+        if (paths.length !== 1) {
+          void showAndLogErrorMessage(
+            `Extension pack ${name} resolves to multiple paths`,
+            {
+              fullMessage: `Extension pack ${name} resolves to multiple paths: ${paths.join(
+                ", ",
+              )}`,
+            },
+          );
+
+          return undefined;
+        }
+
+        const path = paths[0];
+
+        let extensionPack: ExtensionPack;
+        try {
+          extensionPack = await readExtensionPack(path);
+        } catch (e: unknown) {
+          void showAndLogErrorMessage(`Could not read extension pack ${name}`, {
+            fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
+              e,
+            )}`,
+          });
+
+          return undefined;
+        }
+
+        return extensionPack;
+      }),
+    )
+  ).filter((info): info is ExtensionPack => info !== undefined);
+
+  const extensionPacksForLanguage = extensionPacks.filter(
+    (pack) =>
+      pack.extensionTargets[`codeql/${databaseItem.language}-all`] !==
+      undefined,
+  );
+
+  const options: Array<{
+    label: string;
+    description: string | undefined;
+    detail: string | undefined;
+    extensionPack: ExtensionPack | null;
+  }> = extensionPacksForLanguage.map((pack) => ({
+    label: pack.name,
+    description: pack.version,
+    detail: pack.path,
+    extensionPack: pack,
+  }));
+  options.push({
+    label: "Create new extension pack",
+    description: undefined,
+    detail: undefined,
+    extensionPack: null,
+  });
+
+  progress({
+    message: "Choosing extension pack...",
+    step: 2,
+    maxStep,
+  });
+
+  const extensionPackOption = await window.showQuickPick(
+    options,
+    {
+      title: "Select extension pack to use",
+    },
+    token,
+  );
+  if (!extensionPackOption) {
+    return undefined;
+  }
+
+  if (!extensionPackOption.extensionPack) {
+    return pickNewExtensionPack(databaseItem, token);
+  }
+
+  return extensionPackOption.extensionPack;
+}
+
+async function pickModelFile(
+  cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
+  databaseItem: Pick<DatabaseItem, "name">,
+  extensionPack: ExtensionPack,
+  progress: ProgressCallback,
+  token: CancellationToken,
+): Promise<string | undefined> {
+  // Find the existing model files in the extension pack
+  const additionalPacks = getOnDiskWorkspaceFolders();
+  const extensions = await cliServer.resolveExtensions(
+    extensionPack.path,
+    additionalPacks,
+  );
+
+  const modelFiles = new Set<string>();
+
+  if (extensionPack.path in extensions.data) {
+    for (const extension of extensions.data[extensionPack.path]) {
+      modelFiles.add(extension.file);
+    }
+  }
+
+  if (modelFiles.size === 0) {
+    return pickNewModelFile(databaseItem, extensionPack, token);
+  }
+
+  const fileOptions: Array<{ label: string; file: string | null }> = [];
+  for (const file of modelFiles) {
+    fileOptions.push({
+      label: relative(extensionPack.path, file).replaceAll(sep, "/"),
+      file,
+    });
+  }
+  fileOptions.push({
+    label: "Create new model file",
+    file: null,
+  });
+
+  progress({
+    message: "Choosing model file...",
+    step: 3,
+    maxStep,
+  });
+
+  const fileOption = await window.showQuickPick(
+    fileOptions,
+    {
+      title: "Select model file to use",
+    },
+    token,
+  );
+
+  if (!fileOption) {
+    return undefined;
+  }
+
+  if (fileOption.file) {
+    return fileOption.file;
+  }
+
+  return pickNewModelFile(databaseItem, extensionPack, token);
+}
+
+async function pickNewExtensionPack(
+  databaseItem: Pick<DatabaseItem, "name" | "language">,
+  token: CancellationToken,
+): Promise<ExtensionPack | undefined> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
+    label: folder.name,
+    detail: folder.uri.fsPath,
+    path: folder.uri.fsPath,
+  }));
+
+  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
+  // we only want to include on-disk workspace folders.
+  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
+    title: "Select workspace folder to create extension pack in",
+  });
+  if (!workspaceFolder) {
+    return undefined;
+  }
+
+  let examplePackName = `${databaseItem.name}-extensions`;
+  if (!examplePackName.includes("/")) {
+    examplePackName = `pack/${examplePackName}`;
+  }
+
+  const packName = await window.showInputBox(
+    {
+      title: "Create new extension pack",
+      prompt: "Enter name of extension pack",
+      placeHolder: `e.g. ${examplePackName}`,
+      validateInput: async (value: string): Promise<string | undefined> => {
+        if (!value) {
+          return "Pack name must not be empty";
+        }
+
+        if (value.length > packNameLength) {
+          return `Pack name must be no longer than ${packNameLength} characters`;
+        }
+
+        const matches = packNameRegex.exec(value);
+        if (!matches?.groups) {
+          if (!value.includes("/")) {
+            return "Invalid package name: a pack name must contain a slash to separate the scope from the pack name";
+          }
+
+          return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
+        }
+
+        const packPath = join(workspaceFolder.path, matches.groups.name);
+        if (await pathExists(packPath)) {
+          return `A pack already exists at ${packPath}`;
+        }
+
+        return undefined;
+      },
+    },
+    token,
+  );
+  if (!packName) {
+    return undefined;
+  }
+
+  const matches = packNameRegex.exec(packName);
+  if (!matches?.groups) {
+    return;
+  }
+
+  const name = matches.groups.name;
+  const packPath = join(workspaceFolder.path, name);
+
+  if (await pathExists(packPath)) {
+    return undefined;
+  }
+
+  const packYamlPath = join(packPath, "codeql-pack.yml");
+
+  const extensionPack: ExtensionPack = {
+    path: packPath,
+    yamlPath: packYamlPath,
+    name: packName,
+    version: "0.0.0",
+    extensionTargets: {
+      [`codeql/${databaseItem.language}-all`]: "*",
+    },
+    dataExtensions: ["models/**/*.yml"],
+  };
+
+  await outputFile(
+    packYamlPath,
+    dumpYaml({
+      name: extensionPack.name,
+      version: extensionPack.version,
+      library: true,
+      extensionTargets: extensionPack.extensionTargets,
+      dataExtensions: extensionPack.dataExtensions,
+    }),
+  );
+
+  return extensionPack;
+}
+
+async function pickNewModelFile(
+  databaseItem: Pick<DatabaseItem, "name">,
+  extensionPack: ExtensionPack,
+  token: CancellationToken,
+) {
+  const filename = await window.showInputBox(
+    {
+      title: "Enter the name of the new model file",
+      value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
+      validateInput: async (value: string): Promise<string | undefined> => {
+        if (value === "") {
+          return "File name must not be empty";
+        }
+
+        const path = resolve(extensionPack.path, value);
+
+        if (await pathExists(path)) {
+          return "File already exists";
+        }
+
+        const notInExtensionPack = relative(
+          extensionPack.path,
+          path,
+        ).startsWith("..");
+        if (notInExtensionPack) {
+          return "File must be in the extension pack";
+        }
+
+        const matchesPattern = extensionPack.dataExtensions.some((pattern) =>
+          minimatch(value, pattern, { matchBase: true }),
+        );
+        if (!matchesPattern) {
+          return `File must match one of the patterns in 'dataExtensions' in ${extensionPack.yamlPath}`;
+        }
+
+        return undefined;
+      },
+    },
+    token,
+  );
+  if (!filename) {
+    return undefined;
+  }
+
+  return resolve(extensionPack.path, filename);
+}
+
+async function readExtensionPack(path: string): Promise<ExtensionPack> {
+  const qlpackPath = await getQlPackPath(path);
+  if (!qlpackPath) {
+    throw new Error(
+      `Could not find any of ${QLPACK_FILENAMES.join(", ")} in ${path}`,
+    );
+  }
+
+  const qlpack = await loadYaml(await readFile(qlpackPath, "utf8"), {
+    filename: qlpackPath,
+  });
+  if (typeof qlpack !== "object" || qlpack === null) {
+    throw new Error(`Could not parse ${qlpackPath}`);
+  }
+
+  const dataExtensionValue = qlpack.dataExtensions;
+  if (
+    !(
+      Array.isArray(dataExtensionValue) ||
+      typeof dataExtensionValue === "string"
+    )
+  ) {
+    throw new Error(
+      `Expected 'dataExtensions' to be a string or an array in ${qlpackPath}`,
+    );
+  }
+
+  // The YAML allows either a string or an array of strings
+  const dataExtensions = Array.isArray(dataExtensionValue)
+    ? dataExtensionValue
+    : [dataExtensionValue];
+
+  return {
+    path,
+    yamlPath: qlpackPath,
+    name: qlpack.name,
+    version: qlpack.version,
+    extensionTargets: qlpack.extensionTargets,
+    dataExtensions,
+  };
+}

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts

@@ -0,0 +1,139 @@
+import { CoreCompletedQuery, QueryRunner } from "../query-server";
+import { dir } from "tmp-promise";
+import { writeFile } from "fs-extra";
+import { dump as dumpYaml } from "js-yaml";
+import {
+  getOnDiskWorkspaceFolders,
+  isQueryLanguage,
+  showAndLogExceptionWithTelemetry,
+} from "../helpers";
+import { TeeLogger } from "../common";
+import { CancellationToken } from "vscode";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { DatabaseItem } from "../databases/local-databases";
+import { ProgressCallback } from "../common/vscode/progress";
+import { fetchExternalApiQueries } from "./queries";
+import { QueryResultType } from "../pure/new-messages";
+import { join } from "path";
+import { redactableError } from "../pure/errors";
+
+export type RunQueryOptions = {
+  cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
+  queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
+  databaseItem: Pick<DatabaseItem, "contents" | "databaseUri" | "language">;
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+  token: CancellationToken;
+};
+
+export async function runQuery({
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+  token,
+}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
+  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
+  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
+  // This is intentionally not pretty code, as it will be removed soon.
+  // For a reference of what this should do in the future, see the previous implementation in
+  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
+
+  if (!isQueryLanguage(databaseItem.language)) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`Unsupported database language ${databaseItem.language}`,
+    );
+    return;
+  }
+
+  const query = fetchExternalApiQueries[databaseItem.language];
+  if (!query) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`No external API usage query found for language ${databaseItem.language}`,
+    );
+    return;
+  }
+
+  const queryDir = (await dir({ unsafeCleanup: true })).path;
+  const queryFile = join(queryDir, "FetchExternalApis.ql");
+  await writeFile(queryFile, query.mainQuery, "utf8");
+
+  if (query.dependencies) {
+    for (const [filename, contents] of Object.entries(query.dependencies)) {
+      const dependencyFile = join(queryDir, filename);
+      await writeFile(dependencyFile, contents, "utf8");
+    }
+  }
+
+  const syntheticQueryPack = {
+    name: "codeql/external-api-usage",
+    version: "0.0.0",
+    dependencies: {
+      [`codeql/${databaseItem.language}-all`]: "*",
+    },
+  };
+
+  const qlpackFile = join(queryDir, "codeql-pack.yml");
+  await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8");
+
+  const additionalPacks = getOnDiskWorkspaceFolders();
+  const extensionPacks = Object.keys(
+    await cliServer.resolveQlpacks(additionalPacks, true),
+  );
+
+  const queryRun = queryRunner.createQueryRun(
+    databaseItem.databaseUri.fsPath,
+    {
+      queryPath: queryFile,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
+    false,
+    getOnDiskWorkspaceFolders(),
+    extensionPacks,
+    queryStorageDir,
+    undefined,
+    undefined,
+  );
+
+  const completedQuery = await queryRun.evaluate(
+    progress,
+    token,
+    new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
+  );
+
+  if (completedQuery.resultType !== QueryResultType.SUCCESS) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`External API usage query failed: ${
+        completedQuery.message ?? "No message"
+      }`,
+    );
+    return;
+  }
+
+  return completedQuery;
+}
+
+export type GetResultsOptions = {
+  cliServer: Pick<CodeQLCliServer, "bqrsInfo" | "bqrsDecode">;
+  bqrsPath: string;
+};
+
+export async function readQueryResults({
+  cliServer,
+  bqrsPath,
+}: GetResultsOptions) {
+  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
+  if (bqrsInfo["result-sets"].length !== 1) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
+    );
+    return undefined;
+  }
+
+  const resultSet = bqrsInfo["result-sets"][0];
+
+  return cliServer.bqrsDecode(bqrsPath, resultSet.name);
+}

extensions/ql-vscode/src/data-extensions-editor/external-api-usage.ts

@@ -0,0 +1,19 @@
+import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
+
+export type Call = {
+  label: string;
+  url: ResolvableLocationValue;
+};
+
+export type ExternalApiUsage = {
+  /**
+   * Contains the full method signature, e.g. `org.sql2o.Connection#createQuery(String)`
+   */
+  signature: string;
+  packageName: string;
+  typeName: string;
+  methodName: string;
+  methodParameters: string;
+  supported: boolean;
+  usages: Call[];
+};

extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts

@@ -0,0 +1,206 @@
+import { CancellationToken } from "vscode";
+import { DatabaseItem } from "../databases/local-databases";
+import { basename } from "path";
+import { QueryRunner } from "../query-server";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { TeeLogger } from "../common";
+import { extensiblePredicateDefinitions } from "./predicates";
+import { ProgressCallback } from "../common/vscode/progress";
+import {
+  getOnDiskWorkspaceFolders,
+  showAndLogExceptionWithTelemetry,
+} from "../helpers";
+import {
+  ModeledMethodType,
+  ModeledMethodWithSignature,
+} from "./modeled-method";
+import { redactableError } from "../pure/errors";
+import { QueryResultType } from "../pure/new-messages";
+import { file } from "tmp-promise";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { qlpackOfDatabase } from "../language-support";
+
+type FlowModelOptions = {
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  queryStorageDir: string;
+  databaseItem: DatabaseItem;
+  progress: ProgressCallback;
+  token: CancellationToken;
+  onResults: (results: ModeledMethodWithSignature[]) => void | Promise<void>;
+};
+
+async function resolveQueries(
+  cliServer: CodeQLCliServer,
+  databaseItem: DatabaseItem,
+): Promise<string[]> {
+  const qlpacks = await qlpackOfDatabase(cliServer, databaseItem);
+
+  const packsToSearch: string[] = [];
+
+  // The CLI can handle both library packs and query packs, so search both packs in order.
+  packsToSearch.push(qlpacks.dbschemePack);
+  if (qlpacks.queryPack !== undefined) {
+    packsToSearch.push(qlpacks.queryPack);
+  }
+
+  const suiteFile = (
+    await file({
+      postfix: ".qls",
+    })
+  ).path;
+  const suiteYaml = [];
+  for (const qlpack of packsToSearch) {
+    suiteYaml.push({
+      from: qlpack,
+      queries: ".",
+      include: {
+        "tags contain": "modelgenerator",
+      },
+    });
+  }
+  await writeFile(suiteFile, dump(suiteYaml), "utf8");
+
+  return await cliServer.resolveQueriesInSuite(
+    suiteFile,
+    getOnDiskWorkspaceFolders(),
+  );
+}
+
+async function getModeledMethodsFromFlow(
+  type: Exclude<ModeledMethodType, "none">,
+  queryPath: string | undefined,
+  queryStep: number,
+  {
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    progress,
+    token,
+  }: Omit<FlowModelOptions, "onResults">,
+): Promise<ModeledMethodWithSignature[]> {
+  if (queryPath === undefined) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`Failed to find ${type} query`,
+    );
+    return [];
+  }
+
+  const definition = extensiblePredicateDefinitions[type];
+
+  const queryRun = queryRunner.createQueryRun(
+    databaseItem.databaseUri.fsPath,
+    {
+      queryPath,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
+    false,
+    getOnDiskWorkspaceFolders(),
+    undefined,
+    queryStorageDir,
+    undefined,
+    undefined,
+  );
+
+  const queryResult = await queryRun.evaluate(
+    ({ step, message }) =>
+      progress({
+        message: `Generating ${type} model: ${message}`,
+        step: queryStep * 1000 + step,
+        maxStep: 4000,
+      }),
+    token,
+    new TeeLogger(queryRunner.logger, queryRun.outputDir.logPath),
+  );
+  if (queryResult.resultType !== QueryResultType.SUCCESS) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`Failed to run ${basename(queryPath)} query: ${
+        queryResult.message ?? "No message"
+      }`,
+    );
+    return [];
+  }
+
+  const bqrsPath = queryResult.outputDir.bqrsPath;
+
+  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
+  if (bqrsInfo["result-sets"].length !== 1) {
+    void showAndLogExceptionWithTelemetry(
+      redactableError`Expected exactly one result set, got ${
+        bqrsInfo["result-sets"].length
+      } for ${basename(queryPath)}`,
+    );
+  }
+
+  const resultSet = bqrsInfo["result-sets"][0];
+
+  const decodedResults = await cliServer.bqrsDecode(bqrsPath, resultSet.name);
+
+  const results = decodedResults.tuples;
+
+  return (
+    results
+      // This is just a sanity check. The query should only return strings.
+      .filter((result) => typeof result[0] === "string")
+      .map((result) => {
+        const row = result[0] as string;
+
+        return definition.readModeledMethod(row.split(";"));
+      })
+  );
+}
+
+export async function generateFlowModel({
+  onResults,
+  ...options
+}: FlowModelOptions) {
+  const queries = await resolveQueries(options.cliServer, options.databaseItem);
+
+  const queriesByBasename: Record<string, string> = {};
+  for (const query of queries) {
+    queriesByBasename[basename(query)] = query;
+  }
+
+  const summaryResults = await getModeledMethodsFromFlow(
+    "summary",
+    queriesByBasename["CaptureSummaryModels.ql"],
+    0,
+    options,
+  );
+  if (summaryResults) {
+    await onResults(summaryResults);
+  }
+
+  const sinkResults = await getModeledMethodsFromFlow(
+    "sink",
+    queriesByBasename["CaptureSinkModels.ql"],
+    1,
+    options,
+  );
+  if (sinkResults) {
+    await onResults(sinkResults);
+  }
+
+  const sourceResults = await getModeledMethodsFromFlow(
+    "source",
+    queriesByBasename["CaptureSourceModels.ql"],
+    2,
+    options,
+  );
+  if (sourceResults) {
+    await onResults(sourceResults);
+  }
+
+  const neutralResults = await getModeledMethodsFromFlow(
+    "neutral",
+    queriesByBasename["CaptureNeutralModels.ql"],
+    3,
+    options,
+  );
+  if (neutralResults) {
+    await onResults(neutralResults);
+  }
+}

extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts

@@ -0,0 +1,31 @@
+export type ModeledMethodType =
+  | "none"
+  | "source"
+  | "sink"
+  | "summary"
+  | "neutral";
+
+export type Provenance =
+  // Generated by the dataflow model
+  | "df-generated"
+  // Generated by the dataflow model and manually edited
+  | "df-manual"
+  // Generated by the auto-model
+  | "ai-generated"
+  // Generated by the auto-model and manually edited
+  | "ai-manual"
+  // Entered by the user in the editor manually
+  | "manual";
+
+export type ModeledMethod = {
+  type: ModeledMethodType;
+  input: string;
+  output: string;
+  kind: string;
+  provenance: Provenance;
+};
+
+export type ModeledMethodWithSignature = {
+  signature: string;
+  modeledMethod: ModeledMethod;
+};

extensions/ql-vscode/src/data-extensions-editor/predicates.ts

@@ -0,0 +1,145 @@
+import { ExternalApiUsage } from "./external-api-usage";
+import {
+  ModeledMethod,
+  ModeledMethodType,
+  ModeledMethodWithSignature,
+  Provenance,
+} from "./modeled-method";
+
+export type ExternalApiUsageByType = {
+  externalApiUsage: ExternalApiUsage;
+  modeledMethod: ModeledMethod;
+};
+
+export type ExtensiblePredicateDefinition = {
+  extensiblePredicate: string;
+  generateMethodDefinition: (method: ExternalApiUsageByType) => Tuple[];
+  readModeledMethod: (row: Tuple[]) => ModeledMethodWithSignature;
+
+  supportedKinds?: string[];
+};
+
+type Tuple = boolean | number | string;
+
+function readRowToMethod(row: Tuple[]): string {
+  return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
+}
+
+export const extensiblePredicateDefinitions: Record<
+  Exclude<ModeledMethodType, "none">,
+  ExtensiblePredicateDefinition
+> = {
+  source: {
+    extensiblePredicate: "sourceModel",
+    // extensible predicate sourceModel(
+    //   string package, string type, boolean subtypes, string name, string signature, string ext,
+    //   string output, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      true,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "",
+      method.modeledMethod.output,
+      method.modeledMethod.kind,
+      method.modeledMethod.provenance,
+    ],
+    readModeledMethod: (row) => ({
+      signature: readRowToMethod(row),
+      modeledMethod: {
+        type: "source",
+        input: "",
+        output: row[6] as string,
+        kind: row[7] as string,
+        provenance: row[8] as Provenance,
+      },
+    }),
+    supportedKinds: ["remote"],
+  },
+  sink: {
+    extensiblePredicate: "sinkModel",
+    // extensible predicate sinkModel(
+    //   string package, string type, boolean subtypes, string name, string signature, string ext,
+    //   string input, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      true,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "",
+      method.modeledMethod.input,
+      method.modeledMethod.kind,
+      method.modeledMethod.provenance,
+    ],
+    readModeledMethod: (row) => ({
+      signature: readRowToMethod(row),
+      modeledMethod: {
+        type: "sink",
+        input: row[6] as string,
+        output: "",
+        kind: row[7] as string,
+        provenance: row[8] as Provenance,
+      },
+    }),
+    supportedKinds: ["sql", "xss", "logging"],
+  },
+  summary: {
+    extensiblePredicate: "summaryModel",
+    // extensible predicate summaryModel(
+    //   string package, string type, boolean subtypes, string name, string signature, string ext,
+    //   string input, string output, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      true,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      "",
+      method.modeledMethod.input,
+      method.modeledMethod.output,
+      method.modeledMethod.kind,
+      method.modeledMethod.provenance,
+    ],
+    readModeledMethod: (row) => ({
+      signature: readRowToMethod(row),
+      modeledMethod: {
+        type: "summary",
+        input: row[6] as string,
+        output: row[7] as string,
+        kind: row[8] as string,
+        provenance: row[9] as Provenance,
+      },
+    }),
+    supportedKinds: ["taint", "value"],
+  },
+  neutral: {
+    extensiblePredicate: "neutralModel",
+    // extensible predicate neutralModel(
+    //   string package, string type, string name, string signature, string kind, string provenance
+    // );
+    generateMethodDefinition: (method) => [
+      method.externalApiUsage.packageName,
+      method.externalApiUsage.typeName,
+      method.externalApiUsage.methodName,
+      method.externalApiUsage.methodParameters,
+      method.modeledMethod.kind,
+      method.modeledMethod.provenance,
+    ],
+    readModeledMethod: (row) => ({
+      signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
+      modeledMethod: {
+        type: "neutral",
+        input: "",
+        output: "",
+        kind: row[4] as string,
+        provenance: row[5] as Provenance,
+      },
+    }),
+    supportedKinds: ["summary", "source", "sink"],
+  },
+};

extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts

@@ -0,0 +1,187 @@
+import { Query } from "./query";
+
+export const fetchExternalApisQuery: Query = {
+  mainQuery: `/**
+ * @name Usage of APIs coming from external libraries
+ * @description A list of 3rd party APIs used in the codebase.
+ * @tags telemetry
+ * @kind problem
+ * @id cs/telemetry/fetch-external-apis
+ */
+
+import csharp
+import ExternalApi
+
+private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
+
+private boolean isSupported(ExternalApi api) {
+  api.isSupported() and result = true
+  or
+  not api.isSupported() and
+  result = false
+}
+
+from ExternalApi api, string apiName, boolean supported, Call usage
+where
+  apiName = api.getApiName() and
+  supported = isSupported(api) and
+  usage = aUsage(api)
+select usage, apiName, supported.toString(), "supported"
+`,
+  dependencies: {
+    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+
+private import csharp
+private import dotnet
+private import semmle.code.csharp.dispatch.Dispatch
+private import semmle.code.csharp.dataflow.ExternalFlow
+private import semmle.code.csharp.dataflow.FlowSummary
+private import semmle.code.csharp.dataflow.internal.DataFlowImplCommon as DataFlowImplCommon
+private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
+private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
+private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
+private import semmle.code.csharp.security.dataflow.flowsources.Remote
+
+pragma[nomagic]
+private predicate isTestNamespace(Namespace ns) {
+  ns.getFullName()
+      .matches([
+          "NUnit.Framework%", "Xunit%", "Microsoft.VisualStudio.TestTools.UnitTesting%", "Moq%"
+        ])
+}
+
+/**
+ * A test library.
+ */
+class TestLibrary extends RefType {
+  TestLibrary() { isTestNamespace(this.getNamespace()) }
+}
+
+/** Holds if the given callable is not worth supporting. */
+private predicate isUninteresting(DotNet::Callable c) {
+  c.getDeclaringType() instanceof TestLibrary or
+  c.(Constructor).isParameterless()
+}
+
+/**
+ * An external API from either the C# Standard Library or a 3rd party library.
+ */
+class ExternalApi extends DotNet::Callable {
+  ExternalApi() {
+    this.isUnboundDeclaration() and
+    this.fromLibrary() and
+    this.(Modifiable).isEffectivelyPublic() and
+    not isUninteresting(this)
+  }
+
+  /**
+   * Gets the namespace of this API.
+   */
+  bindingset[this]
+  string getNamespace() { this.getDeclaringType().hasQualifiedName(result, _) }
+
+  /**
+   * Gets the namespace and signature of this API.
+   */
+  bindingset[this]
+  string getApiName() { result = this.getNamespace() + "." + this.getDeclaringType().getUnboundDeclaration() + "#" + this.getName() + "(" +
+  parameterQualifiedTypeNamesToString(this) + ")" }
+
+  /** Gets a node that is an input to a call to this API. */
+  private ArgumentNode getAnInput() {
+    result
+        .getCall()
+        .(DataFlowDispatch::NonDelegateDataFlowCall)
+        .getATarget(_)
+        .getUnboundDeclaration() = this
+  }
+
+  /** Gets a node that is an output from a call to this API. */
+  private DataFlow::Node getAnOutput() {
+    exists(
+      Call c, DataFlowDispatch::NonDelegateDataFlowCall dc, DataFlowImplCommon::ReturnKindExt ret
+    |
+      dc.getDispatchCall().getCall() = c and
+      c.getTarget().getUnboundDeclaration() = this
+    |
+      result = ret.getAnOutNode(dc)
+    )
+  }
+
+  /** Holds if this API has a supported summary. */
+  pragma[nomagic]
+  predicate hasSummary() {
+    this instanceof SummarizedCallable
+    or
+    defaultAdditionalTaintStep(this.getAnInput(), _)
+  }
+
+  /** Holds if this API is a known source. */
+  pragma[nomagic]
+  predicate isSource() {
+    this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
+  }
+
+  /** Holds if this API is a known sink. */
+  pragma[nomagic]
+  predicate isSink() { sinkNode(this.getAnInput(), _) }
+
+  /** Holds if this API is a known neutral. */
+  pragma[nomagic]
+  predicate isNeutral() { this instanceof FlowSummaryImpl::Public::NeutralCallable }
+
+  /**
+   * Holds if this API is supported by existing CodeQL libraries, that is, it is either a
+   * recognized source, sink or neutral or it has a flow summary.
+   */
+  predicate isSupported() {
+    this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
+  }
+}
+
+/**
+ * Gets the limit for the number of results produced by a telemetry query.
+ */
+int resultLimit() { result = 1000 }
+
+/**
+ * Holds if it is relevant to count usages of "api".
+ */
+signature predicate relevantApi(ExternalApi api);
+
+/**
+ * Given a predicate to count relevant API usages, this module provides a predicate
+ * for restricting the number or returned results based on a certain limit.
+ */
+module Results<relevantApi/1 getRelevantUsages> {
+  private int getUsages(string apiName) {
+    result =
+      strictcount(Call c, ExternalApi api |
+        c.getTarget().getUnboundDeclaration() = api and
+        apiName = api.getApiName() and
+        getRelevantUsages(api)
+      )
+  }
+
+  private int getOrder(string apiName) {
+    apiName =
+      rank[result](string name, int usages |
+        usages = getUsages(name)
+      |
+        name order by usages desc, name
+      )
+  }
+
+  /**
+   * Holds if there exists an API with "apiName" that is being used "usages" times
+   * and if it is in the top results (guarded by resultLimit).
+   */
+  predicate restrict(string apiName, int usages) {
+    usages = getUsages(apiName) and
+    getOrder(apiName) <= resultLimit()
+  }
+}
+`,
+  },
+};

extensions/ql-vscode/src/data-extensions-editor/queries/index.ts

@@ -0,0 +1,9 @@
+import { fetchExternalApisQuery as csharpFetchExternalApisQuery } from "./csharp";
+import { fetchExternalApisQuery as javaFetchExternalApisQuery } from "./java";
+import { Query } from "./query";
+import { QueryLanguage } from "../../common/query-language";
+
+export const fetchExternalApiQueries: Partial<Record<QueryLanguage, Query>> = {
+  [QueryLanguage.CSharp]: csharpFetchExternalApisQuery,
+  [QueryLanguage.Java]: javaFetchExternalApisQuery,
+};

extensions/ql-vscode/src/data-extensions-editor/queries/java.ts

@@ -0,0 +1,189 @@
+import { Query } from "./query";
+
+export const fetchExternalApisQuery: Query = {
+  mainQuery: `/**
+ * @name Usage of APIs coming from external libraries
+ * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
+ * @tags telemetry
+ * @kind problem
+ * @id java/telemetry/fetch-external-apis
+ */
+
+import java
+import ExternalApi
+
+private Call aUsage(ExternalApi api) {
+  result.getCallee().getSourceDeclaration() = api and
+  not result.getFile() instanceof GeneratedFile
+}
+
+private boolean isSupported(ExternalApi api) {
+  api.isSupported() and result = true
+  or
+  not api.isSupported() and result = false
+}
+
+from ExternalApi api, string apiName, boolean supported, Call usage
+where
+  apiName = api.getApiName() and
+  supported = isSupported(api) and
+  usage = aUsage(api)
+select usage, apiName, supported.toString(), "supported"
+`,
+  dependencies: {
+    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+
+private import java
+private import semmle.code.java.dataflow.DataFlow
+private import semmle.code.java.dataflow.ExternalFlow
+private import semmle.code.java.dataflow.FlowSources
+private import semmle.code.java.dataflow.FlowSummary
+private import semmle.code.java.dataflow.internal.DataFlowPrivate
+private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+private import semmle.code.java.dataflow.TaintTracking
+
+pragma[nomagic]
+private predicate isTestPackage(Package p) {
+  p.getName()
+      .matches([
+          "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
+          "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
+          "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
+          "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
+          "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
+          "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
+          "org.testng%"
+        ])
+}
+
+/**
+ * A test library.
+ */
+private class TestLibrary extends RefType {
+  TestLibrary() { isTestPackage(this.getPackage()) }
+}
+
+private string containerAsJar(Container container) {
+  if container instanceof JarFile then result = container.getBaseName() else result = "rt.jar"
+}
+
+/** Holds if the given callable is not worth supporting. */
+private predicate isUninteresting(Callable c) {
+  c.getDeclaringType() instanceof TestLibrary or
+  c.(Constructor).isParameterless()
+}
+
+/**
+ * An external API from either the Standard Library or a 3rd party library.
+ */
+class ExternalApi extends Callable {
+  ExternalApi() { not this.fromSource() and not isUninteresting(this) }
+
+  /**
+   * Gets information about the external API in the form expected by the MaD modeling framework.
+   */
+  string getApiName() {
+    result =
+      this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
+        "#" + this.getName() + paramsString(this)
+  }
+
+  /**
+   * Gets the jar file containing this API. Normalizes the Java Runtime to "rt.jar" despite the presence of modules.
+   */
+  string jarContainer() { result = containerAsJar(this.getCompilationUnit().getParentContainer*()) }
+
+  /** Gets a node that is an input to a call to this API. */
+  private DataFlow::Node getAnInput() {
+    exists(Call call | call.getCallee().getSourceDeclaration() = this |
+      result.asExpr().(Argument).getCall() = call or
+      result.(ArgumentNode).getCall().asCall() = call
+    )
+  }
+
+  /** Gets a node that is an output from a call to this API. */
+  private DataFlow::Node getAnOutput() {
+    exists(Call call | call.getCallee().getSourceDeclaration() = this |
+      result.asExpr() = call or
+      result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
+    )
+  }
+
+  /** Holds if this API has a supported summary. */
+  pragma[nomagic]
+  predicate hasSummary() {
+    this = any(SummarizedCallable sc).asCallable() or
+    TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
+  }
+
+  pragma[nomagic]
+  predicate isSource() {
+    this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
+  }
+
+  /** Holds if this API is a known sink. */
+  pragma[nomagic]
+  predicate isSink() { sinkNode(this.getAnInput(), _) }
+
+  /** Holds if this API is a known neutral. */
+  pragma[nomagic]
+  predicate isNeutral() { this = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() }
+
+  /**
+   * Holds if this API is supported by existing CodeQL libraries, that is, it is either a
+   * recognized source, sink or neutral or it has a flow summary.
+   */
+  predicate isSupported() {
+    this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
+  }
+}
+
+/** DEPRECATED: Alias for ExternalApi */
+deprecated class ExternalAPI = ExternalApi;
+
+/**
+ * Gets the limit for the number of results produced by a telemetry query.
+ */
+int resultLimit() { result = 1000 }
+
+/**
+ * Holds if it is relevant to count usages of \`api\`.
+ */
+signature predicate relevantApi(ExternalApi api);
+
+/**
+ * Given a predicate to count relevant API usages, this module provides a predicate
+ * for restricting the number or returned results based on a certain limit.
+ */
+module Results<relevantApi/1 getRelevantUsages> {
+  private int getUsages(string apiName) {
+    result =
+      strictcount(Call c, ExternalApi api |
+        c.getCallee().getSourceDeclaration() = api and
+        not c.getFile() instanceof GeneratedFile and
+        apiName = api.getApiName() and
+        getRelevantUsages(api)
+      )
+  }
+
+  private int getOrder(string apiInfo) {
+    apiInfo =
+      rank[result](string info, int usages |
+        usages = getUsages(info)
+      |
+        info order by usages desc, info
+      )
+  }
+
+  /**
+   * Holds if there exists an API with \`apiName\` that is being used \`usages\` times
+   * and if it is in the top results (guarded by resultLimit).
+   */
+  predicate restrict(string apiName, int usages) {
+    usages = getUsages(apiName) and
+    getOrder(apiName) <= resultLimit()
+  }
+}
+`,
+  },
+};

extensions/ql-vscode/src/data-extensions-editor/queries/query.ts

@@ -0,0 +1,15 @@
+export type Query = {
+  /**
+   * The main query.
+   *
+   * It should select all usages of external APIs, and return the following result pattern:
+   * - usage: the usage of the external API. This is an entity.
+   * - apiName: the name of the external API. This is a string.
+   * - supported: whether the external API is supported by the extension. This should be a string representation of a boolean to satify the result pattern for a problem query.
+   * - "supported": a string literal. This is required to make the query a valid problem query.
+   */
+  mainQuery: string;
+  dependencies?: {
+    [filename: string]: string;
+  };
+};

extensions/ql-vscode/src/data-extensions-editor/shared/extension-pack.ts

@@ -0,0 +1,15 @@
+export interface ExtensionPack {
+  path: string;
+  yamlPath: string;
+
+  name: string;
+  version: string;
+
+  extensionTargets: Record<string, string>;
+  dataExtensions: string[];
+}
+
+export interface ExtensionPackModelFile {
+  filename: string;
+  extensionPack: ExtensionPack;
+}

extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts

@@ -0,0 +1,7 @@
+import { ExtensionPackModelFile } from "./extension-pack";
+
+export interface DataExtensionEditorViewState {
+  extensionPackModelFile: ExtensionPackModelFile;
+  modelFileExists: boolean;
+  showLlmButton: boolean;
+}

extensions/ql-vscode/src/data-extensions-editor/yaml.ts

@@ -0,0 +1,131 @@
+import Ajv from "ajv";
+
+import { ExternalApiUsage } from "./external-api-usage";
+import {
+  ModeledMethod,
+  ModeledMethodType,
+  ModeledMethodWithSignature,
+} from "./modeled-method";
+import { extensiblePredicateDefinitions } from "./predicates";
+
+import * as dataSchemaJson from "./data-schema.json";
+
+const ajv = new Ajv({ allErrors: true });
+const dataSchemaValidate = ajv.compile(dataSchemaJson);
+
+type ExternalApiUsageByType = {
+  externalApiUsage: ExternalApiUsage;
+  modeledMethod: ModeledMethod;
+};
+
+type ExtensiblePredicateDefinition = {
+  extensiblePredicate: string;
+  generateMethodDefinition: (method: ExternalApiUsageByType) => any[];
+  readModeledMethod: (row: any[]) => ModeledMethodWithSignature;
+};
+
+function createDataProperty(
+  methods: ExternalApiUsageByType[],
+  definition: ExtensiblePredicateDefinition,
+) {
+  if (methods.length === 0) {
+    return " []";
+  }
+
+  return `\n${methods
+    .map(
+      (method) =>
+        `      - ${JSON.stringify(
+          definition.generateMethodDefinition(method),
+        )}`,
+    )
+    .join("\n")}`;
+}
+
+export function createDataExtensionYaml(
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+) {
+  const methodsByType: Record<
+    Exclude<ModeledMethodType, "none">,
+    ExternalApiUsageByType[]
+  > = {
+    source: [],
+    sink: [],
+    summary: [],
+    neutral: [],
+  };
+
+  for (const externalApiUsage of externalApiUsages) {
+    const modeledMethod = modeledMethods[externalApiUsage.signature];
+
+    if (modeledMethod?.type && modeledMethod.type !== "none") {
+      methodsByType[modeledMethod.type].push({
+        externalApiUsage,
+        modeledMethod,
+      });
+    }
+  }
+
+  const extensions = Object.entries(extensiblePredicateDefinitions).map(
+    ([type, definition]) => `  - addsTo:
+      pack: codeql/${language}-all
+      extensible: ${definition.extensiblePredicate}
+    data:${createDataProperty(
+      methodsByType[type as Exclude<ModeledMethodType, "none">],
+      definition,
+    )}
+`,
+  );
+
+  return `extensions:
+${extensions.join("\n")}`;
+}
+
+export function loadDataExtensionYaml(
+  data: any,
+): Record<string, ModeledMethod> | undefined {
+  dataSchemaValidate(data);
+
+  if (dataSchemaValidate.errors) {
+    throw new Error(
+      `Invalid data extension YAML: ${dataSchemaValidate.errors
+        .map((error) => `${error.instancePath} ${error.message}`)
+        .join(", ")}`,
+    );
+  }
+
+  const extensions = data.extensions;
+  if (!Array.isArray(extensions)) {
+    return undefined;
+  }
+
+  const modeledMethods: Record<string, ModeledMethod> = {};
+
+  for (const extension of extensions) {
+    const addsTo = extension.addsTo;
+    const extensible = addsTo.extensible;
+    const data = extension.data;
+
+    const definition = Object.values(extensiblePredicateDefinitions).find(
+      (definition) => definition.extensiblePredicate === extensible,
+    );
+    if (!definition) {
+      continue;
+    }
+
+    for (const row of data) {
+      const result = definition.readModeledMethod(row);
+      if (!result) {
+        continue;
+      }
+
+      const { signature, modeledMethod } = result;
+
+      modeledMethods[signature] = modeledMethod;
+    }
+  }
+
+  return modeledMethods;
+}

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -0,0 +1,70 @@
+import { retry } from "@octokit/plugin-retry";
+import { throttling } from "@octokit/plugin-throttling";
+import { Octokit } from "@octokit/rest";
+import { Progress, CancellationToken } from "vscode";
+import { showAndLogWarningMessage } from "../helpers";
+import { Credentials } from "../common/authentication";
+
+export async function getCodeSearchRepositories(
+  query: string,
+  progress: Progress<{
+    message?: string | undefined;
+    increment?: number | undefined;
+  }>,
+  token: CancellationToken,
+  credentials: Credentials,
+): Promise<string[]> {
+  let nwos: string[] = [];
+  const octokit = await provideOctokitWithThrottling(credentials);
+
+  for await (const response of octokit.paginate.iterator(
+    octokit.rest.search.code,
+    {
+      q: query,
+      per_page: 100,
+    },
+  )) {
+    nwos.push(...response.data.map((item) => item.repository.full_name));
+    // calculate progress bar: 80% of the progress bar is used for the code search
+    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
+    // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10
+    const increment =
+      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
+    progress.report({ increment });
+
+    if (token.isCancellationRequested) {
+      nwos = [];
+      break;
+    }
+  }
+
+  return [...new Set(nwos)];
+}
+
+async function provideOctokitWithThrottling(
+  credentials: Credentials,
+): Promise<Octokit> {
+  const MyOctokit = Octokit.plugin(throttling);
+  const auth = await credentials.getAccessToken();
+
+  const octokit = new MyOctokit({
+    auth,
+    retry,
+    throttle: {
+      onRateLimit: (retryAfter: number, options: any): boolean => {
+        void showAndLogWarningMessage(
+          `Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`,
+        );
+
+        return true;
+      },
+      onSecondaryRateLimit: (_retryAfter: number, options: any): void => {
+        void showAndLogWarningMessage(
+          `Secondary Rate Limit detected for request ${options.method} ${options.url}`,
+        );
+      },
+    },
+  });
+
+  return octokit;
+}

extensions/ql-vscode/src/databases/config/db-config-store.ts

@@ -61,7 +61,9 @@ export class DbConfigStore extends DisposableObject {
     this.configErrors = [];
     this.configWatcher = undefined;
     this.configValidator = new DbConfigValidator(app.extensionPath);
-    this.onDidChangeConfigEventEmitter = app.createEventEmitter<void>();
+    this.onDidChangeConfigEventEmitter = this.push(
+      app.createEventEmitter<void>(),
+    );
     this.onDidChangeConfig = this.onDidChangeConfigEventEmitter.event;
   }
 
@@ -145,10 +147,46 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
+  /**
+   * Adds a list of remote repositories to an existing repository list and removes duplicates.
+   * @returns a list of repositories that were not added because the list reached 1000 entries.
+   */
+  public async addRemoteReposToList(
+    repoNwoList: string[],
+    parentList: string,
+  ): Promise<string[]> {
+    if (!this.config) {
+      throw Error("Cannot add variant analysis repos if config is not loaded");
+    }
+
+    const config = cloneDbConfig(this.config);
+    const parent = config.databases.variantAnalysis.repositoryLists.find(
+      (list) => list.name === parentList,
+    );
+    if (!parent) {
+      throw Error(`Cannot find parent list '${parentList}'`);
+    }
+
+    // Remove duplicates from the list of repositories.
+    const newRepositoriesList = [
+      ...new Set([...parent.repositories, ...repoNwoList]),
+    ];
+
+    parent.repositories = newRepositoriesList.slice(0, 1000);
+    const truncatedRepositories = newRepositoriesList.slice(1000);
+
+    await this.writeConfig(config);
+    return truncatedRepositories;
+  }
+
+  /**
+   * Adds one remote repository
+   * @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added.
+   */
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repo if config is not loaded");
     }
@@ -163,6 +201,7 @@ export class DbConfigStore extends DisposableObject {
       );
     }
 
+    const truncatedRepositories = [];
     const config = cloneDbConfig(this.config);
     if (parentList) {
       const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -171,12 +210,15 @@ export class DbConfigStore extends DisposableObject {
       if (!parent) {
         throw Error(`Cannot find parent list '${parentList}'`);
       } else {
-        parent.repositories.push(repoNwo);
+        const newRepositories = [...parent.repositories, repoNwo];
+        parent.repositories = newRepositories.slice(0, 1000);
+        truncatedRepositories.push(...newRepositories.slice(1000));
       }
     } else {
       config.databases.variantAnalysis.repositories.push(repoNwo);
     }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteOwner(owner: string): Promise<void> {
@@ -391,14 +433,14 @@ export class DbConfigStore extends DisposableObject {
 
     if (this.configErrors.length === 0) {
       this.config = newConfig;
-      await this.app.executeCommand(
+      await this.app.commands.execute(
         "setContext",
         "codeQLVariantAnalysisRepositories.configError",
         false,
       );
     } else {
       this.config = undefined;
-      await this.app.executeCommand(
+      await this.app.commands.execute(
         "setContext",
         "codeQLVariantAnalysisRepositories.configError",
         true,
@@ -426,14 +468,14 @@ export class DbConfigStore extends DisposableObject {
 
     if (this.configErrors.length === 0) {
       this.config = newConfig;
-      void this.app.executeCommand(
+      void this.app.commands.execute(
         "setContext",
         "codeQLVariantAnalysisRepositories.configError",
         false,
       );
     } else {
       this.config = undefined;
-      void this.app.executeCommand(
+      void this.app.commands.execute(
         "setContext",
         "codeQLVariantAnalysisRepositories.configError",
         true,

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -1,8 +1,8 @@
 import fetch, { Response } from "node-fetch";
 import { zip } from "zip-a-folder";
 import { Open } from "unzipper";
-import { Uri, CancellationToken, commands, window } from "vscode";
-import { CodeQLCliServer } from "./cli";
+import { Uri, CancellationToken, window, InputBoxOptions } from "vscode";
+import { CodeQLCliServer } from "../codeql-cli/cli";
 import {
   ensureDir,
   realpath as fs_realpath,
@@ -17,15 +17,20 @@ import * as Octokit from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
 
 import { DatabaseManager, DatabaseItem } from "./local-databases";
-import { showAndLogInformationMessage, tmpDir } from "./helpers";
-import { reportStreamProgress, ProgressCallback } from "./commandRunner";
-import { extLogger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { showAndLogInformationMessage, tmpDir } from "../helpers";
+import {
+  reportStreamProgress,
+  ProgressCallback,
+} from "../common/vscode/progress";
+import { extLogger } from "../common";
+import { getErrorMessage } from "../pure/helpers-pure";
 import {
   getNwoFromGitHubUrl,
   isValidGitHubNwo,
-} from "./common/github-url-identifier-helper";
-import { Credentials } from "./common/authentication";
+} from "../common/github-url-identifier-helper";
+import { Credentials } from "../common/authentication";
+import { AppCommandManager } from "../common/commands";
+import { ALLOW_HTTP_SETTING } from "../config";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -34,6 +39,7 @@ import { Credentials } from "./common/authentication";
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportInternetDatabase(
+  commandManager: AppCommandManager,
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
@@ -47,7 +53,7 @@ export async function promptImportInternetDatabase(
     return;
   }
 
-  validateHttpsUrl(databaseUrl);
+  validateUrl(databaseUrl);
 
   const item = await databaseArchiveFetcher(
     databaseUrl,
@@ -61,7 +67,7 @@ export async function promptImportInternetDatabase(
   );
 
   if (item) {
-    await commands.executeCommand("codeQLDatabases.focus");
+    await commandManager.execute("codeQLDatabases.focus");
     void showAndLogInformationMessage(
       "Database downloaded and imported successfully.",
     );
@@ -76,8 +82,13 @@ export async function promptImportInternetDatabase(
  *
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
+ * @param credentials the credentials to use to authenticate with GitHub
+ * @param progress the progress callback
+ * @param token the cancellation token
+ * @param cli the CodeQL CLI server
  */
 export async function promptImportGithubDatabase(
+  commandManager: AppCommandManager,
   databaseManager: DatabaseManager,
   storagePath: string,
   credentials: Credentials | undefined,
@@ -85,21 +96,78 @@ export async function promptImportGithubDatabase(
   token: CancellationToken,
   cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
-  progress({
-    message: "Choose repository",
-    step: 1,
-    maxStep: 2,
-  });
-  const githubRepo = await window.showInputBox({
-    title:
-      'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
-    placeHolder: "https://github.com/<owner>/<repo> or <owner>/<repo>",
-    ignoreFocusOut: true,
-  });
+  const githubRepo = await askForGitHubRepo(progress);
   if (!githubRepo) {
     return;
   }
 
+  const databaseItem = await downloadGitHubDatabase(
+    githubRepo,
+    databaseManager,
+    storagePath,
+    credentials,
+    progress,
+    token,
+    cli,
+  );
+
+  if (databaseItem) {
+    await commandManager.execute("codeQLDatabases.focus");
+    void showAndLogInformationMessage(
+      "Database downloaded and imported successfully.",
+    );
+    return databaseItem;
+  }
+
+  return;
+}
+
+export async function askForGitHubRepo(
+  progress?: ProgressCallback,
+  suggestedValue?: string,
+): Promise<string | undefined> {
+  progress?.({
+    message: "Choose repository",
+    step: 1,
+    maxStep: 2,
+  });
+
+  const options: InputBoxOptions = {
+    title:
+      'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
+    placeHolder: "https://github.com/<owner>/<repo> or <owner>/<repo>",
+    ignoreFocusOut: true,
+  };
+
+  if (suggestedValue) {
+    options.value = suggestedValue;
+  }
+
+  return await window.showInputBox(options);
+}
+
+/**
+ * Downloads a database from GitHub
+ *
+ * @param githubRepo the GitHub repository to download the database from
+ * @param databaseManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ * @param credentials the credentials to use to authenticate with GitHub
+ * @param progress the progress callback
+ * @param token the cancellation token
+ * @param cli the CodeQL CLI server
+ * @param language the language to download. If undefined, the user will be prompted to choose a language.
+ **/
+export async function downloadGitHubDatabase(
+  githubRepo: string,
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  credentials: Credentials | undefined,
+  progress: ProgressCallback,
+  token: CancellationToken,
+  cli?: CodeQLCliServer,
+  language?: string,
+): Promise<DatabaseItem | undefined> {
   const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo;
   if (!isValidGitHubNwo(nwo)) {
     throw new Error(`Invalid GitHub repository: ${githubRepo}`);
@@ -109,7 +177,12 @@ export async function promptImportGithubDatabase(
     ? await credentials.getOctokit()
     : new Octokit.Octokit({ retry });
 
-  const result = await convertGithubNwoToDatabaseUrl(nwo, octokit, progress);
+  const result = await convertGithubNwoToDatabaseUrl(
+    nwo,
+    octokit,
+    progress,
+    language,
+  );
   if (!result) {
     return;
   }
@@ -127,7 +200,7 @@ export async function promptImportGithubDatabase(
    * We only need the actual token string.
    */
   const octokitToken = ((await octokit.auth()) as { token: string })?.token;
-  const item = await databaseArchiveFetcher(
+  return await databaseArchiveFetcher(
     databaseUrl,
     {
       Accept: "application/zip",
@@ -140,14 +213,6 @@ export async function promptImportGithubDatabase(
     token,
     cli,
   );
-  if (item) {
-    await commands.executeCommand("codeQLDatabases.focus");
-    void showAndLogInformationMessage(
-      "Database downloaded and imported successfully.",
-    );
-    return item;
-  }
-  return;
 }
 
 /**
@@ -158,6 +223,7 @@ export async function promptImportGithubDatabase(
  * @param storagePath where to store the unzipped database.
  */
 export async function importArchiveDatabase(
+  commandManager: AppCommandManager,
   databaseUrl: string,
   databaseManager: DatabaseManager,
   storagePath: string,
@@ -177,7 +243,7 @@ export async function importArchiveDatabase(
       cli,
     );
     if (item) {
-      await commands.executeCommand("codeQLDatabases.focus");
+      await commandManager.execute("codeQLDatabases.focus");
       void showAndLogInformationMessage(
         "Database unzipped and imported successfully.",
       );
@@ -254,13 +320,15 @@ async function databaseArchiveFetcher(
     });
     await ensureZippedSourceLocation(dbPath);
 
+    const makeSelected = true;
+
     const item = await databaseManager.openDatabase(
       progress,
       token,
       Uri.file(dbPath),
+      makeSelected,
       nameOverride,
     );
-    await databaseManager.setCurrentDatabaseItem(item);
     return item;
   } else {
     throw new Error("Database not found in archive.");
@@ -294,7 +362,7 @@ async function getStorageFolder(storagePath: string, urlStr: string) {
   return folderName;
 }
 
-function validateHttpsUrl(databaseUrl: string) {
+function validateUrl(databaseUrl: string) {
   let uri;
   try {
     uri = Uri.parse(databaseUrl, true);
@@ -302,7 +370,7 @@ function validateHttpsUrl(databaseUrl: string) {
     throw new Error(`Invalid url: ${databaseUrl}`);
   }
 
-  if (uri.scheme !== "https") {
+  if (!ALLOW_HTTP_SETTING.getValue() && uri.scheme !== "https") {
     throw new Error("Must use https for downloading a database.");
   }
 }
@@ -446,6 +514,7 @@ export async function convertGithubNwoToDatabaseUrl(
   nwo: string,
   octokit: Octokit.Octokit,
   progress: ProgressCallback,
+  language?: string,
 ): Promise<
   | {
       databaseUrl: string;
@@ -464,9 +533,11 @@ export async function convertGithubNwoToDatabaseUrl(
 
     const languages = response.data.map((db: any) => db.language);
 
-    const language = await promptForLanguage(languages, progress);
-    if (!language) {
-      return;
+    if (!language || !languages.includes(language)) {
+      language = await promptForLanguage(languages, progress);
+      if (!language) {
+        return;
+      }
     }
 
     return {
@@ -480,7 +551,7 @@ export async function convertGithubNwoToDatabaseUrl(
   }
 }
 
-async function promptForLanguage(
+export async function promptForLanguage(
   languages: string[],
   progress: ProgressCallback,
 ): Promise<string | undefined> {

extensions/ql-vscode/src/databases/db-manager.ts

@@ -1,6 +1,7 @@
 import { App } from "../common/app";
 import { AppEvent, AppEventEmitter } from "../common/events";
 import { ValueResult } from "../common/value-result";
+import { DisposableObject } from "../pure/disposable-object";
 import { DbConfigStore } from "./config/db-config-store";
 import {
   DbItem,
@@ -23,7 +24,7 @@ import {
 import { createRemoteTree } from "./db-tree-creator";
 import { DbConfigValidationError } from "./db-validation-errors";
 
-export class DbManager {
+export class DbManager extends DisposableObject {
   public readonly onDbItemsChanged: AppEvent<void>;
   public static readonly DB_EXPANDED_STATE_KEY = "db_expanded";
   private readonly onDbItemsChangesEventEmitter: AppEventEmitter<void>;
@@ -32,7 +33,11 @@ export class DbManager {
     private readonly app: App,
     private readonly dbConfigStore: DbConfigStore,
   ) {
-    this.onDbItemsChangesEventEmitter = app.createEventEmitter<void>();
+    super();
+
+    this.onDbItemsChangesEventEmitter = this.push(
+      app.createEventEmitter<void>(),
+    );
     this.onDbItemsChanged = this.onDbItemsChangesEventEmitter.event;
 
     this.dbConfigStore.onDidChangeConfig(() => {
@@ -96,8 +101,15 @@ export class DbManager {
   public async addNewRemoteRepo(
     nwo: string,
     parentList?: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  }
+
+  public async addNewRemoteReposToList(
+    nwoList: string[],
+    parentList: string,
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
   }
 
   public async addNewRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/db-module.ts

@@ -1,42 +1,41 @@
 import { window } from "vscode";
-import { App, AppMode } from "../common/app";
+import { App } from "../common/app";
 import { extLogger } from "../common";
 import { DisposableObject } from "../pure/disposable-object";
 import { DbConfigStore } from "./config/db-config-store";
 import { DbManager } from "./db-manager";
 import { DbPanel } from "./ui/db-panel";
 import { DbSelectionDecorationProvider } from "./ui/db-selection-decoration-provider";
-import { isCanary } from "../config";
+import { DatabasePanelCommands } from "../common/commands";
 
 export class DbModule extends DisposableObject {
   public readonly dbManager: DbManager;
   private readonly dbConfigStore: DbConfigStore;
+  private dbPanel: DbPanel | undefined;
 
   private constructor(app: App) {
     super();
 
     this.dbConfigStore = new DbConfigStore(app);
-    this.dbManager = new DbManager(app, this.dbConfigStore);
+    this.dbManager = this.push(new DbManager(app, this.dbConfigStore));
   }
 
-  public static async initialize(app: App): Promise<DbModule | undefined> {
-    if (DbModule.shouldEnableModule(app.mode)) {
-      const dbModule = new DbModule(app);
-      app.subscriptions.push(dbModule);
+  public static async initialize(app: App): Promise<DbModule> {
+    const dbModule = new DbModule(app);
+    app.subscriptions.push(dbModule);
 
-      await dbModule.initialize(app);
-      return dbModule;
-    }
-
-    return undefined;
+    await dbModule.initialize(app);
+    return dbModule;
   }
 
-  private static shouldEnableModule(app: AppMode): boolean {
-    if (app === AppMode.Development || app === AppMode.Test) {
-      return true;
+  public getCommands(): DatabasePanelCommands {
+    if (!this.dbPanel) {
+      throw new Error("Database panel not initialized");
     }
 
-    return isCanary();
+    return {
+      ...this.dbPanel.getCommands(),
+    };
   }
 
   private async initialize(app: App): Promise<void> {
@@ -44,10 +43,9 @@ export class DbModule extends DisposableObject {
 
     await this.dbConfigStore.initialize();
 
-    const dbPanel = new DbPanel(this.dbManager, app.credentials);
-    await dbPanel.initialize();
+    this.dbPanel = new DbPanel(app, this.dbManager);
 
-    this.push(dbPanel);
+    this.push(this.dbPanel);
     this.push(this.dbConfigStore);
 
     const dbSelectionDecorationProvider = new DbSelectionDecorationProvider();

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -0,0 +1,803 @@
+import { join, basename, dirname as path_dirname } from "path";
+import { DisposableObject } from "../pure/disposable-object";
+import {
+  Event,
+  EventEmitter,
+  ProviderResult,
+  TreeDataProvider,
+  TreeItem,
+  Uri,
+  window,
+  env,
+  CancellationToken,
+  ThemeIcon,
+  ThemeColor,
+  workspace,
+  ProgressLocation,
+} from "vscode";
+import { pathExists, stat, readdir, remove } from "fs-extra";
+
+import {
+  DatabaseChangedEvent,
+  DatabaseItem,
+  DatabaseManager,
+} from "./local-databases";
+import {
+  ProgressCallback,
+  ProgressContext,
+  withInheritedProgress,
+  withProgress,
+} from "../common/vscode/progress";
+import {
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder,
+  showAndLogErrorMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../helpers";
+import { extLogger } from "../common";
+import {
+  importArchiveDatabase,
+  promptImportGithubDatabase,
+  promptImportInternetDatabase,
+} from "./database-fetcher";
+import { asError, asyncFilter, getErrorMessage } from "../pure/helpers-pure";
+import { QueryRunner } from "../query-server";
+import { isCanary } from "../config";
+import { App } from "../common/app";
+import { redactableError } from "../pure/errors";
+import { LocalDatabasesCommands } from "../common/commands";
+import {
+  createMultiSelectionCommand,
+  createSingleSelectionCommand,
+} from "../common/vscode/selection-commands";
+
+enum SortOrder {
+  NameAsc = "NameAsc",
+  NameDesc = "NameDesc",
+  DateAddedAsc = "DateAddedAsc",
+  DateAddedDesc = "DateAddedDesc",
+}
+
+/**
+ * Tree data provider for the databases view.
+ */
+class DatabaseTreeDataProvider
+  extends DisposableObject
+  implements TreeDataProvider<DatabaseItem>
+{
+  private _sortOrder = SortOrder.NameAsc;
+
+  private readonly _onDidChangeTreeData = this.push(
+    new EventEmitter<DatabaseItem | undefined>(),
+  );
+  private currentDatabaseItem: DatabaseItem | undefined;
+
+  constructor(private databaseManager: DatabaseManager) {
+    super();
+
+    this.currentDatabaseItem = databaseManager.currentDatabaseItem;
+
+    this.push(
+      this.databaseManager.onDidChangeDatabaseItem(
+        this.handleDidChangeDatabaseItem.bind(this),
+      ),
+    );
+    this.push(
+      this.databaseManager.onDidChangeCurrentDatabaseItem(
+        this.handleDidChangeCurrentDatabaseItem.bind(this),
+      ),
+    );
+  }
+
+  public get onDidChangeTreeData(): Event<DatabaseItem | undefined> {
+    return this._onDidChangeTreeData.event;
+  }
+
+  private handleDidChangeDatabaseItem(event: DatabaseChangedEvent): void {
+    // Note that events from the database manager are instances of DatabaseChangedEvent
+    // and events fired by the UI are instances of DatabaseItem
+
+    // When event.item is undefined, then the entire tree is refreshed.
+    // When event.item is a db item, then only that item is refreshed.
+    this._onDidChangeTreeData.fire(event.item);
+  }
+
+  private handleDidChangeCurrentDatabaseItem(
+    event: DatabaseChangedEvent,
+  ): void {
+    if (this.currentDatabaseItem) {
+      this._onDidChangeTreeData.fire(this.currentDatabaseItem);
+    }
+    this.currentDatabaseItem = event.item;
+    if (this.currentDatabaseItem) {
+      this._onDidChangeTreeData.fire(this.currentDatabaseItem);
+    }
+  }
+
+  public getTreeItem(element: DatabaseItem): TreeItem {
+    const item = new TreeItem(element.name);
+    if (element === this.currentDatabaseItem) {
+      item.iconPath = new ThemeIcon("check");
+
+      item.contextValue = "currentDatabase";
+    } else if (element.error !== undefined) {
+      item.iconPath = new ThemeIcon("error", new ThemeColor("errorForeground"));
+    }
+    item.tooltip = element.databaseUri.fsPath;
+    item.description = element.language;
+    return item;
+  }
+
+  public getChildren(element?: DatabaseItem): ProviderResult<DatabaseItem[]> {
+    if (element === undefined) {
+      return this.databaseManager.databaseItems.slice(0).sort((db1, db2) => {
+        switch (this.sortOrder) {
+          case SortOrder.NameAsc:
+            return db1.name.localeCompare(db2.name, env.language);
+          case SortOrder.NameDesc:
+            return db2.name.localeCompare(db1.name, env.language);
+          case SortOrder.DateAddedAsc:
+            return (db1.dateAdded || 0) - (db2.dateAdded || 0);
+          case SortOrder.DateAddedDesc:
+            return (db2.dateAdded || 0) - (db1.dateAdded || 0);
+        }
+      });
+    } else {
+      return [];
+    }
+  }
+
+  public getParent(_element: DatabaseItem): ProviderResult<DatabaseItem> {
+    return null;
+  }
+
+  public getCurrent(): DatabaseItem | undefined {
+    return this.currentDatabaseItem;
+  }
+
+  public get sortOrder() {
+    return this._sortOrder;
+  }
+
+  public set sortOrder(newSortOrder: SortOrder) {
+    this._sortOrder = newSortOrder;
+    this._onDidChangeTreeData.fire(undefined);
+  }
+}
+
+/** Gets the first element in the given list, if any, or undefined if the list is empty or undefined. */
+function getFirst(list: Uri[] | undefined): Uri | undefined {
+  if (list === undefined || list.length === 0) {
+    return undefined;
+  } else {
+    return list[0];
+  }
+}
+
+/**
+ * Displays file selection dialog. Expects the user to choose a
+ * database directory, which should be the parent directory of a
+ * directory of the form `db-[language]`, for example, `db-cpp`.
+ *
+ * XXX: no validation is done other than checking the directory name
+ * to make sure it really is a database directory.
+ */
+async function chooseDatabaseDir(byFolder: boolean): Promise<Uri | undefined> {
+  const chosen = await window.showOpenDialog({
+    openLabel: byFolder ? "Choose Database folder" : "Choose Database archive",
+    canSelectFiles: !byFolder,
+    canSelectFolders: byFolder,
+    canSelectMany: false,
+    filters: byFolder ? {} : { Archives: ["zip"] },
+  });
+  return getFirst(chosen);
+}
+
+export class DatabaseUI extends DisposableObject {
+  private treeDataProvider: DatabaseTreeDataProvider;
+
+  public constructor(
+    private app: App,
+    private databaseManager: DatabaseManager,
+    private readonly queryServer: QueryRunner | undefined,
+    private readonly storagePath: string,
+    readonly extensionPath: string,
+  ) {
+    super();
+
+    this.treeDataProvider = this.push(
+      new DatabaseTreeDataProvider(databaseManager),
+    );
+    this.push(
+      window.createTreeView("codeQLDatabases", {
+        treeDataProvider: this.treeDataProvider,
+        canSelectMany: true,
+      }),
+    );
+  }
+
+  public getCommands(): LocalDatabasesCommands {
+    return {
+      "codeQL.getCurrentDatabase": this.handleGetCurrentDatabase.bind(this),
+      "codeQL.chooseDatabaseFolder":
+        this.handleChooseDatabaseFolderFromPalette.bind(this),
+      "codeQL.chooseDatabaseArchive":
+        this.handleChooseDatabaseArchiveFromPalette.bind(this),
+      "codeQL.chooseDatabaseInternet":
+        this.handleChooseDatabaseInternet.bind(this),
+      "codeQL.chooseDatabaseGithub": this.handleChooseDatabaseGithub.bind(this),
+      "codeQL.setCurrentDatabase": this.handleSetCurrentDatabase.bind(this),
+      "codeQL.setDefaultTourDatabase":
+        this.handleSetDefaultTourDatabase.bind(this),
+      "codeQL.upgradeCurrentDatabase":
+        this.handleUpgradeCurrentDatabase.bind(this),
+      "codeQL.clearCache": this.handleClearCache.bind(this),
+      "codeQLDatabases.chooseDatabaseFolder":
+        this.handleChooseDatabaseFolder.bind(this),
+      "codeQLDatabases.chooseDatabaseArchive":
+        this.handleChooseDatabaseArchive.bind(this),
+      "codeQLDatabases.chooseDatabaseInternet":
+        this.handleChooseDatabaseInternet.bind(this),
+      "codeQLDatabases.chooseDatabaseGithub":
+        this.handleChooseDatabaseGithub.bind(this),
+      "codeQLDatabases.setCurrentDatabase":
+        this.handleMakeCurrentDatabase.bind(this),
+      "codeQLDatabases.sortByName": this.handleSortByName.bind(this),
+      "codeQLDatabases.sortByDateAdded": this.handleSortByDateAdded.bind(this),
+      "codeQLDatabases.removeDatabase": createMultiSelectionCommand(
+        this.handleRemoveDatabase.bind(this),
+      ),
+      "codeQLDatabases.upgradeDatabase": createMultiSelectionCommand(
+        this.handleUpgradeDatabase.bind(this),
+      ),
+      "codeQLDatabases.renameDatabase": createSingleSelectionCommand(
+        this.handleRenameDatabase.bind(this),
+        "database",
+      ),
+      "codeQLDatabases.openDatabaseFolder": createMultiSelectionCommand(
+        this.handleOpenFolder.bind(this),
+      ),
+      "codeQLDatabases.addDatabaseSource": createMultiSelectionCommand(
+        this.handleAddSource.bind(this),
+      ),
+      "codeQLDatabases.removeOrphanedDatabases":
+        this.handleRemoveOrphanedDatabases.bind(this),
+    };
+  }
+
+  private async handleMakeCurrentDatabase(
+    databaseItem: DatabaseItem,
+  ): Promise<void> {
+    await this.databaseManager.setCurrentDatabaseItem(databaseItem);
+  }
+
+  private async chooseDatabaseFolder(
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> {
+    try {
+      await this.chooseAndSetDatabase(true, { progress, token });
+    } catch (e) {
+      void showAndLogExceptionWithTelemetry(
+        redactableError(
+          asError(e),
+        )`Failed to choose and set database: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+
+  private async handleChooseDatabaseFolder(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        await this.chooseDatabaseFolder(progress, token);
+      },
+      {
+        title: "Adding database from folder",
+      },
+    );
+  }
+
+  private async handleChooseDatabaseFolderFromPalette(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        await this.chooseDatabaseFolder(progress, token);
+      },
+      {
+        title: "Choose a Database from a Folder",
+      },
+    );
+  }
+
+  private async handleSetDefaultTourDatabase(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        try {
+          if (!workspace.workspaceFolders?.length) {
+            throw new Error("No workspace folder is open.");
+          } else {
+            // This specifically refers to the database folder in
+            // https://github.com/github/codespaces-codeql
+            const uri = Uri.parse(
+              `${workspace.workspaceFolders[0].uri}/.tours/codeql-tutorial-database`,
+            );
+
+            const databaseItem = this.databaseManager.findDatabaseItem(uri);
+            if (databaseItem === undefined) {
+              const makeSelected = true;
+              const nameOverride = "CodeQL Tutorial Database";
+              const isTutorialDatabase = true;
+
+              await this.databaseManager.openDatabase(
+                progress,
+                token,
+                uri,
+                makeSelected,
+                nameOverride,
+                isTutorialDatabase,
+              );
+            }
+            await this.handleTourDependencies();
+          }
+        } catch (e) {
+          // rethrow and let this be handled by default error handling.
+          throw new Error(
+            `Could not set the database for the Code Tour. Please make sure you are using the default workspace in your codespace: ${getErrorMessage(
+              e,
+            )}`,
+          );
+        }
+      },
+      {
+        title: "Set Default Database for Codespace CodeQL Tour",
+      },
+    );
+  }
+
+  private async handleTourDependencies(): Promise<void> {
+    if (!workspace.workspaceFolders?.length) {
+      throw new Error("No workspace folder is open.");
+    } else {
+      const tutorialQueriesPath = join(
+        workspace.workspaceFolders[0].uri.fsPath,
+        "tutorial-queries",
+      );
+      const cli = this.queryServer?.cliServer;
+      if (!cli) {
+        throw new Error("No CLI server found");
+      }
+      await cli.packInstall(tutorialQueriesPath);
+    }
+  }
+
+  // Public because it's used in tests
+  public async handleRemoveOrphanedDatabases(): Promise<void> {
+    void extLogger.log("Removing orphaned databases from workspace storage.");
+    let dbDirs = undefined;
+
+    if (
+      !(await pathExists(this.storagePath)) ||
+      !(await stat(this.storagePath)).isDirectory()
+    ) {
+      void extLogger.log(
+        "Missing or invalid storage directory. Not trying to remove orphaned databases.",
+      );
+      return;
+    }
+
+    dbDirs =
+      // read directory
+      (await readdir(this.storagePath, { withFileTypes: true }))
+        // remove non-directories
+        .filter((dirent) => dirent.isDirectory())
+        // get the full path
+        .map((dirent) => join(this.storagePath, dirent.name))
+        // remove databases still in workspace
+        .filter((dbDir) => {
+          const dbUri = Uri.file(dbDir);
+          return this.databaseManager.databaseItems.every(
+            (item) => item.databaseUri.fsPath !== dbUri.fsPath,
+          );
+        });
+
+    // remove non-databases
+    dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
+
+    if (!dbDirs.length) {
+      void extLogger.log("No orphaned databases found.");
+      return;
+    }
+
+    // delete
+    const failures = [] as string[];
+    await Promise.all(
+      dbDirs.map(async (dbDir) => {
+        try {
+          void extLogger.log(`Deleting orphaned database '${dbDir}'.`);
+          await remove(dbDir);
+        } catch (e) {
+          void showAndLogExceptionWithTelemetry(
+            redactableError(
+              asError(e),
+            )`Failed to delete orphaned database: ${getErrorMessage(e)}`,
+          );
+          failures.push(`${basename(dbDir)}`);
+        }
+      }),
+    );
+
+    if (failures.length) {
+      const dirname = path_dirname(failures[0]);
+      void showAndLogErrorMessage(
+        `Failed to delete unused databases (${failures.join(
+          ", ",
+        )}).\nTo delete unused databases, please remove them manually from the storage folder ${dirname}.`,
+      );
+    }
+  }
+
+  private async chooseDatabaseArchive(
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<void> {
+    try {
+      await this.chooseAndSetDatabase(false, { progress, token });
+    } catch (e: unknown) {
+      void showAndLogExceptionWithTelemetry(
+        redactableError(
+          asError(e),
+        )`Failed to choose and set database: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+
+  private async handleChooseDatabaseArchive(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        await this.chooseDatabaseArchive(progress, token);
+      },
+      {
+        title: "Adding database from archive",
+      },
+    );
+  }
+
+  private async handleChooseDatabaseArchiveFromPalette(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        await this.chooseDatabaseArchive(progress, token);
+      },
+      {
+        title: "Choose a Database from an Archive",
+      },
+    );
+  }
+
+  private async handleChooseDatabaseInternet(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        await promptImportInternetDatabase(
+          this.app.commands,
+          this.databaseManager,
+          this.storagePath,
+          progress,
+          token,
+          this.queryServer?.cliServer,
+        );
+      },
+      {
+        title: "Adding database from URL",
+      },
+    );
+  }
+
+  private async handleChooseDatabaseGithub(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        const credentials = isCanary() ? this.app.credentials : undefined;
+
+        await promptImportGithubDatabase(
+          this.app.commands,
+          this.databaseManager,
+          this.storagePath,
+          credentials,
+          progress,
+          token,
+          this.queryServer?.cliServer,
+        );
+      },
+      {
+        title: "Adding database from GitHub",
+      },
+    );
+  }
+
+  private async handleSortByName() {
+    if (this.treeDataProvider.sortOrder === SortOrder.NameAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.NameDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.NameAsc;
+    }
+  }
+
+  private async handleSortByDateAdded() {
+    if (this.treeDataProvider.sortOrder === SortOrder.DateAddedAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.DateAddedDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.DateAddedAsc;
+    }
+  }
+
+  private async handleUpgradeCurrentDatabase(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        if (this.databaseManager.currentDatabaseItem !== undefined) {
+          await this.handleUpgradeDatabasesInternal(progress, token, [
+            this.databaseManager.currentDatabaseItem,
+          ]);
+        }
+      },
+      {
+        title: "Upgrading current database",
+        cancellable: true,
+      },
+    );
+  }
+
+  private async handleUpgradeDatabase(
+    databaseItems: DatabaseItem[],
+  ): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        return await this.handleUpgradeDatabasesInternal(
+          progress,
+          token,
+          databaseItems,
+        );
+      },
+      {
+        title: "Upgrading database",
+        cancellable: true,
+      },
+    );
+  }
+
+  private async handleUpgradeDatabasesInternal(
+    progress: ProgressCallback,
+    token: CancellationToken,
+    databaseItems: DatabaseItem[],
+  ): Promise<void> {
+    await Promise.all(
+      databaseItems.map(async (databaseItem) => {
+        if (this.queryServer === undefined) {
+          throw new Error(
+            "Received request to upgrade database, but there is no running query server.",
+          );
+        }
+        if (databaseItem.contents === undefined) {
+          throw new Error(
+            "Received request to upgrade database, but database contents could not be found.",
+          );
+        }
+        if (databaseItem.contents.dbSchemeUri === undefined) {
+          throw new Error(
+            "Received request to upgrade database, but database has no schema.",
+          );
+        }
+
+        // Search for upgrade scripts in any workspace folders available
+
+        await this.queryServer.upgradeDatabaseExplicit(
+          databaseItem,
+          progress,
+          token,
+        );
+      }),
+    );
+  }
+
+  private async handleClearCache(): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        if (
+          this.queryServer !== undefined &&
+          this.databaseManager.currentDatabaseItem !== undefined
+        ) {
+          await this.queryServer.clearCacheInDatabase(
+            this.databaseManager.currentDatabaseItem,
+            progress,
+            token,
+          );
+        }
+      },
+      {
+        title: "Clearing cache",
+      },
+    );
+  }
+
+  private async handleGetCurrentDatabase(): Promise<string | undefined> {
+    const dbItem = await this.getDatabaseItemInternal(undefined);
+    return dbItem?.databaseUri.fsPath;
+  }
+
+  private async handleSetCurrentDatabase(uri: Uri): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        try {
+          // Assume user has selected an archive if the file has a .zip extension
+          if (uri.path.endsWith(".zip")) {
+            await importArchiveDatabase(
+              this.app.commands,
+              uri.toString(true),
+              this.databaseManager,
+              this.storagePath,
+              progress,
+              token,
+              this.queryServer?.cliServer,
+            );
+          } else {
+            await this.databaseManager.openDatabase(progress, token, uri);
+          }
+        } catch (e) {
+          // rethrow and let this be handled by default error handling.
+          throw new Error(
+            `Could not set database to ${basename(
+              uri.fsPath,
+            )}. Reason: ${getErrorMessage(e)}`,
+          );
+        }
+      },
+      {
+        title: "Importing database from archive",
+      },
+    );
+  }
+
+  private async handleRemoveDatabase(
+    databaseItems: DatabaseItem[],
+  ): Promise<void> {
+    return withProgress(
+      async (progress, token) => {
+        await Promise.all(
+          databaseItems.map((dbItem) =>
+            this.databaseManager.removeDatabaseItem(progress, token, dbItem),
+          ),
+        );
+      },
+      {
+        title: "Removing database",
+        cancellable: false,
+      },
+    );
+  }
+
+  private async handleRenameDatabase(
+    databaseItem: DatabaseItem,
+  ): Promise<void> {
+    const newName = await window.showInputBox({
+      prompt: "Choose new database name",
+      value: databaseItem.name,
+    });
+
+    if (newName) {
+      await this.databaseManager.renameDatabaseItem(databaseItem, newName);
+    }
+  }
+
+  private async handleOpenFolder(databaseItems: DatabaseItem[]): Promise<void> {
+    await Promise.all(
+      databaseItems.map((dbItem) => env.openExternal(dbItem.databaseUri)),
+    );
+  }
+
+  /**
+   * Adds the source folder of a CodeQL database to the workspace.
+   * When a database is first added in the "Databases" view, its source folder is added to the workspace.
+   * If the source folder is removed from the workspace for some reason, we want to be able to re-add it if need be.
+   */
+  private async handleAddSource(databaseItems: DatabaseItem[]): Promise<void> {
+    for (const dbItem of databaseItems) {
+      await this.databaseManager.addDatabaseSourceArchiveFolder(dbItem);
+    }
+  }
+
+  /**
+   * Return the current database directory. If we don't already have a
+   * current database, ask the user for one, and return that, or
+   * undefined if they cancel.
+   */
+  public async getDatabaseItem(
+    progress: ProgressCallback,
+    token: CancellationToken,
+  ): Promise<DatabaseItem | undefined> {
+    return await this.getDatabaseItemInternal({ progress, token });
+  }
+
+  /**
+   * Return the current database directory. If we don't already have a
+   * current database, ask the user for one, and return that, or
+   * undefined if they cancel.
+   *
+   * Unlike `getDatabaseItem()`, this function does not require the caller to pass in a progress
+   * context. If `progress` is `undefined`, then this command will create a new progress
+   * notification if it tries to perform any long-running operations.
+   */
+  private async getDatabaseItemInternal(
+    progress: ProgressContext | undefined,
+  ): Promise<DatabaseItem | undefined> {
+    if (this.databaseManager.currentDatabaseItem === undefined) {
+      await this.chooseAndSetDatabase(false, progress);
+    }
+
+    return this.databaseManager.currentDatabaseItem;
+  }
+
+  /**
+   * Ask the user for a database directory. Returns the chosen database, or `undefined` if the
+   * operation was canceled.
+   */
+  private async chooseAndSetDatabase(
+    byFolder: boolean,
+    progress: ProgressContext | undefined,
+  ): Promise<DatabaseItem | undefined> {
+    const uri = await chooseDatabaseDir(byFolder);
+    if (!uri) {
+      return undefined;
+    }
+
+    return await withInheritedProgress(
+      progress,
+      async (progress, token) => {
+        if (byFolder) {
+          const fixedUri = await this.fixDbUri(uri);
+          // we are selecting a database folder
+          return await this.databaseManager.openDatabase(
+            progress,
+            token,
+            fixedUri,
+          );
+        } else {
+          // we are selecting a database archive. Must unzip into a workspace-controlled area
+          // before importing.
+          return await importArchiveDatabase(
+            this.app.commands,
+            uri.toString(true),
+            this.databaseManager,
+            this.storagePath,
+            progress,
+            token,
+            this.queryServer?.cliServer,
+          );
+        }
+      },
+      {
+        location: ProgressLocation.Notification,
+        cancellable: true,
+        title: "Opening database",
+      },
+    );
+  }
+
+  /**
+   * Perform some heuristics to ensure a proper database location is chosen.
+   *
+   * 1. If the selected URI to add is a file, choose the containing directory
+   * 2. If the selected URI appears to be a db language folder, choose the containing directory
+   * 3. choose the current directory
+   *
+   * @param uri a URI that is a database folder or inside it
+   *
+   * @return the actual database folder found by using the heuristics above.
+   */
+  private async fixDbUri(uri: Uri): Promise<Uri> {
+    let dbPath = uri.fsPath;
+    if ((await stat(dbPath)).isFile()) {
+      dbPath = path_dirname(dbPath);
+    }
+
+    if (await isLikelyDbLanguageFolder(dbPath)) {
+      dbPath = path_dirname(dbPath);
+    }
+    return Uri.file(dbPath);
+  }
+}

extensions/ql-vscode/src/databases/local-databases/database-contents.ts

@@ -0,0 +1,30 @@
+import vscode from "vscode";
+
+/**
+ * The layout of the database.
+ */
+export enum DatabaseKind {
+  /** A CodeQL database */
+  Database,
+  /** A raw QL dataset */
+  RawDataset,
+}
+
+export interface DatabaseContents {
+  /** The layout of the database */
+  kind: DatabaseKind;
+  /**
+   * The name of the database.
+   */
+  name: string;
+  /** The URI of the QL dataset within the database. */
+  datasetUri: vscode.Uri;
+  /** The URI of the source archive within the database, if one exists. */
+  sourceArchiveUri?: vscode.Uri;
+  /** The URI of the CodeQL database scheme within the database, if exactly one exists. */
+  dbSchemeUri?: vscode.Uri;
+}
+
+export interface DatabaseContentsWithDbScheme extends DatabaseContents {
+  dbSchemeUri: vscode.Uri; // Always present
+}

extensions/ql-vscode/src/databases/local-databases/database-events.ts

@@ -0,0 +1,19 @@
+import { DatabaseItem } from "./database-item";
+
+export enum DatabaseEventKind {
+  Add = "Add",
+  Remove = "Remove",
+
+  // Fired when databases are refreshed from persisted state
+  Refresh = "Refresh",
+
+  // Fired when the current database changes
+  Change = "Change",
+
+  Rename = "Rename",
+}
+
+export interface DatabaseChangedEvent {
+  kind: DatabaseEventKind;
+  item: DatabaseItem | undefined;
+}

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -0,0 +1,218 @@
+// Exported for testing
+import * as cli from "../../codeql-cli/cli";
+import vscode from "vscode";
+import { FullDatabaseOptions } from "./database-options";
+import { basename, dirname, extname, join, relative } from "path";
+import {
+  decodeSourceArchiveUri,
+  encodeArchiveBasePath,
+  encodeSourceArchiveUri,
+  zipArchiveScheme,
+} from "../../common/vscode/archive-filesystem-provider";
+import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
+import { isLikelyDatabaseRoot } from "../../helpers";
+import { stat } from "fs-extra";
+import { pathsEqual } from "../../pure/files";
+import { DatabaseContents } from "./database-contents";
+
+export class DatabaseItemImpl implements DatabaseItem {
+  // These are only public in the implementation, they are readonly in the interface
+  public error: Error | undefined = undefined;
+  public contents: DatabaseContents | undefined;
+  /** A cache of database info */
+  private _dbinfo: cli.DbInfo | undefined;
+
+  public constructor(
+    public readonly databaseUri: vscode.Uri,
+    contents: DatabaseContents | undefined,
+    private options: FullDatabaseOptions,
+  ) {
+    this.contents = contents;
+  }
+
+  public get name(): string {
+    if (this.options.displayName) {
+      return this.options.displayName;
+    } else if (this.contents) {
+      return this.contents.name;
+    } else {
+      return basename(this.databaseUri.fsPath);
+    }
+  }
+
+  public set name(newName: string) {
+    this.options.displayName = newName;
+  }
+
+  public get sourceArchive(): vscode.Uri | undefined {
+    if (this.ignoreSourceArchive || this.contents === undefined) {
+      return undefined;
+    } else {
+      return this.contents.sourceArchiveUri;
+    }
+  }
+
+  private get ignoreSourceArchive(): boolean {
+    // Ignore the source archive for QLTest databases.
+    return extname(this.databaseUri.fsPath) === ".testproj";
+  }
+
+  public get dateAdded(): number | undefined {
+    return this.options.dateAdded;
+  }
+
+  public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
+    const sourceArchive = this.sourceArchive;
+    const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
+    if (uri && uri.scheme !== "file") {
+      throw new Error(
+        `Invalid uri scheme in ${uriStr}. Only 'file' is allowed.`,
+      );
+    }
+    if (!sourceArchive) {
+      if (uri) {
+        return uri;
+      } else {
+        return this.databaseUri;
+      }
+    }
+
+    if (uri) {
+      const relativeFilePath = decodeURI(uri.path)
+        .replace(":", "_")
+        .replace(/^\/*/, "");
+      if (sourceArchive.scheme === zipArchiveScheme) {
+        const zipRef = decodeSourceArchiveUri(sourceArchive);
+        const pathWithinSourceArchive =
+          zipRef.pathWithinSourceArchive === "/"
+            ? relativeFilePath
+            : `${zipRef.pathWithinSourceArchive}/${relativeFilePath}`;
+        return encodeSourceArchiveUri({
+          pathWithinSourceArchive,
+          sourceArchiveZipPath: zipRef.sourceArchiveZipPath,
+        });
+      } else {
+        let newPath = sourceArchive.path;
+        if (!newPath.endsWith("/")) {
+          // Ensure a trailing slash.
+          newPath += "/";
+        }
+        newPath += relativeFilePath;
+
+        return sourceArchive.with({ path: newPath });
+      }
+    } else {
+      return sourceArchive;
+    }
+  }
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  public getPersistedState(): PersistedDatabaseItem {
+    return {
+      uri: this.databaseUri.toString(true),
+      options: this.options,
+    };
+  }
+
+  /**
+   * Holds if the database item refers to an exported snapshot
+   */
+  public async hasMetadataFile(): Promise<boolean> {
+    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
+  }
+
+  /**
+   * Returns information about a database.
+   */
+  private async getDbInfo(server: cli.CodeQLCliServer): Promise<cli.DbInfo> {
+    if (this._dbinfo === undefined) {
+      this._dbinfo = await server.resolveDatabase(this.databaseUri.fsPath);
+    }
+    return this._dbinfo;
+  }
+
+  /**
+   * Returns `sourceLocationPrefix` of database. Requires that the database
+   * has a `.dbinfo` file, which is the source of the prefix.
+   */
+  public async getSourceLocationPrefix(
+    server: cli.CodeQLCliServer,
+  ): Promise<string> {
+    const dbInfo = await this.getDbInfo(server);
+    return dbInfo.sourceLocationPrefix;
+  }
+
+  /**
+   * Returns path to dataset folder of database.
+   */
+  public async getDatasetFolder(server: cli.CodeQLCliServer): Promise<string> {
+    const dbInfo = await this.getDbInfo(server);
+    return dbInfo.datasetFolder;
+  }
+
+  public get language() {
+    return this.options.language || "";
+  }
+
+  /**
+   * Returns the root uri of the virtual filesystem for this database's source archive.
+   */
+  public getSourceArchiveExplorerUri(): vscode.Uri {
+    const sourceArchive = this.sourceArchive;
+    if (sourceArchive === undefined || !sourceArchive.fsPath.endsWith(".zip")) {
+      throw new Error(this.verifyZippedSources());
+    }
+    return encodeArchiveBasePath(sourceArchive.fsPath);
+  }
+
+  public verifyZippedSources(): string | undefined {
+    const sourceArchive = this.sourceArchive;
+    if (sourceArchive === undefined) {
+      return `${this.name} has no source archive.`;
+    }
+
+    if (!sourceArchive.fsPath.endsWith(".zip")) {
+      return `${this.name} has a source folder that is unzipped.`;
+    }
+    return;
+  }
+
+  /**
+   * Holds if `uri` belongs to this database's source archive.
+   */
+  public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean {
+    if (this.sourceArchive === undefined) return false;
+    return (
+      uri.scheme === zipArchiveScheme &&
+      decodeSourceArchiveUri(uri).sourceArchiveZipPath ===
+        this.sourceArchive.fsPath
+    );
+  }
+
+  public async isAffectedByTest(testPath: string): Promise<boolean> {
+    const databasePath = this.databaseUri.fsPath;
+    if (!databasePath.endsWith(".testproj")) {
+      return false;
+    }
+    try {
+      const stats = await stat(testPath);
+      if (stats.isDirectory()) {
+        return !relative(testPath, databasePath).startsWith("..");
+      } else {
+        // database for /one/two/three/test.ql is at /one/two/three/three.testproj
+        const testdir = dirname(testPath);
+        const testdirbase = basename(testdir);
+        return pathsEqual(
+          databasePath,
+          join(testdir, `${testdirbase}.testproj`),
+          process.platform,
+        );
+      }
+    } catch {
+      // No information available for test path - assume database is unaffected.
+      return false;
+    }
+  }
+}

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -0,0 +1,83 @@
+import vscode from "vscode";
+import * as cli from "../../codeql-cli/cli";
+import { DatabaseContents } from "./database-contents";
+import { DatabaseOptions } from "./database-options";
+
+/** An item in the list of available databases */
+export interface DatabaseItem {
+  /** The URI of the database */
+  readonly databaseUri: vscode.Uri;
+  /** The name of the database to be displayed in the UI */
+  name: string;
+
+  /** The primary language of the database or empty string if unknown */
+  readonly language: string;
+  /** The URI of the database's source archive, or `undefined` if no source archive is to be used. */
+  readonly sourceArchive: vscode.Uri | undefined;
+  /**
+   * The contents of the database.
+   * Will be `undefined` if the database is invalid. Can be updated by calling `refresh()`.
+   */
+  readonly contents: DatabaseContents | undefined;
+
+  /**
+   * The date this database was added as a unix timestamp. Or undefined if we don't know.
+   */
+  readonly dateAdded: number | undefined;
+
+  /** If the database is invalid, describes why. */
+  readonly error: Error | undefined;
+
+  /**
+   * Resolves a filename to its URI in the source archive.
+   *
+   * @param file Filename within the source archive. May be `undefined` to return a dummy file path.
+   */
+  resolveSourceFile(file: string | undefined): vscode.Uri;
+
+  /**
+   * Holds if the database item has a `.dbinfo` or `codeql-database.yml` file.
+   */
+  hasMetadataFile(): Promise<boolean>;
+
+  /**
+   * Returns `sourceLocationPrefix` of exported database.
+   */
+  getSourceLocationPrefix(server: cli.CodeQLCliServer): Promise<string>;
+
+  /**
+   * Returns dataset folder of exported database.
+   */
+  getDatasetFolder(server: cli.CodeQLCliServer): Promise<string>;
+
+  /**
+   * Returns the root uri of the virtual filesystem for this database's source archive,
+   * as displayed in the filesystem explorer.
+   */
+  getSourceArchiveExplorerUri(): vscode.Uri;
+
+  /**
+   * Holds if `uri` belongs to this database's source archive.
+   */
+  belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean;
+
+  /**
+   * Whether the database may be affected by test execution for the given path.
+   */
+  isAffectedByTest(testPath: string): Promise<boolean>;
+
+  /**
+   * Gets the state of this database, to be persisted in the workspace state.
+   */
+  getPersistedState(): PersistedDatabaseItem;
+
+  /**
+   * Verifies that this database item has a zipped source folder. Returns an error message if it does not.
+   */
+  verifyZippedSources(): string | undefined;
+}
+
+export interface PersistedDatabaseItem {
+  uri: string;
+  options?: DatabaseOptions;
+}

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -0,0 +1,664 @@
+import vscode, { ExtensionContext } from "vscode";
+import { extLogger, Logger } from "../../common";
+import { DisposableObject } from "../../pure/disposable-object";
+import { App } from "../../common/app";
+import { QueryRunner } from "../../query-server";
+import * as cli from "../../codeql-cli/cli";
+import { ProgressCallback, withProgress } from "../../common/vscode/progress";
+import {
+  getAutogenerateQlPacks,
+  isCodespacesTemplate,
+  setAutogenerateQlPacks,
+} from "../../config";
+import { join } from "path";
+import { FullDatabaseOptions } from "./database-options";
+import { DatabaseItemImpl } from "./database-item-impl";
+import {
+  getFirstWorkspaceFolder,
+  isFolderAlreadyInWorkspace,
+  isQueryLanguage,
+  showAndLogExceptionWithTelemetry,
+  showNeverAskAgainDialog,
+} from "../../helpers";
+import { existsSync } from "fs";
+import { QlPackGenerator } from "../../qlpack-generator";
+import { asError, getErrorMessage } from "../../pure/helpers-pure";
+import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
+import { redactableError } from "../../pure/errors";
+import { remove } from "fs-extra";
+import { containsPath } from "../../pure/files";
+import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
+import { DatabaseResolver } from "./database-resolver";
+
+/**
+ * The name of the key in the workspaceState dictionary in which we
+ * persist the current database across sessions.
+ */
+const CURRENT_DB = "currentDatabase";
+
+/**
+ * The name of the key in the workspaceState dictionary in which we
+ * persist the list of databases across sessions.
+ */
+const DB_LIST = "databaseList";
+
+/**
+ * A promise that resolves to an event's result value when the event
+ * `event` fires. If waiting for the event takes too long (by default
+ * >1000ms) log a warning, and resolve to undefined.
+ */
+function eventFired<T>(
+  event: vscode.Event<T>,
+  timeoutMs = 1000,
+): Promise<T | undefined> {
+  return new Promise((res, _rej) => {
+    const timeout = setTimeout(() => {
+      void extLogger.log(
+        `Waiting for event ${event} timed out after ${timeoutMs}ms`,
+      );
+      res(undefined);
+      dispose();
+    }, timeoutMs);
+    const disposable = event((e) => {
+      res(e);
+      dispose();
+    });
+    function dispose() {
+      clearTimeout(timeout);
+      disposable.dispose();
+    }
+  });
+}
+
+export class DatabaseManager extends DisposableObject {
+  private readonly _onDidChangeDatabaseItem = this.push(
+    new vscode.EventEmitter<DatabaseChangedEvent>(),
+  );
+
+  readonly onDidChangeDatabaseItem = this._onDidChangeDatabaseItem.event;
+
+  private readonly _onDidChangeCurrentDatabaseItem = this.push(
+    new vscode.EventEmitter<DatabaseChangedEvent>(),
+  );
+  readonly onDidChangeCurrentDatabaseItem =
+    this._onDidChangeCurrentDatabaseItem.event;
+
+  private readonly _databaseItems: DatabaseItemImpl[] = [];
+  private _currentDatabaseItem: DatabaseItem | undefined = undefined;
+
+  constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly app: App,
+    private readonly qs: QueryRunner,
+    private readonly cli: cli.CodeQLCliServer,
+    public logger: Logger,
+  ) {
+    super();
+
+    qs.onStart(this.reregisterDatabases.bind(this));
+  }
+
+  /**
+   * Creates a {@link DatabaseItem} for the specified database, and adds it to the list of open
+   * databases.
+   */
+  public async openDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    uri: vscode.Uri,
+    makeSelected = true,
+    displayName?: string,
+    isTutorialDatabase?: boolean,
+  ): Promise<DatabaseItem> {
+    const databaseItem = await this.createDatabaseItem(uri, displayName);
+
+    return await this.addExistingDatabaseItem(
+      databaseItem,
+      progress,
+      makeSelected,
+      token,
+      isTutorialDatabase,
+    );
+  }
+
+  /**
+   * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
+   * the list.
+   *
+   * Typically, the item will have been created by {@link createOrOpenDatabaseItem} or {@link openDatabase}.
+   */
+  private async addExistingDatabaseItem(
+    databaseItem: DatabaseItemImpl,
+    progress: ProgressCallback,
+    makeSelected: boolean,
+    token: vscode.CancellationToken,
+    isTutorialDatabase?: boolean,
+  ): Promise<DatabaseItem> {
+    const existingItem = this.findDatabaseItem(databaseItem.databaseUri);
+    if (existingItem !== undefined) {
+      if (makeSelected) {
+        await this.setCurrentDatabaseItem(existingItem);
+      }
+      return existingItem;
+    }
+
+    await this.addDatabaseItem(progress, token, databaseItem);
+    if (makeSelected) {
+      await this.setCurrentDatabaseItem(databaseItem);
+    }
+    await this.addDatabaseSourceArchiveFolder(databaseItem);
+
+    if (isCodespacesTemplate() && !isTutorialDatabase) {
+      await this.createSkeletonPacks(databaseItem);
+    }
+
+    return databaseItem;
+  }
+
+  /**
+   * Creates a {@link DatabaseItem} for the specified database, without adding it to the list of
+   * open databases.
+   */
+  private async createDatabaseItem(
+    uri: vscode.Uri,
+    displayName: string | undefined,
+  ): Promise<DatabaseItemImpl> {
+    const contents = await DatabaseResolver.resolveDatabaseContents(uri);
+    const fullOptions: FullDatabaseOptions = {
+      // If a displayName is not passed in, the basename of folder containing the database is used.
+      displayName,
+      dateAdded: Date.now(),
+      language: await this.getPrimaryLanguage(uri.fsPath),
+    };
+    const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
+
+    return databaseItem;
+  }
+
+  /**
+   * If the specified database is already on the list of open databases, returns that database's
+   * {@link DatabaseItem}. Otherwise, creates a new {@link DatabaseItem} without adding it to the
+   * list of open databases.
+   *
+   * The {@link DatabaseItem} can be added to the list of open databases later, via {@link addExistingDatabaseItem}.
+   */
+  public async createOrOpenDatabaseItem(
+    uri: vscode.Uri,
+  ): Promise<DatabaseItem> {
+    const existingItem = this.findDatabaseItem(uri);
+    if (existingItem !== undefined) {
+      // Use the one we already have.
+      return existingItem;
+    }
+
+    // We don't add this to the list automatically, but the user can add it later.
+    return this.createDatabaseItem(uri, undefined);
+  }
+
+  public async createSkeletonPacks(databaseItem: DatabaseItem) {
+    if (databaseItem === undefined) {
+      void this.logger.log(
+        "Could not create QL pack because no database is selected. Please add a database.",
+      );
+      return;
+    }
+
+    if (databaseItem.language === "") {
+      void this.logger.log(
+        "Could not create skeleton QL pack because the selected database's language is not set.",
+      );
+      return;
+    }
+
+    if (!isQueryLanguage(databaseItem.language)) {
+      void this.logger.log(
+        "Could not create skeleton QL pack because the selected database's language is not supported.",
+      );
+      return;
+    }
+
+    const firstWorkspaceFolder = getFirstWorkspaceFolder();
+    const folderName = `codeql-custom-queries-${databaseItem.language}`;
+
+    if (
+      existsSync(join(firstWorkspaceFolder, folderName)) ||
+      isFolderAlreadyInWorkspace(folderName)
+    ) {
+      return;
+    }
+
+    if (getAutogenerateQlPacks() === "never") {
+      return;
+    }
+
+    const answer = await showNeverAskAgainDialog(
+      `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`,
+    );
+
+    if (answer === "No") {
+      return;
+    }
+
+    if (answer === "No, and never ask me again") {
+      await setAutogenerateQlPacks("never");
+      return;
+    }
+
+    try {
+      const qlPackGenerator = new QlPackGenerator(
+        folderName,
+        databaseItem.language,
+        this.cli,
+        firstWorkspaceFolder,
+      );
+      await qlPackGenerator.generate();
+    } catch (e: unknown) {
+      void this.logger.log(
+        `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
+      );
+    }
+  }
+
+  private async reregisterDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+  ) {
+    let completed = 0;
+    await Promise.all(
+      this._databaseItems.map(async (databaseItem) => {
+        await this.registerDatabase(progress, token, databaseItem);
+        completed++;
+        progress({
+          maxStep: this._databaseItems.length,
+          step: completed,
+          message: "Re-registering databases",
+        });
+      }),
+    );
+  }
+
+  public async addDatabaseSourceArchiveFolder(item: DatabaseItem) {
+    // The folder may already be in workspace state from a previous
+    // session. If not, add it.
+    const index = this.getDatabaseWorkspaceFolderIndex(item);
+    if (index === -1) {
+      // Add that filesystem as a folder to the current workspace.
+      //
+      // It's important that we add workspace folders to the end,
+      // rather than beginning of the list, because the first
+      // workspace folder is special; if it gets updated, the entire
+      // extension host is restarted. (cf.
+      // https://github.com/microsoft/vscode/blob/e0d2ed907d1b22808c56127678fb436d604586a7/src/vs/workbench/contrib/relauncher/browser/relauncher.contribution.ts#L209-L214)
+      //
+      // This is undesirable, as we might be adding and removing many
+      // workspace folders as the user adds and removes databases.
+      const end = (vscode.workspace.workspaceFolders || []).length;
+
+      const msg = item.verifyZippedSources();
+      if (msg) {
+        void extLogger.log(`Could not add source folder because ${msg}`);
+        return;
+      }
+
+      const uri = item.getSourceArchiveExplorerUri();
+      void extLogger.log(
+        `Adding workspace folder for ${item.name} source archive at index ${end}`,
+      );
+      if ((vscode.workspace.workspaceFolders || []).length < 2) {
+        // Adding this workspace folder makes the workspace
+        // multi-root, which may surprise the user. Let them know
+        // we're doing this.
+        void vscode.window.showInformationMessage(
+          `Adding workspace folder for source archive of database ${item.name}.`,
+        );
+      }
+      vscode.workspace.updateWorkspaceFolders(end, 0, {
+        name: `[${item.name} source archive]`,
+        uri,
+      });
+      // vscode api documentation says we must to wait for this event
+      // between multiple `updateWorkspaceFolders` calls.
+      await eventFired(vscode.workspace.onDidChangeWorkspaceFolders);
+    }
+  }
+
+  private async createDatabaseItemFromPersistedState(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    state: PersistedDatabaseItem,
+  ): Promise<DatabaseItemImpl> {
+    let displayName: string | undefined = undefined;
+    let dateAdded = undefined;
+    let language = undefined;
+    if (state.options) {
+      if (typeof state.options.displayName === "string") {
+        displayName = state.options.displayName;
+      }
+      if (typeof state.options.dateAdded === "number") {
+        dateAdded = state.options.dateAdded;
+      }
+      language = state.options.language;
+    }
+
+    const dbBaseUri = vscode.Uri.parse(state.uri, true);
+    if (language === undefined) {
+      // we haven't been successful yet at getting the language. try again
+      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
+    }
+
+    const fullOptions: FullDatabaseOptions = {
+      displayName,
+      dateAdded,
+      language,
+    };
+    const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
+
+    // Avoid persisting the database state after adding since that should happen only after
+    // all databases have been added.
+    await this.addDatabaseItem(progress, token, item, false);
+    return item;
+  }
+
+  public async loadPersistedState(): Promise<void> {
+    return withProgress(async (progress, token) => {
+      const currentDatabaseUri =
+        this.ctx.workspaceState.get<string>(CURRENT_DB);
+      const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(
+        DB_LIST,
+        [],
+      );
+      let step = 0;
+      progress({
+        maxStep: databases.length,
+        message: "Loading persisted databases",
+        step,
+      });
+      try {
+        void this.logger.log(
+          `Found ${databases.length} persisted databases: ${databases
+            .map((db) => db.uri)
+            .join(", ")}`,
+        );
+        for (const database of databases) {
+          progress({
+            maxStep: databases.length,
+            message: `Loading ${database.options?.displayName || "databases"}`,
+            step: ++step,
+          });
+
+          const databaseItem = await this.createDatabaseItemFromPersistedState(
+            progress,
+            token,
+            database,
+          );
+          try {
+            await this.refreshDatabase(databaseItem);
+            await this.registerDatabase(progress, token, databaseItem);
+            if (currentDatabaseUri === database.uri) {
+              await this.setCurrentDatabaseItem(databaseItem, true);
+            }
+            void this.logger.log(
+              `Loaded database ${databaseItem.name} at URI ${database.uri}.`,
+            );
+          } catch (e) {
+            // When loading from persisted state, leave invalid databases in the list. They will be
+            // marked as invalid, and cannot be set as the current database.
+            void this.logger.log(
+              `Error loading database ${database.uri}: ${e}.`,
+            );
+          }
+        }
+        await this.updatePersistedDatabaseList();
+      } catch (e) {
+        // database list had an unexpected type - nothing to be done?
+        void showAndLogExceptionWithTelemetry(
+          redactableError(
+            asError(e),
+          )`Database list loading failed: ${getErrorMessage(e)}`,
+        );
+      }
+
+      void this.logger.log("Finished loading persisted databases.");
+    });
+  }
+
+  public get databaseItems(): readonly DatabaseItem[] {
+    return this._databaseItems;
+  }
+
+  public get currentDatabaseItem(): DatabaseItem | undefined {
+    return this._currentDatabaseItem;
+  }
+
+  public async setCurrentDatabaseItem(
+    item: DatabaseItem | undefined,
+    skipRefresh = false,
+  ): Promise<void> {
+    if (
+      !skipRefresh &&
+      item !== undefined &&
+      item instanceof DatabaseItemImpl
+    ) {
+      await this.refreshDatabase(item); // Will throw on invalid database.
+    }
+    if (this._currentDatabaseItem !== item) {
+      this._currentDatabaseItem = item;
+      this.updatePersistedCurrentDatabaseItem();
+
+      await this.app.commands.execute(
+        "setContext",
+        "codeQL.currentDatabaseItem",
+        item?.name,
+      );
+
+      this._onDidChangeCurrentDatabaseItem.fire({
+        item,
+        kind: DatabaseEventKind.Change,
+      });
+    }
+  }
+
+  /**
+   * Returns the index of the workspace folder that corresponds to the source archive of `item`
+   * if there is one, and -1 otherwise.
+   */
+  private getDatabaseWorkspaceFolderIndex(item: DatabaseItem): number {
+    return (vscode.workspace.workspaceFolders || []).findIndex((folder) =>
+      item.belongsToSourceArchiveExplorerUri(folder.uri),
+    );
+  }
+
+  public findDatabaseItem(uri: vscode.Uri): DatabaseItem | undefined {
+    const uriString = uri.toString(true);
+    return this._databaseItems.find(
+      (item) => item.databaseUri.toString(true) === uriString,
+    );
+  }
+
+  public findDatabaseItemBySourceArchive(
+    uri: vscode.Uri,
+  ): DatabaseItem | undefined {
+    const uriString = uri.toString(true);
+    return this._databaseItems.find(
+      (item) =>
+        item.sourceArchive && item.sourceArchive.toString(true) === uriString,
+    );
+  }
+
+  private async addDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItemImpl,
+    updatePersistedState = true,
+  ) {
+    this._databaseItems.push(item);
+
+    if (updatePersistedState) {
+      await this.updatePersistedDatabaseList();
+    }
+
+    // Add this database item to the allow-list
+    // Database items reconstituted from persisted state
+    // will not have their contents yet.
+    if (item.contents?.datasetUri) {
+      await this.registerDatabase(progress, token, item);
+    }
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Add,
+    });
+  }
+
+  public async renameDatabaseItem(item: DatabaseItem, newName: string) {
+    item.name = newName;
+    await this.updatePersistedDatabaseList();
+    this._onDidChangeDatabaseItem.fire({
+      // pass undefined so that the entire tree is rebuilt in order to re-sort
+      item: undefined,
+      kind: DatabaseEventKind.Rename,
+    });
+  }
+
+  public async removeDatabaseItem(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    item: DatabaseItem,
+  ) {
+    if (this._currentDatabaseItem === item) {
+      this._currentDatabaseItem = undefined;
+    }
+    const index = this.databaseItems.findIndex(
+      (searchItem) => searchItem === item,
+    );
+    if (index >= 0) {
+      this._databaseItems.splice(index, 1);
+    }
+    await this.updatePersistedDatabaseList();
+
+    // Delete folder from workspace, if it is still there
+    const folderIndex = (vscode.workspace.workspaceFolders || []).findIndex(
+      (folder) => item.belongsToSourceArchiveExplorerUri(folder.uri),
+    );
+    if (folderIndex >= 0) {
+      void extLogger.log(`Removing workspace folder at index ${folderIndex}`);
+      vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
+    }
+
+    // Remove this database item from the allow-list
+    await this.deregisterDatabase(progress, token, item);
+
+    // Delete folder from file system only if it is controlled by the extension
+    if (this.isExtensionControlledLocation(item.databaseUri)) {
+      void extLogger.log("Deleting database from filesystem.");
+      await remove(item.databaseUri.fsPath).then(
+        () => void extLogger.log(`Deleted '${item.databaseUri.fsPath}'`),
+        (e: unknown) =>
+          void extLogger.log(
+            `Failed to delete '${
+              item.databaseUri.fsPath
+            }'. Reason: ${getErrorMessage(e)}`,
+          ),
+      );
+    }
+
+    // note that we use undefined as the item in order to reset the entire tree
+    this._onDidChangeDatabaseItem.fire({
+      item: undefined,
+      kind: DatabaseEventKind.Remove,
+    });
+  }
+
+  public async removeAllDatabases(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+  ) {
+    for (const item of this.databaseItems) {
+      await this.removeDatabaseItem(progress, token, item);
+    }
+  }
+
+  private async deregisterDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    try {
+      await this.qs.deregisterDatabase(progress, token, dbItem);
+    } catch (e) {
+      const message = getErrorMessage(e);
+      if (message === "Connection is disposed.") {
+        // This is expected if the query server is not running.
+        void extLogger.log(
+          `Could not de-register database '${dbItem.name}' because query server is not running.`,
+        );
+        return;
+      }
+      throw e;
+    }
+  }
+  private async registerDatabase(
+    progress: ProgressCallback,
+    token: vscode.CancellationToken,
+    dbItem: DatabaseItem,
+  ) {
+    await this.qs.registerDatabase(progress, token, dbItem);
+  }
+
+  /**
+   * Resolves the contents of the database.
+   *
+   * @remarks
+   * The contents include the database directory, source archive, and metadata about the database.
+   * If the database is invalid, `databaseItem.error` is updated with the error object that describes why
+   * the database is invalid. This error is also thrown.
+   */
+  private async refreshDatabase(databaseItem: DatabaseItemImpl) {
+    try {
+      try {
+        databaseItem.contents = await DatabaseResolver.resolveDatabaseContents(
+          databaseItem.databaseUri,
+        );
+        databaseItem.error = undefined;
+      } catch (e) {
+        databaseItem.contents = undefined;
+        databaseItem.error = asError(e);
+        throw e;
+      }
+    } finally {
+      this._onDidChangeDatabaseItem.fire({
+        kind: DatabaseEventKind.Refresh,
+        item: databaseItem,
+      });
+    }
+  }
+
+  private updatePersistedCurrentDatabaseItem(): void {
+    void this.ctx.workspaceState.update(
+      CURRENT_DB,
+      this._currentDatabaseItem
+        ? this._currentDatabaseItem.databaseUri.toString(true)
+        : undefined,
+    );
+  }
+
+  private async updatePersistedDatabaseList(): Promise<void> {
+    await this.ctx.workspaceState.update(
+      DB_LIST,
+      this._databaseItems.map((item) => item.getPersistedState()),
+    );
+  }
+
+  private isExtensionControlledLocation(uri: vscode.Uri) {
+    const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri;
+    if (storageUri) {
+      return containsPath(storageUri.fsPath, uri.fsPath, process.platform);
+    }
+    return false;
+  }
+
+  private async getPrimaryLanguage(dbPath: string) {
+    const dbInfo = await this.cli.resolveDatabase(dbPath);
+    return dbInfo.languages?.[0] || "";
+  }
+}

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -0,0 +1,10 @@
+export interface DatabaseOptions {
+  displayName?: string;
+  dateAdded?: number | undefined;
+  language?: string;
+}
+
+export interface FullDatabaseOptions extends DatabaseOptions {
+  dateAdded: number | undefined;
+  language: string | undefined;
+}

extensions/ql-vscode/src/databases/local-databases/database-resolver.ts

@@ -0,0 +1,144 @@
+import vscode from "vscode";
+import { pathExists } from "fs-extra";
+import { basename, join, resolve } from "path";
+import {
+  DatabaseContents,
+  DatabaseContentsWithDbScheme,
+  DatabaseKind,
+} from "./database-contents";
+import { glob } from "glob";
+import {
+  showAndLogInformationMessage,
+  showAndLogWarningMessage,
+} from "../../helpers";
+import { encodeArchiveBasePath } from "../../common/vscode/archive-filesystem-provider";
+
+export class DatabaseResolver {
+  public static async resolveDatabaseContents(
+    uri: vscode.Uri,
+  ): Promise<DatabaseContentsWithDbScheme> {
+    if (uri.scheme !== "file") {
+      throw new Error(
+        `Database URI scheme '${uri.scheme}' not supported; only 'file' URIs are supported.`,
+      );
+    }
+    const databasePath = uri.fsPath;
+    if (!(await pathExists(databasePath))) {
+      throw new InvalidDatabaseError(
+        `Database '${databasePath}' does not exist.`,
+      );
+    }
+
+    const contents = await this.resolveDatabase(databasePath);
+
+    if (contents === undefined) {
+      throw new InvalidDatabaseError(
+        `'${databasePath}' is not a valid database.`,
+      );
+    }
+
+    // Look for a single dbscheme file within the database.
+    // This should be found in the dataset directory, regardless of the form of database.
+    const dbPath = contents.datasetUri.fsPath;
+    const dbSchemeFiles = await getDbSchemeFiles(dbPath);
+    if (dbSchemeFiles.length === 0) {
+      throw new InvalidDatabaseError(
+        `Database '${databasePath}' does not contain a CodeQL dbscheme under '${dbPath}'.`,
+      );
+    } else if (dbSchemeFiles.length > 1) {
+      throw new InvalidDatabaseError(
+        `Database '${databasePath}' contains multiple CodeQL dbschemes under '${dbPath}'.`,
+      );
+    } else {
+      const dbSchemeUri = vscode.Uri.file(resolve(dbPath, dbSchemeFiles[0]));
+      return {
+        ...contents,
+        dbSchemeUri,
+      };
+    }
+  }
+
+  public static async resolveDatabase(
+    databasePath: string,
+  ): Promise<DatabaseContents> {
+    const name = basename(databasePath);
+
+    // Look for dataset and source archive.
+    const datasetUri = await findDataset(databasePath);
+    const sourceArchiveUri = await findSourceArchive(databasePath);
+
+    return {
+      kind: DatabaseKind.Database,
+      name,
+      datasetUri,
+      sourceArchiveUri,
+    };
+  }
+}
+
+/**
+ * An error thrown when we cannot find a valid database in a putative
+ * database directory.
+ */
+class InvalidDatabaseError extends Error {}
+
+async function findDataset(parentDirectory: string): Promise<vscode.Uri> {
+  /*
+   * Look directly in the root
+   */
+  let dbRelativePaths = await glob("db-*/", {
+    cwd: parentDirectory,
+  });
+
+  if (dbRelativePaths.length === 0) {
+    /*
+     * Check If they are in the old location
+     */
+    dbRelativePaths = await glob("working/db-*/", {
+      cwd: parentDirectory,
+    });
+  }
+  if (dbRelativePaths.length === 0) {
+    throw new InvalidDatabaseError(
+      `'${parentDirectory}' does not contain a dataset directory.`,
+    );
+  }
+
+  const dbAbsolutePath = join(parentDirectory, dbRelativePaths[0]);
+  if (dbRelativePaths.length > 1) {
+    void showAndLogWarningMessage(
+      `Found multiple dataset directories in database, using '${dbAbsolutePath}'.`,
+    );
+  }
+
+  return vscode.Uri.file(dbAbsolutePath);
+}
+
+/** Gets the relative paths of all `.dbscheme` files in the given directory. */
+async function getDbSchemeFiles(dbDirectory: string): Promise<string[]> {
+  return await glob("*.dbscheme", { cwd: dbDirectory });
+}
+
+// exported for testing
+export async function findSourceArchive(
+  databasePath: string,
+): Promise<vscode.Uri | undefined> {
+  const relativePaths = ["src", "output/src_archive"];
+
+  for (const relativePath of relativePaths) {
+    const basePath = join(databasePath, relativePath);
+    const zipPath = `${basePath}.zip`;
+
+    // Prefer using a zip archive over a directory.
+    if (await pathExists(zipPath)) {
+      return encodeArchiveBasePath(zipPath);
+    } else if (await pathExists(basePath)) {
+      return vscode.Uri.file(basePath);
+    }
+  }
+
+  void showAndLogInformationMessage(
+    `Could not find source archive for database '${databasePath}'. Assuming paths are absolute.`,
+  );
+  return undefined;
+}

extensions/ql-vscode/src/databases/local-databases/index.ts

@@ -0,0 +1,11 @@
+export {
+  DatabaseContents,
+  DatabaseContentsWithDbScheme,
+  DatabaseKind,
+} from "./database-contents";
+export { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
+export { DatabaseItem } from "./database-item";
+export { DatabaseItemImpl } from "./database-item-impl";
+export { DatabaseManager } from "./database-manager";
+export { DatabaseResolver } from "./database-resolver";
+export { DatabaseOptions, FullDatabaseOptions } from "./database-options";

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -1,5 +1,5 @@
 import {
-  commands,
+  ProgressLocation,
   QuickPickItem,
   TreeView,
   TreeViewExpansionEvent,
@@ -7,14 +7,17 @@ import {
   window,
   workspace,
 } from "vscode";
-import { commandRunner, UserCancellationException } from "../../commandRunner";
+import { UserCancellationException } from "../../common/vscode/progress";
 import {
   getNwoFromGitHubUrl,
   isValidGitHubNwo,
   getOwnerFromGitHubUrl,
   isValidGitHubOwner,
 } from "../../common/github-url-identifier-helper";
-import { showAndLogErrorMessage } from "../../helpers";
+import {
+  showAndLogErrorMessage,
+  showAndLogInformationMessage,
+} from "../../helpers";
 import { DisposableObject } from "../../pure/disposable-object";
 import {
   DbItem,
@@ -31,14 +34,21 @@ import { DbTreeViewItem } from "./db-tree-view-item";
 import { getGitHubUrl } from "./db-tree-view-item-action";
 import { getControllerRepo } from "../../variant-analysis/run-remote-query";
 import { getErrorMessage } from "../../pure/helpers-pure";
-import { Credentials } from "../../common/authentication";
+import { DatabasePanelCommands } from "../../common/commands";
+import { App } from "../../common/app";
+import { QueryLanguage } from "../../common/query-language";
+import { getCodeSearchRepositories } from "../code-search-api";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
-  kind: string;
+  remoteDatabaseKind: string;
 }
 
 export interface AddListQuickPickItem extends QuickPickItem {
-  kind: DbListKind;
+  databaseKind: DbListKind;
+}
+
+export interface CodeSearchQuickPickItem extends QuickPickItem {
+  language: string;
 }
 
 export class DbPanel extends DisposableObject {
@@ -46,8 +56,8 @@ export class DbPanel extends DisposableObject {
   private readonly treeView: TreeView<DbTreeViewItem>;
 
   public constructor(
+    private readonly app: App,
     private readonly dbManager: DbManager,
-    private readonly credentials: Credentials,
   ) {
     super();
 
@@ -72,58 +82,30 @@ export class DbPanel extends DisposableObject {
     this.push(this.treeView);
   }
 
-  public async initialize(): Promise<void> {
-    this.push(
-      commandRunner("codeQLVariantAnalysisRepositories.openConfigFile", () =>
-        this.openConfigFile(),
-      ),
-    );
-    this.push(
-      commandRunner("codeQLVariantAnalysisRepositories.addNewDatabase", () =>
-        this.addNewRemoteDatabase(),
-      ),
-    );
-    this.push(
-      commandRunner("codeQLVariantAnalysisRepositories.addNewList", () =>
-        this.addNewList(),
-      ),
-    );
-    this.push(
-      commandRunner(
-        "codeQLVariantAnalysisRepositories.setSelectedItem",
-        (treeViewItem: DbTreeViewItem) => this.setSelectedItem(treeViewItem),
-      ),
-    );
-    this.push(
-      commandRunner(
-        "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu",
-        (treeViewItem: DbTreeViewItem) => this.setSelectedItem(treeViewItem),
-      ),
-    );
-    this.push(
-      commandRunner(
-        "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu",
-        (treeViewItem: DbTreeViewItem) => this.openOnGitHub(treeViewItem),
-      ),
-    );
-    this.push(
-      commandRunner(
-        "codeQLVariantAnalysisRepositories.renameItemContextMenu",
-        (treeViewItem: DbTreeViewItem) => this.renameItem(treeViewItem),
-      ),
-    );
-    this.push(
-      commandRunner(
-        "codeQLVariantAnalysisRepositories.removeItemContextMenu",
-        (treeViewItem: DbTreeViewItem) => this.removeItem(treeViewItem),
-      ),
-    );
-    this.push(
-      commandRunner(
-        "codeQLVariantAnalysisRepositories.setupControllerRepository",
-        () => this.setupControllerRepository(),
-      ),
-    );
+  public getCommands(): DatabasePanelCommands {
+    return {
+      "codeQLVariantAnalysisRepositories.openConfigFile":
+        this.openConfigFile.bind(this),
+      "codeQLVariantAnalysisRepositories.addNewDatabase":
+        this.addNewRemoteDatabase.bind(this),
+      "codeQLVariantAnalysisRepositories.addNewList":
+        this.addNewList.bind(this),
+      "codeQLVariantAnalysisRepositories.setupControllerRepository":
+        this.setupControllerRepository.bind(this),
+
+      "codeQLVariantAnalysisRepositories.setSelectedItem":
+        this.setSelectedItem.bind(this),
+      "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu":
+        this.setSelectedItem.bind(this),
+      "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu":
+        this.openOnGitHub.bind(this),
+      "codeQLVariantAnalysisRepositories.renameItemContextMenu":
+        this.renameItem.bind(this),
+      "codeQLVariantAnalysisRepositories.removeItemContextMenu":
+        this.removeItem.bind(this),
+      "codeQLVariantAnalysisRepositories.importFromCodeSearch":
+        this.importFromCodeSearch.bind(this),
+    };
   }
 
   private async openConfigFile(): Promise<void> {
@@ -143,19 +125,19 @@ export class DbPanel extends DisposableObject {
     ) {
       await this.addNewRemoteRepo(highlightedItem.parentListName);
     } else {
-      const quickPickItems = [
+      const quickPickItems: RemoteDatabaseQuickPickItem[] = [
         {
           label: "$(repo) From a GitHub repository",
           detail: "Add a variant analysis repository from GitHub",
           alwaysShow: true,
-          kind: "repo",
+          remoteDatabaseKind: "repo",
         },
         {
           label: "$(organization) All repositories of a GitHub org or owner",
           detail:
             "Add a variant analysis list of repositories from a GitHub organization/owner",
           alwaysShow: true,
-          kind: "owner",
+          remoteDatabaseKind: "owner",
         },
       ];
       const databaseKind =
@@ -172,9 +154,9 @@ export class DbPanel extends DisposableObject {
         // We set 'true' to make this a silent exception.
         throw new UserCancellationException("No repository selected", true);
       }
-      if (databaseKind.kind === "repo") {
+      if (databaseKind.remoteDatabaseKind === "repo") {
         await this.addNewRemoteRepo();
-      } else if (databaseKind.kind === "owner") {
+      } else if (databaseKind.remoteDatabaseKind === "owner") {
         await this.addNewRemoteOwner();
       }
     }
@@ -201,7 +183,14 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await this.dbManager.addNewRemoteRepo(nwo, parentList);
+    const truncatedRepositories = await this.dbManager.addNewRemoteRepo(
+      nwo,
+      parentList,
+    );
+
+    if (parentList) {
+      this.reportAnyTruncatedRepos(truncatedRepositories, parentList);
+    }
   }
 
   private async addNewRemoteOwner(): Promise<void> {
@@ -353,6 +342,99 @@ export class DbPanel extends DisposableObject {
     await this.dbManager.removeDbItem(treeViewItem.dbItem);
   }
 
+  private async importFromCodeSearch(
+    treeViewItem: DbTreeViewItem,
+  ): Promise<void> {
+    if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) {
+      throw new Error("Please select a valid list to add code search results.");
+    }
+
+    const listName = treeViewItem.dbItem.listName;
+
+    const languageQuickPickItems: CodeSearchQuickPickItem[] = [
+      {
+        label: "No specific language",
+        alwaysShow: true,
+        language: "",
+      },
+    ].concat(
+      Object.values(QueryLanguage).map((language) => ({
+        label: language.toString(),
+        alwaysShow: true,
+        language: language.toString(),
+      })),
+    );
+
+    const codeSearchLanguage =
+      await window.showQuickPick<CodeSearchQuickPickItem>(
+        languageQuickPickItems,
+        {
+          title: "Select a language for your search",
+          placeHolder: "Select an option",
+          ignoreFocusOut: true,
+        },
+      );
+    if (!codeSearchLanguage) {
+      return;
+    }
+
+    const languagePrompt = codeSearchLanguage.language
+      ? `language:${codeSearchLanguage.language}`
+      : "";
+
+    const codeSearchQuery = await window.showInputBox({
+      title: "GitHub Code Search",
+      prompt:
+        "Use [GitHub's Code Search syntax](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax), including code qualifiers, regular expressions, and boolean operations, to search for repositories.",
+      placeHolder: "org:github",
+    });
+    if (codeSearchQuery === undefined || codeSearchQuery === "") {
+      return;
+    }
+
+    void window.withProgress(
+      {
+        location: ProgressLocation.Notification,
+        title: "Searching for repositories... This might take a while",
+        cancellable: true,
+      },
+      async (progress, token) => {
+        progress.report({ increment: 10 });
+
+        const repositories = await getCodeSearchRepositories(
+          `${codeSearchQuery} ${languagePrompt}`,
+          progress,
+          token,
+          this.app.credentials,
+        );
+
+        token.onCancellationRequested(() => {
+          void showAndLogInformationMessage("Code search cancelled");
+          return;
+        });
+
+        progress.report({ increment: 10, message: "Processing results..." });
+
+        const truncatedRepositories =
+          await this.dbManager.addNewRemoteReposToList(repositories, listName);
+        this.reportAnyTruncatedRepos(truncatedRepositories, listName);
+      },
+    );
+  }
+
+  private reportAnyTruncatedRepos(
+    truncatedRepositories: string[],
+    listName: string,
+  ) {
+    if (truncatedRepositories.length > 0) {
+      void showAndLogErrorMessage(
+        `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join(
+          ", ",
+        )}`,
+      );
+    }
+  }
+
   private async onDidCollapseElement(
     event: TreeViewExpansionEvent<DbTreeViewItem>,
   ): Promise<void> {
@@ -398,13 +480,13 @@ export class DbPanel extends DisposableObject {
       );
     }
 
-    await commands.executeCommand("vscode.open", Uri.parse(githubUrl));
+    await this.app.commands.execute("vscode.open", Uri.parse(githubUrl));
   }
 
   private async setupControllerRepository(): Promise<void> {
     try {
       // This will also validate that the controller repository is valid
-      await getControllerRepo(this.credentials);
+      await getControllerRepo(this.app.credentials);
     } catch (e: unknown) {
       if (e instanceof UserCancellationException) {
         return;

extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts

@@ -4,7 +4,8 @@ export type DbTreeViewItemAction =
   | "canBeSelected"
   | "canBeRemoved"
   | "canBeRenamed"
-  | "canBeOpenedOnGitHub";
+  | "canBeOpenedOnGitHub"
+  | "canImportCodeSearch";
 
 export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   const actions: DbTreeViewItemAction[] = [];
@@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   if (canBeOpenedOnGitHub(dbItem)) {
     actions.push("canBeOpenedOnGitHub");
   }
-
+  if (canImportCodeSearch(dbItem)) {
+    actions.push("canImportCodeSearch");
+  }
   return actions;
 }
 
@@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean {
   return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind);
 }
 
+function canImportCodeSearch(dbItem: DbItem): boolean {
+  return DbItemKind.RemoteUserDefinedList === dbItem.kind;
+}
+
 export function getGitHubUrl(dbItem: DbItem): string | undefined {
   switch (dbItem.kind) {
     case DbItemKind.RemoteOwner:

extensions/ql-vscode/src/debugger/debug-configuration.ts

@@ -0,0 +1,132 @@
+import {
+  CancellationToken,
+  DebugConfiguration,
+  DebugConfigurationProvider,
+  WorkspaceFolder,
+} from "vscode";
+import { getOnDiskWorkspaceFolders, showAndLogErrorMessage } from "../helpers";
+import { LocalQueries } from "../local-queries";
+import { getQuickEvalContext, validateQueryPath } from "../run-queries-shared";
+import * as CodeQLProtocol from "./debug-protocol";
+import { getErrorMessage } from "../pure/helpers-pure";
+
+/**
+ * The CodeQL launch arguments, as specified in "launch.json".
+ */
+export interface QLDebugArgs {
+  query?: string;
+  database?: string;
+  additionalPacks?: string[] | string;
+  extensionPacks?: string[] | string;
+  quickEval?: boolean;
+  noDebug?: boolean;
+}
+
+/**
+ * The debug configuration for a CodeQL configuration.
+ *
+ * This just combines `QLDebugArgs` with the standard debug configuration properties.
+ */
+export type QLDebugConfiguration = DebugConfiguration & QLDebugArgs;
+
+/**
+ * A CodeQL debug configuration after all variables and defaults have been resolved. This is what
+ * is passed to the debug adapter via the `launch` request.
+ */
+export type QLResolvedDebugConfiguration = DebugConfiguration &
+  CodeQLProtocol.LaunchConfig;
+
+/** If the specified value is a single element, then turn it into an array containing that element. */
+function makeArray<T extends Exclude<any, any[]>>(value: T | T[]): T[] {
+  if (Array.isArray(value)) {
+    return value;
+  } else {
+    return [value];
+  }
+}
+
+/**
+ * Implementation of `DebugConfigurationProvider` for CodeQL.
+ */
+export class QLDebugConfigurationProvider
+  implements DebugConfigurationProvider
+{
+  public constructor(private readonly localQueries: LocalQueries) {}
+
+  public resolveDebugConfiguration(
+    _folder: WorkspaceFolder | undefined,
+    debugConfiguration: DebugConfiguration,
+    _token?: CancellationToken,
+  ): DebugConfiguration {
+    const qlConfiguration = <QLDebugConfiguration>debugConfiguration;
+
+    // Fill in defaults for properties whose default value is a command invocation. VS Code will
+    // invoke any commands to fill in  actual values, then call
+    // `resolveDebugConfigurationWithSubstitutedVariables()`with the result.
+    const resultConfiguration: QLDebugConfiguration = {
+      ...qlConfiguration,
+      query: qlConfiguration.query ?? "${command:currentQuery}",
+      database: qlConfiguration.database ?? "${command:currentDatabase}",
+    };
+
+    return resultConfiguration;
+  }
+
+  public async resolveDebugConfigurationWithSubstitutedVariables(
+    _folder: WorkspaceFolder | undefined,
+    debugConfiguration: DebugConfiguration,
+    _token?: CancellationToken,
+  ): Promise<DebugConfiguration | null> {
+    try {
+      const qlConfiguration = debugConfiguration as QLDebugConfiguration;
+      if (qlConfiguration.query === undefined) {
+        throw new Error("No query was specified in the debug configuration.");
+      }
+      if (qlConfiguration.database === undefined) {
+        throw new Error(
+          "No database was specified in the debug configuration.",
+        );
+      }
+
+      // Fill in defaults here, instead of in `resolveDebugConfiguration`, to avoid the highly
+      // unusual case where one of the computed default values looks like a variable substitution.
+      const additionalPacks = makeArray(
+        qlConfiguration.additionalPacks ?? getOnDiskWorkspaceFolders(),
+      );
+
+      // Default to computing the extension packs based on the extension configuration and the search
+      // path.
+      const extensionPacks = makeArray(
+        qlConfiguration.extensionPacks ??
+          (await this.localQueries.getDefaultExtensionPacks(additionalPacks)),
+      );
+
+      const quickEval = qlConfiguration.quickEval ?? false;
+      validateQueryPath(qlConfiguration.query, quickEval);
+
+      const quickEvalContext = quickEval
+        ? await getQuickEvalContext(undefined, false)
+        : undefined;
+
+      const resultConfiguration: QLResolvedDebugConfiguration = {
+        name: qlConfiguration.name,
+        request: qlConfiguration.request,
+        type: qlConfiguration.type,
+        query: qlConfiguration.query,
+        database: qlConfiguration.database,
+        additionalPacks,
+        extensionPacks,
+        quickEvalContext,
+        noDebug: qlConfiguration.noDebug ?? false,
+      };
+
+      return resultConfiguration;
+    } catch (e) {
+      // Any unhandled exception will result in an OS-native error message box, which seems ugly.
+      // We'll just show a real VS Code error message, then return null to prevent the debug session
+      // from starting.
+      void showAndLogErrorMessage(getErrorMessage(e));
+      return null;
+    }
+  }
+}

extensions/ql-vscode/src/debugger/debug-protocol.ts

@@ -0,0 +1,102 @@
+import { DebugProtocol } from "@vscode/debugprotocol";
+import { QueryResultType } from "../pure/new-messages";
+import { QuickEvalContext } from "../run-queries-shared";
+
+// Events
+
+export type Event = { type: "event" };
+
+export type StoppedEvent = DebugProtocol.StoppedEvent &
+  Event & { event: "stopped" };
+
+export type InitializedEvent = DebugProtocol.InitializedEvent &
+  Event & { event: "initialized" };
+
+export type ExitedEvent = DebugProtocol.ExitedEvent &
+  Event & { event: "exited" };
+
+export type OutputEvent = DebugProtocol.OutputEvent &
+  Event & { event: "output" };
+
+/**
+ * Custom event to provide additional information about a running evaluation.
+ */
+export interface EvaluationStartedEvent extends Event {
+  event: "codeql-evaluation-started";
+  body: {
+    id: string;
+    outputDir: string;
+    quickEvalContext: QuickEvalContext | undefined;
+  };
+}
+
+/**
+ * Custom event to provide additional information about a completed evaluation.
+ */
+export interface EvaluationCompletedEvent extends Event {
+  event: "codeql-evaluation-completed";
+  body: {
+    resultType: QueryResultType;
+    message: string | undefined;
+    evaluationTime: number;
+  };
+}
+
+export type AnyEvent =
+  | StoppedEvent
+  | ExitedEvent
+  | InitializedEvent
+  | OutputEvent
+  | EvaluationStartedEvent
+  | EvaluationCompletedEvent;
+
+// Requests
+
+export type Request = DebugProtocol.Request & { type: "request" };
+
+export type InitializeRequest = DebugProtocol.InitializeRequest &
+  Request & { command: "initialize" };
+
+export interface LaunchConfig {
+  /** Full path to query (.ql) file. */
+  query: string;
+  /** Full path to the database directory. */
+  database: string;
+  /** Full paths to `--additional-packs` directories. */
+  additionalPacks: string[];
+  /** Pack names of extension packs. */
+  extensionPacks: string[];
+  /** Optional quick evaluation context. */
+  quickEvalContext: QuickEvalContext | undefined;
+  /** Run the query without debugging it. */
+  noDebug: boolean;
+}
+
+export interface LaunchRequest extends Request, DebugProtocol.LaunchRequest {
+  type: "request";
+  command: "launch";
+  arguments: DebugProtocol.LaunchRequestArguments & LaunchConfig;
+}
+
+export interface QuickEvalRequest extends Request {
+  command: "codeql-quickeval";
+  arguments: {
+    quickEvalContext: QuickEvalContext;
+  };
+}
+
+export type AnyRequest = InitializeRequest | LaunchRequest | QuickEvalRequest;
+
+// Responses
+
+export type Response = DebugProtocol.Response & { type: "response" };
+
+export type InitializeResponse = DebugProtocol.InitializeResponse &
+  Response & { command: "initialize" };
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface QuickEvalResponse extends Response {}
+
+export type AnyResponse = InitializeResponse | QuickEvalResponse;
+
+export type AnyProtocolMessage = AnyEvent | AnyRequest | AnyResponse;